From 8a4ab611bebc81091aedee94a8218de92083b092 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 11 Feb 2026 10:50:11 -0500 Subject: [PATCH] Broad service settings refactor, with the primary aim of making service settings discoverable and strongly-typed. Service settings can be updated at runtime with `*UpdateSettingsFrame`s. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Does not (yet) touch `InputParams`, to avoid scope creep and touching something currently part of the public API. But there is a lot of overlap between `*Settings` object fields and `InputParams` fields. Other than discoverability/typing, these are some other improvements brought by this refactor: - There is now a single code path (see `_update_settings_from_typed`) where services can respond to settings changes (by, say, reconnecting if needed), improving maintainability and guaranteeing one and only one reconnection no matter which settings changed - `set_language`/`set_model`/`set_voice`—which we're assuming are usable as public methods, though *not* recommended over `*UpdateSettingsFrame`—all use the same code path as settings updates. They're also now all consistent in that, if a service needs to respond to a change (by, say, reconnecting if needed), any of these methods will kick off that process. Note that this is technically a behavior change. - Several services now properly react to changed settings by reconnecting: - `AWSTranscribeSTTService` - `AzureSTTService` - `SonioxSTTService` - `GladiaSTTService` - `SpeechmaticsSTTService` - `AssemblyAISTTService` - `CartesiaSTTService` - `FishAudioTTSService` (would previously only reconnect when `model` changed) - `GoogleSTTService` - `SpeechmaticsSTTService` (which previously only handled *some* settings updates through a nonstandard public `update_params` method) - `GradiumSTTService` - `NvidiaSegmentedSTTService` (which previously only handled changes to language) - Bookkeeping across various services has been reduced, mostly by deduping ivars; the `self._settings` ivar is treated as the source of truth NOTE: I pretty much guarantee that there are services missed in this PR in terms of bringing to consistency with how updates are handled (like whether changes in certain fields trigger reconnects when they need to). We can squash remaining inconsistencies as we stumble onto them, service by service. The goal here is to get things *mostly* in order, and establish the infrastructure and patterns we'll need going forward. --- .claude/skills/cleanup/SKILL.md | 2 +- .../35-pattern-pair-voice-switching.py | 2 +- src/pipecat/frames/frames.py | 11 +- src/pipecat/services/ai_service.py | 41 ++- src/pipecat/services/anthropic/llm.py | 77 +++-- src/pipecat/services/assemblyai/stt.py | 60 +++- src/pipecat/services/asyncai/tts.py | 67 ++-- src/pipecat/services/aws/llm.py | 56 ++-- src/pipecat/services/aws/stt.py | 73 +++-- src/pipecat/services/aws/tts.py | 60 ++-- src/pipecat/services/azure/stt.py | 52 ++- src/pipecat/services/azure/tts.py | 87 +++-- src/pipecat/services/camb/tts.py | 33 +- src/pipecat/services/cartesia/stt.py | 41 ++- src/pipecat/services/cartesia/tts.py | 153 +++++---- src/pipecat/services/cerebras/llm.py | 10 +- src/pipecat/services/deepgram/stt.py | 85 +++-- .../services/deepgram/stt_sagemaker.py | 83 +++-- src/pipecat/services/deepgram/tts.py | 37 ++- src/pipecat/services/deepseek/llm.py | 12 +- src/pipecat/services/elevenlabs/stt.py | 203 +++++++----- src/pipecat/services/elevenlabs/tts.py | 272 +++++++++++----- src/pipecat/services/fal/stt.py | 60 ++-- src/pipecat/services/fireworks/llm.py | 12 +- src/pipecat/services/fish/tts.py | 81 +++-- src/pipecat/services/gladia/stt.py | 86 +++-- .../services/google/gemini_live/llm.py | 117 ++++--- src/pipecat/services/google/llm.py | 52 +-- src/pipecat/services/google/stt.py | 241 +++++++++----- src/pipecat/services/google/tts.py | 201 +++++++----- src/pipecat/services/gradium/stt.py | 46 ++- src/pipecat/services/gradium/tts.py | 47 +-- src/pipecat/services/grok/realtime/llm.py | 65 ++-- src/pipecat/services/groq/tts.py | 33 +- src/pipecat/services/hathora/stt.py | 35 +- src/pipecat/services/hathora/tts.py | 38 ++- src/pipecat/services/hume/tts.py | 4 +- src/pipecat/services/inworld/tts.py | 120 ++++--- src/pipecat/services/kokoro/tts.py | 19 ++ src/pipecat/services/llm_service.py | 13 + src/pipecat/services/lmnt/tts.py | 29 +- src/pipecat/services/minimax/tts.py | 118 +++++-- src/pipecat/services/mistral/llm.py | 16 +- src/pipecat/services/neuphonic/tts.py | 61 ++-- src/pipecat/services/nvidia/stt.py | 128 ++++---- src/pipecat/services/nvidia/tts.py | 2 +- src/pipecat/services/openai/base_llm.py | 62 ++-- src/pipecat/services/openai/realtime/llm.py | 60 +++- src/pipecat/services/openai/stt.py | 51 ++- src/pipecat/services/openai/tts.py | 44 ++- .../services/openai_realtime_beta/openai.py | 51 ++- src/pipecat/services/perplexity/llm.py | 20 +- src/pipecat/services/playht/tts.py | 88 +++-- src/pipecat/services/resembleai/tts.py | 38 ++- src/pipecat/services/rime/tts.py | 279 +++++++++------- src/pipecat/services/sambanova/llm.py | 10 +- src/pipecat/services/sarvam/stt.py | 145 ++++++--- src/pipecat/services/sarvam/tts.py | 187 ++++++++--- src/pipecat/services/settings.py | 297 +++++++++++++++++ src/pipecat/services/soniox/stt.py | 76 ++++- src/pipecat/services/speechmatics/stt.py | 245 +++++++++++--- src/pipecat/services/speechmatics/tts.py | 2 +- src/pipecat/services/stt_service.py | 52 ++- src/pipecat/services/tts_service.py | 72 +++- src/pipecat/services/ultravox/llm.py | 24 +- src/pipecat/services/whisper/base_stt.py | 61 ++-- src/pipecat/services/whisper/stt.py | 79 +++-- src/pipecat/services/xtts/tts.py | 32 +- tests/test_settings.py | 308 ++++++++++++++++++ 69 files changed, 3943 insertions(+), 1481 deletions(-) create mode 100644 src/pipecat/services/settings.py create mode 100644 tests/test_settings.py diff --git a/.claude/skills/cleanup/SKILL.md b/.claude/skills/cleanup/SKILL.md index f7dd6ea98..c0f4945b7 100644 --- a/.claude/skills/cleanup/SKILL.md +++ b/.claude/skills/cleanup/SKILL.md @@ -293,7 +293,7 @@ class NewTTSService(TTSService): """ super().__init__(**kwargs) self._api_key = api_key - self.set_voice(voice) + self._voice_id = voice ``` --- diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 4b269ac3e..cacc04459 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # First flush any existing audio to finish the current context await tts.flush_audio() # Then set the new voice - tts.set_voice(VOICE_IDS[voice_name]) + await tts.set_voice(VOICE_IDS[voice_name]) logger.info(f"Switched to {voice_name} voice") else: logger.warning(f"Unknown voice: {voice_name}") diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 5634d79ee..dd12929b9 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -42,6 +42,7 @@ from pipecat.utils.utils import obj_count, obj_id if TYPE_CHECKING: from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven from pipecat.processors.frame_processor import FrameProcessor + from pipecat.services.settings import ServiceSettings class DeprecatedKeypadEntry: @@ -2112,13 +2113,17 @@ class TTSStoppedFrame(ControlFrame): class ServiceUpdateSettingsFrame(ControlFrame): """Base frame for updating service settings. - A control frame containing a request to update service settings. + Supports both the legacy ``settings`` dict and the new typed ``update`` + object. When both are provided, ``update`` takes precedence. Parameters: - settings: Dictionary of setting name to value mappings. + settings: Dictionary of setting name to value mappings (legacy). + update: Typed :class:`~pipecat.services.settings.ServiceSettings` + object describing the delta to apply. """ - settings: Mapping[str, Any] + settings: Mapping[str, Any] = field(default_factory=dict) + update: Optional["ServiceSettings"] = None @dataclass diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index c03ab9d0e..97b7b6443 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -10,7 +10,7 @@ Provides the foundation for all AI services in the Pipecat framework, including model management, settings handling, and frame processing lifecycle methods. """ -from typing import Any, AsyncGenerator, Dict, Mapping +from typing import Any, AsyncGenerator, Dict, Mapping, Set from loguru import logger @@ -23,6 +23,7 @@ from pipecat.frames.frames import ( ) from pipecat.metrics.metrics import MetricsData from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.settings import ServiceSettings class AIService(FrameProcessor): @@ -42,7 +43,7 @@ class AIService(FrameProcessor): """ super().__init__(**kwargs) self._model_name: str = "" - self._settings: Dict[str, Any] = {} + self._settings: Dict[str, Any] | ServiceSettings = {} self._session_properties: Dict[str, Any] = {} @property @@ -135,6 +136,42 @@ class AIService(FrameProcessor): else: logger.warning(f"Unknown setting for {self.name} service: {key}") + async def _update_settings_from_typed(self, update: ServiceSettings) -> Set[str]: + """Apply a typed settings update and return the set of changed field names. + + If ``_settings`` is a :class:`ServiceSettings` object, the update is + applied to it and the changed-field set is returned. The ``model`` + field is handled specially: when it changes, ``set_model_name`` is + called. + + Services that have been migrated to typed settings should override + this method (calling ``super()``) to react to specific changed fields + (e.g. reconnect on voice change). + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + if not isinstance(self._settings, ServiceSettings): + logger.warning( + f"{self.name}: received typed settings update but _settings " + f"is not a ServiceSettings — falling back to dict-based update" + ) + await self._update_settings(update.to_dict()) + return set() + + changed = self._settings.apply_update(update) + + if "model" in changed: + self.set_model_name(self._settings.model) + + if changed: + logger.info(f"{self.name}: updated settings fields: {changed}") + + return changed + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle service lifecycle. diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index a21296fe3..36ee104f5 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -16,8 +16,8 @@ import copy import io import json import re -from dataclasses import dataclass -from typing import Any, Dict, List, Literal, Optional, Union +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Literal, Optional, Union import httpx from loguru import logger @@ -42,7 +42,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -59,6 +58,8 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN +from pipecat.services.settings import LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm try: @@ -69,6 +70,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AnthropicLLMSettings(LLMSettings): + """Typed settings for Anthropic LLM services. + + Parameters: + enable_prompt_caching: Whether to enable prompt caching. + thinking: Extended thinking configuration. + """ + + enable_prompt_caching: Any = field(default_factory=lambda: _NOT_GIVEN) + thinking: Any = field(default_factory=lambda: _NOT_GIVEN) + + @dataclass class AnthropicContextAggregatorPair: """Pair of context aggregators for Anthropic conversations. @@ -210,9 +224,10 @@ class AnthropicLLMService(LLMService): self.set_model_name(model) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout - self._settings = { - "max_tokens": params.max_tokens, - "enable_prompt_caching": ( + self._settings = AnthropicLLMSettings( + model=model, + max_tokens=params.max_tokens, + enable_prompt_caching=( params.enable_prompt_caching if params.enable_prompt_caching is not None else ( @@ -221,12 +236,12 @@ class AnthropicLLMService(LLMService): else False ) ), - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "thinking": params.thinking, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + thinking=params.thinking, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate usage metrics. @@ -280,7 +295,7 @@ class AnthropicLLMService(LLMService): if isinstance(context, LLMContext): adapter: AnthropicLLMAdapter = self.get_llm_adapter() invocation_params = adapter.get_llm_invocation_params( - context, enable_prompt_caching=self._settings["enable_prompt_caching"] + context, enable_prompt_caching=self._settings.enable_prompt_caching ) messages = invocation_params["messages"] system = invocation_params["system"] @@ -294,20 +309,20 @@ class AnthropicLLMService(LLMService): # Build params using the same method as streaming completions params = { "model": self.model_name, - "max_tokens": max_tokens if max_tokens is not None else self._settings["max_tokens"], + "max_tokens": max_tokens if max_tokens is not None else self._settings.max_tokens, "stream": False, - "temperature": self._settings["temperature"], - "top_k": self._settings["top_k"], - "top_p": self._settings["top_p"], + "temperature": self._settings.temperature, + "top_k": self._settings.top_k, + "top_p": self._settings.top_p, "messages": messages, "system": system, "tools": tools, "betas": ["interleaved-thinking-2025-05-14"], } - if self._settings["thinking"]: - params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True) + if self._settings.thinking: + params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True) - params.update(self._settings["extra"]) + params.update(self._settings.extra) # LLM completion response = await self._client.beta.messages.create(**params) @@ -358,14 +373,14 @@ class AnthropicLLMService(LLMService): if isinstance(context, LLMContext): adapter: AnthropicLLMAdapter = self.get_llm_adapter() params = adapter.get_llm_invocation_params( - context, enable_prompt_caching=self._settings["enable_prompt_caching"] + context, enable_prompt_caching=self._settings.enable_prompt_caching ) return params # Anthropic-specific context messages = ( context.get_messages_with_cache_control_markers() - if self._settings["enable_prompt_caching"] + if self._settings.enable_prompt_caching else context.messages ) return AnthropicLLMInvocationParams( @@ -408,21 +423,21 @@ class AnthropicLLMService(LLMService): params = { "model": self.model_name, - "max_tokens": self._settings["max_tokens"], + "max_tokens": self._settings.max_tokens, "stream": True, - "temperature": self._settings["temperature"], - "top_k": self._settings["top_k"], - "top_p": self._settings["top_p"], + "temperature": self._settings.temperature, + "top_k": self._settings.top_k, + "top_p": self._settings.top_p, } # Add thinking parameter if set - if self._settings["thinking"]: - params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True) + if self._settings.thinking: + params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True) # Messages, system, tools params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) # "Interleaved thinking" needed to allow thinking between sequences # of function calls, when extended thinking is enabled. @@ -576,11 +591,9 @@ class AnthropicLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = AnthropicLLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) elif isinstance(frame, LLMEnablePromptCachingFrame): logger.debug(f"Setting enable prompt caching to: [{frame.enable}]") - self._settings["enable_prompt_caching"] = frame.enable + self._settings.enable_prompt_caching = frame.enable else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 41a0ae2a0..278873fdf 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription. import asyncio import json +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, Optional from urllib.parse import urlencode @@ -29,6 +30,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import ASSEMBLYAI_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -52,6 +54,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AssemblyAISTTSettings(STTSettings): + """Typed settings for the AssemblyAI STT service. + + See :class:`AssemblyAIConnectionParams` for detailed parameter descriptions. + + Parameters: + connection_params: Connection configuration parameters. + """ + + connection_params: AssemblyAIConnectionParams = field(default_factory=lambda: NOT_GIVEN) + + class AssemblyAISTTService(WebsocketSTTService): """AssemblyAI real-time speech-to-text service. @@ -96,9 +111,11 @@ class AssemblyAISTTService(WebsocketSTTService): ) self._api_key = api_key - self._language = language + self._settings: AssemblyAISTTSettings = AssemblyAISTTSettings( + language=language, + connection_params=connection_params, + ) self._api_endpoint_base_url = api_endpoint_base_url - self._connection_params = connection_params self._vad_force_turn_endpoint = vad_force_turn_endpoint self._termination_event = asyncio.Event() @@ -165,6 +182,35 @@ class AssemblyAISTTService(WebsocketSTTService): """ return True + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and reconnect if anything changed. + + Any change triggers a WebSocket reconnect since all connection + parameters are encoded in the WebSocket URL. + + Args: + update: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # Re-apply manual turn mode config if vad_force_turn_endpoint is active + # and connection_params were updated. + if self._vad_force_turn_endpoint and "connection_params" in changed: + self._settings.connection_params = self._configure_manual_turn_mode( + self._settings.connection_params + ) + + await self._disconnect() + await self._connect() + + return changed + async def start(self, frame: StartFrame): """Start the speech-to-text service. @@ -239,7 +285,7 @@ class AssemblyAISTTService(WebsocketSTTService): def _build_ws_url(self) -> str: """Build WebSocket URL with query parameters using urllib.parse.urlencode.""" params = {} - for k, v in self._connection_params.model_dump().items(): + for k, v in self._settings.connection_params.model_dump().items(): if v is not None: if k == "keyterms_prompt": params[k] = json.dumps(v) @@ -415,18 +461,18 @@ class AssemblyAISTTService(WebsocketSTTService): if not message.transcript: return if message.end_of_turn and ( - not self._connection_params.formatted_finals or message.turn_is_formatted + not self._settings.connection_params.formatted_finals or message.turn_is_formatted ): await self.push_frame( TranscriptionFrame( message.transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, message, ) ) - await self._trace_transcription(message.transcript, True, self._language) + await self._trace_transcription(message.transcript, True, self._settings.language) await self.stop_processing_metrics() else: await self.push_frame( @@ -434,7 +480,7 @@ class AssemblyAISTTService(WebsocketSTTService): message.transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, message, ) ) diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index 4ff6c928d..aecf69a26 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -9,6 +9,7 @@ import asyncio import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import AudioContextTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -72,6 +74,21 @@ def language_to_async_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class AsyncAITTSSettings(TTSSettings): + """Typed settings for Async AI TTS services. + + Parameters: + output_container: Audio container format (e.g. "raw"). + output_encoding: Audio encoding format (e.g. "pcm_s16le"). + output_sample_rate: Audio sample rate in Hz. + """ + + output_container: str = field(default_factory=lambda: NOT_GIVEN) + output_encoding: str = field(default_factory=lambda: NOT_GIVEN) + output_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class AsyncAITTSService(AudioContextTTSService): """Async TTS service with WebSocket streaming. @@ -131,19 +148,21 @@ class AsyncAITTSService(AudioContextTTSService): self._api_key = api_key self._api_version = version self._url = url - self._settings = { - "output_format": { + self._settings: AsyncAITTSSettings = AsyncAITTSSettings( + model=model, + voice=voice_id, + output_format={ "container": container, "encoding": encoding, "sample_rate": 0, }, - "language": self.language_to_service_language(params.language) + language=self.language_to_service_language(params.language) if params.language else None, - } + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id self._receive_task = None self._keepalive_task = None @@ -179,7 +198,7 @@ class AsyncAITTSService(AudioContextTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -235,8 +254,12 @@ class AsyncAITTSService(AudioContextTTSService): init_msg = { "model_id": self._model_name, "voice": {"mode": "id", "id": self._voice_id}, - "output_format": self._settings["output_format"], - "language": self._settings["language"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, + "language": self._settings.language, } await self._get_websocket().send(json.dumps(init_msg)) @@ -454,17 +477,17 @@ class AsyncAIHttpTTSService(TTSService): self._api_key = api_key self._base_url = url self._api_version = version - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings: AsyncAITTSSettings = AsyncAITTSSettings( + model=model, + voice=voice_id, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - } - self.set_voice(voice_id) + ) + self._voice_id = voice_id self.set_model_name(model) self._session = aiohttp_session @@ -495,7 +518,7 @@ class AsyncAIHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -517,8 +540,12 @@ class AsyncAIHttpTTSService(TTSService): "model_id": self._model_name, "transcript": text, "voice": voice_config, - "output_format": self._settings["output_format"], - "language": self._settings["language"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, + "language": self._settings.language, } yield TTSStartedFrame(context_id=context_id) headers = { diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 1778ae74e..032cee060 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -18,8 +18,8 @@ import io import json import os import re -from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Optional from loguru import logger from PIL import Image @@ -40,7 +40,6 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, - LLMUpdateSettingsFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -57,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm try: @@ -71,6 +71,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AWSBedrockLLMSettings(LLMSettings): + """Typed settings for AWS Bedrock LLM services. + + Parameters: + latency: Performance mode - "standard" or "optimized". + additional_model_request_fields: Additional model-specific parameters. + """ + + latency: Any = field(default_factory=lambda: NOT_GIVEN) + additional_model_request_fields: Any = field(default_factory=lambda: NOT_GIVEN) + + @dataclass class AWSBedrockContextAggregatorPair: """Container for AWS Bedrock context aggregators. @@ -806,15 +819,16 @@ class AWSBedrockLLMService(LLMService): self.set_model_name(model) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout - self._settings = { - "max_tokens": params.max_tokens, - "temperature": params.temperature, - "top_p": params.top_p, - "latency": params.latency, - "additional_model_request_fields": params.additional_model_request_fields + self._settings = AWSBedrockLLMSettings( + model=model, + max_tokens=params.max_tokens, + temperature=params.temperature, + top_p=params.top_p, + latency=params.latency, + additional_model_request_fields=params.additional_model_request_fields if isinstance(params.additional_model_request_fields, dict) else {}, - } + ) logger.info(f"Using AWS Bedrock model: {model}") @@ -836,12 +850,12 @@ class AWSBedrockLLMService(LLMService): Dictionary containing only the inference parameters that are not None. """ inference_config = {} - if self._settings["max_tokens"] is not None: - inference_config["maxTokens"] = self._settings["max_tokens"] - if self._settings["temperature"] is not None: - inference_config["temperature"] = self._settings["temperature"] - if self._settings["top_p"] is not None: - inference_config["topP"] = self._settings["top_p"] + if self._settings.max_tokens is not None: + inference_config["maxTokens"] = self._settings.max_tokens + if self._settings.temperature is not None: + inference_config["temperature"] = self._settings.temperature + if self._settings.top_p is not None: + inference_config["topP"] = self._settings.top_p return inference_config async def run_inference( @@ -879,7 +893,7 @@ class AWSBedrockLLMService(LLMService): request_params = { "modelId": self.model_name, "messages": messages, - "additionalModelRequestFields": self._settings["additional_model_request_fields"], + "additionalModelRequestFields": self._settings.additional_model_request_fields, } if inference_config: @@ -1036,7 +1050,7 @@ class AWSBedrockLLMService(LLMService): request_params = { "modelId": self.model_name, "messages": messages, - "additionalModelRequestFields": self._settings["additional_model_request_fields"], + "additionalModelRequestFields": self._settings.additional_model_request_fields, } # Only add inference config if it has parameters @@ -1081,8 +1095,8 @@ class AWSBedrockLLMService(LLMService): request_params["toolConfig"] = tool_config # Add performance config if latency is specified - if self._settings["latency"] in ["standard", "optimized"]: - request_params["performanceConfig"] = {"latency": self._settings["latency"]} + if self._settings.latency in ["standard", "optimized"]: + request_params["performanceConfig"] = {"latency": self._settings.latency} # Log request params with messages redacted for logging if isinstance(context, LLMContext): @@ -1207,8 +1221,6 @@ class AWSBedrockLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = AWSBedrockLLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index f78bc4d4b..cb52da12a 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -14,6 +14,7 @@ import json import os import random import string +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -28,6 +29,7 @@ from pipecat.frames.frames import ( TranscriptionFrame, ) from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import AWS_TRANSCRIBE_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -43,6 +45,25 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AWSTranscribeSTTSettings(STTSettings): + """Typed settings for the AWS Transcribe STT service. + + Parameters: + sample_rate: Audio sample rate in Hz (8000 or 16000). + media_encoding: Audio encoding format (e.g. "linear16"). + number_of_channels: Number of audio channels. + show_speaker_label: Whether to show speaker labels. + enable_channel_identification: Whether to enable channel identification. + """ + + sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + media_encoding: str = field(default_factory=lambda: NOT_GIVEN) + number_of_channels: int = field(default_factory=lambda: NOT_GIVEN) + show_speaker_label: bool = field(default_factory=lambda: NOT_GIVEN) + enable_channel_identification: bool = field(default_factory=lambda: NOT_GIVEN) + + class AWSTranscribeSTTService(WebsocketSTTService): """AWS Transcribe Speech-to-Text service using WebSocket streaming. @@ -78,21 +99,21 @@ class AWSTranscribeSTTService(WebsocketSTTService): """ super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs) - self._settings = { - "sample_rate": sample_rate, - "language": language, - "media_encoding": "linear16", # AWS expects raw PCM - "number_of_channels": 1, - "show_speaker_label": False, - "enable_channel_identification": False, - } + self._settings: AWSTranscribeSTTSettings = AWSTranscribeSTTSettings( + language=language, + sample_rate=sample_rate, + media_encoding="linear16", + number_of_channels=1, + show_speaker_label=False, + enable_channel_identification=False, + ) # Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz if sample_rate not in [8000, 16000]: logger.warning( f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz." ) - self._settings["sample_rate"] = 16000 + self._settings.sample_rate = 16000 self._credentials = { "aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"), @@ -117,6 +138,20 @@ class AWSTranscribeSTTService(WebsocketSTTService): } return encoding_map.get(encoding, encoding) + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, reconnecting if needed. + + Any change to connection-relevant settings (model, language, etc.) + triggers a WebSocket reconnect so the new configuration takes effect. + """ + changed = await super()._update_settings_from_typed(update) + + if changed and self._websocket: + await self._disconnect() + await self._connect() + + return changed + async def start(self, frame: StartFrame): """Initialize the connection when the service starts. @@ -208,9 +243,9 @@ class AWSTranscribeSTTService(WebsocketSTTService): logger.debug("Connecting to AWS Transcribe WebSocket") - language_code = self.language_to_service_language(Language(self._settings["language"])) + language_code = self.language_to_service_language(Language(self._settings.language)) if not language_code: - raise ValueError(f"Unsupported language: {self._settings['language']}") + raise ValueError(f"Unsupported language: {self._settings.language}") # Generate random websocket key websocket_key = "".join( @@ -237,14 +272,14 @@ class AWSTranscribeSTTService(WebsocketSTTService): }, language_code=language_code, media_encoding=self.get_service_encoding( - self._settings["media_encoding"] + self._settings.media_encoding ), # Convert to AWS format - sample_rate=self._settings["sample_rate"], - number_of_channels=self._settings["number_of_channels"], + sample_rate=self._settings.sample_rate, + number_of_channels=self._settings.number_of_channels, enable_partial_results_stabilization=True, partial_results_stability="high", - show_speaker_label=self._settings["show_speaker_label"], - enable_channel_identification=self._settings["enable_channel_identification"], + show_speaker_label=self._settings.show_speaker_label, + enable_channel_identification=self._settings.enable_channel_identification, ) logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...") @@ -479,14 +514,14 @@ class AWSTranscribeSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, result=result, ) ) await self._handle_transcription( transcript, is_final, - self._settings["language"], + self._settings.language, ) await self.stop_processing_metrics() else: @@ -495,7 +530,7 @@ class AWSTranscribeSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, result=result, ) ) diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index b902564d2..5086b1469 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -11,6 +11,7 @@ supporting multiple languages, voices, and SSML features. """ import os +from dataclasses import dataclass, field from typing import AsyncGenerator, List, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -121,6 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class AWSPollyTTSSettings(TTSSettings): + """Typed settings for AWS Polly TTS service. + + Parameters: + engine: TTS engine to use ('standard', 'neural', etc.). + pitch: Voice pitch adjustment (for standard engine only). + rate: Speech rate adjustment. + volume: Voice volume adjustment. + lexicon_names: List of pronunciation lexicons to apply. + """ + + engine: str = field(default_factory=lambda: NOT_GIVEN) + pitch: str = field(default_factory=lambda: NOT_GIVEN) + rate: str = field(default_factory=lambda: NOT_GIVEN) + volume: str = field(default_factory=lambda: NOT_GIVEN) + lexicon_names: List[str] = field(default_factory=lambda: NOT_GIVEN) + + class AWSPollyTTSService(TTSService): """AWS Polly text-to-speech service. @@ -185,20 +206,21 @@ class AWSPollyTTSService(TTSService): } self._aws_session = aioboto3.Session() - self._settings = { - "engine": params.engine, - "language": self.language_to_service_language(params.language) + self._settings: AWSPollyTTSSettings = AWSPollyTTSSettings( + voice=voice_id, + engine=params.engine, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "pitch": params.pitch, - "rate": params.rate, - "volume": params.volume, - "lexicon_names": params.lexicon_names, - } + pitch=params.pitch, + rate=params.rate, + volume=params.volume, + lexicon_names=params.lexicon_names, + ) self._resampler = create_stream_resampler() - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -222,19 +244,19 @@ class AWSPollyTTSService(TTSService): def _construct_ssml(self, text: str) -> str: ssml = "" - language = self._settings["language"] + language = self._settings.language ssml += f"" prosody_attrs = [] # Prosody tags are only supported for standard and neural engines - if self._settings["engine"] == "standard": - if self._settings["pitch"]: - prosody_attrs.append(f"pitch='{self._settings['pitch']}'") + if self._settings.engine == "standard": + if self._settings.pitch: + prosody_attrs.append(f"pitch='{self._settings.pitch}'") - if self._settings["rate"]: - prosody_attrs.append(f"rate='{self._settings['rate']}'") - if self._settings["volume"]: - prosody_attrs.append(f"volume='{self._settings['volume']}'") + if self._settings.rate: + prosody_attrs.append(f"rate='{self._settings.rate}'") + if self._settings.volume: + prosody_attrs.append(f"volume='{self._settings.volume}'") if prosody_attrs: ssml += f"" @@ -276,10 +298,10 @@ class AWSPollyTTSService(TTSService): "TextType": "ssml", "OutputFormat": "pcm", "VoiceId": self._voice_id, - "Engine": self._settings["engine"], + "Engine": self._settings.engine, # AWS only supports 8000 and 16000 for PCM. We select 16000. "SampleRate": "16000", - "LexiconNames": self._settings["lexicon_names"], + "LexiconNames": self._settings.lexicon_names, } # Filter out None values diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index 1bc7ec70a..bf3f70653 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -11,6 +11,7 @@ Speech SDK for real-time audio transcription. """ import asyncio +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( TranscriptionFrame, ) from pipecat.services.azure.common import language_to_azure_language +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import AZURE_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -48,6 +50,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AzureSTTSettings(STTSettings): + """Typed settings for the Azure STT service. + + Parameters: + region: Azure region for the Speech service. + sample_rate: Audio sample rate in Hz. + """ + + region: str = field(default_factory=lambda: NOT_GIVEN) + sample_rate: Optional[int] = field(default_factory=lambda: NOT_GIVEN) + + class AzureSTTService(STTService): """Azure Speech-to-Text service for real-time audio transcription. @@ -92,11 +107,11 @@ class AzureSTTService(STTService): self._audio_stream = None self._speech_recognizer = None - self._settings = { - "region": region, - "language": language_to_azure_language(language), - "sample_rate": sample_rate, - } + self._settings: AzureSTTSettings = AzureSTTSettings( + region=region, + language=language_to_azure_language(language), + sample_rate=sample_rate, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate performance metrics. @@ -106,6 +121,29 @@ class AzureSTTService(STTService): """ return True + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, reconfiguring the recognizer if needed. + + When ``language`` changes the ``SpeechConfig`` is updated and the + speech recognizer is restarted so that the new language takes effect. + """ + changed = await super()._update_settings_from_typed(update) + + if "language" in changed: + # Convert Language enum to Azure language code if needed. + lang = self._settings.language + if isinstance(lang, Language): + lang = language_to_azure_language(lang) + self._settings.language = lang + self._speech_config.speech_recognition_language = lang + + # Restart the recognizer with the new config. + if self._speech_recognizer: + self._speech_recognizer.stop_continuous_recognition_async() + self._speech_recognizer.start_continuous_recognition_async() + + return changed + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process audio data for speech-to-text conversion. @@ -198,7 +236,7 @@ class AzureSTTService(STTService): def _on_handle_recognized(self, event): if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or self._settings.get("language") + language = getattr(event.result, "language", None) or self._settings.language frame = TranscriptionFrame( event.result.text, self._user_id, @@ -213,7 +251,7 @@ class AzureSTTService(STTService): def _on_handle_recognizing(self, event): if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or self._settings.get("language") + language = getattr(event.result, "language", None) or self._settings.language frame = InterimTranscriptionFrame( event.result.text, self._user_id, diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 7d4aa0253..04b51d10b 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -7,6 +7,7 @@ """Azure Cognitive Services Text-to-Speech service implementations.""" import asyncio +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.azure.common import language_to_azure_language +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService, WordTTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -65,6 +67,31 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma return sample_rate_map.get(sample_rate, SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm) +@dataclass +class AzureTTSSettings(TTSSettings): + """Typed settings for Azure TTS services. + + Parameters: + emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). + language: Language for synthesis. Defaults to English (US). + pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high"). + rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast"). + role: Voice role for expression (e.g., "YoungAdultFemale"). + style: Speaking style (e.g., "cheerful", "sad", "excited"). + style_degree: Intensity of the speaking style (0.01 to 2.0). + volume: Volume level (e.g., "+20%", "loud", "x-soft"). + """ + + emphasis: str = field(default_factory=lambda: NOT_GIVEN) + language: str = field(default_factory=lambda: NOT_GIVEN) + pitch: str = field(default_factory=lambda: NOT_GIVEN) + rate: str = field(default_factory=lambda: NOT_GIVEN) + role: str = field(default_factory=lambda: NOT_GIVEN) + style: str = field(default_factory=lambda: NOT_GIVEN) + style_degree: str = field(default_factory=lambda: NOT_GIVEN) + volume: str = field(default_factory=lambda: NOT_GIVEN) + + class AzureBaseTTSService: """Base mixin class for Azure Cognitive Services text-to-speech implementations. @@ -126,18 +153,18 @@ class AzureBaseTTSService: """ params = params or AzureBaseTTSService.InputParams() - self._settings = { - "emphasis": params.emphasis, - "language": self.language_to_service_language(params.language) + self._settings: AzureTTSSettings = AzureTTSSettings( + emphasis=params.emphasis, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "pitch": params.pitch, - "rate": params.rate, - "role": params.role, - "style": params.style, - "style_degree": params.style_degree, - "volume": params.volume, - } + pitch=params.pitch, + rate=params.rate, + role=params.role, + style=params.style, + style_degree=params.style_degree, + volume=params.volume, + ) self._api_key = api_key self._region = region @@ -156,7 +183,7 @@ class AzureBaseTTSService: return language_to_azure_language(language) def _construct_ssml(self, text: str) -> str: - language = self._settings["language"] + language = self._settings.language # Escape special characters escaped_text = self._escape_text(text) @@ -169,38 +196,38 @@ class AzureBaseTTSService: "" ) - if self._settings["style"]: - ssml += f"" - if self._settings["emphasis"]: - ssml += f"" + if self._settings.emphasis: + ssml += f"" ssml += escaped_text - if self._settings["emphasis"]: + if self._settings.emphasis: ssml += "" if prosody_attrs: ssml += "" - if self._settings["style"]: + if self._settings.style: ssml += "" ssml += "" @@ -314,7 +341,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): subscription=self._api_key, region=self._region, ) - self._speech_config.speech_synthesis_language = self._settings["language"] + self._speech_config.speech_synthesis_language = self._settings.language self._speech_config.set_speech_synthesis_output_format( sample_rate_to_output_format(self.sample_rate) ) @@ -364,7 +391,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): Returns: True if the language is CJK, False otherwise. """ - language = self._settings.get("language", "").lower() + language = (self._settings.language if self._settings.language else "").lower() # Check if language starts with CJK language codes return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu")) @@ -735,7 +762,7 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService): subscription=self._api_key, region=self._region, ) - self._speech_config.speech_synthesis_language = self._settings["language"] + self._speech_config.speech_synthesis_language = self._settings.language self._speech_config.set_speech_synthesis_output_format( sample_rate_to_output_format(self.sample_rate) ) diff --git a/src/pipecat/services/camb/tts.py b/src/pipecat/services/camb/tts.py index def57d3a0..8a6f67231 100644 --- a/src/pipecat/services/camb/tts.py +++ b/src/pipecat/services/camb/tts.py @@ -16,7 +16,8 @@ Features: - Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz) """ -from typing import Any, AsyncGenerator, Dict, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Dict, Optional from camb import StreamTtsOutputConfiguration from camb.client import AsyncCambAI @@ -31,6 +32,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -133,6 +135,18 @@ def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]: return buffer[:aligned_size], buffer[aligned_size:] +@dataclass +class CambTTSSettings(TTSSettings): + """Typed settings for Camb.ai TTS service. + + Parameters: + user_instructions: Custom instructions for mars-instruct model only. + Ignored for other models. Max 1000 characters. + """ + + user_instructions: str = field(default_factory=lambda: NOT_GIVEN) + + class CambTTSService(TTSService): """Camb.ai MARS text-to-speech service using the official SDK. @@ -212,15 +226,16 @@ class CambTTSService(TTSService): ) # Build settings - self._settings = { - "language": ( + self._settings: CambTTSSettings = CambTTSSettings( + model=model, + voice=voice_id, + language=( self.language_to_service_language(params.language) if params.language else "en-us" ), - "user_instructions": params.user_instructions, - } + user_instructions=params.user_instructions, + ) self.set_model_name(model) - self.set_voice(str(voice_id)) self._voice_id = voice_id self._client = None @@ -283,14 +298,14 @@ class CambTTSService(TTSService): tts_kwargs: Dict[str, Any] = { "text": text, "voice_id": self._voice_id, - "language": self._settings["language"], + "language": self._settings.language, "speech_model": self.model_name, "output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"), } # Add user instructions if using mars-instruct model - if self._model_name == "mars-instruct" and self._settings.get("user_instructions"): - tts_kwargs["user_instructions"] = self._settings["user_instructions"] + if self._model_name == "mars-instruct" and self._settings.user_instructions: + tts_kwargs["user_instructions"] = self._settings.user_instructions await self.start_tts_usage_metrics(text) yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index c4429226f..624801bfb 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -12,6 +12,7 @@ the Cartesia Live transcription API for real-time speech recognition. import json import urllib.parse +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import CARTESIA_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -42,6 +44,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class CartesiaSTTSettings(STTSettings): + """Typed settings for the Cartesia STT service. + + Parameters: + encoding: Audio encoding format (e.g. ``"pcm_s16le"``). + """ + + encoding: str = field(default_factory=lambda: NOT_GIVEN) + + class CartesiaLiveOptions: """Configuration options for Cartesia Live STT service. @@ -181,7 +194,11 @@ class CartesiaSTTService(WebsocketSTTService): k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None" } - self._settings = merged_options + self._settings: CartesiaSTTSettings = CartesiaSTTSettings( + model=merged_options["model"], + language=merged_options.get("language"), + encoding=merged_options.get("encoding", "pcm_s16le"), + ) self.set_model_name(merged_options["model"]) self._api_key = api_key self._base_url = base_url or "api.cartesia.ai" @@ -275,13 +292,33 @@ class CartesiaSTTService(WebsocketSTTService): await self._disconnect_websocket() + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and reconnect if anything changed. + + Args: + update: A :class:`STTSettings` (or ``CartesiaSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + if changed: + await self._disconnect() + await self._connect() + return changed + async def _connect_websocket(self): try: if self._websocket and self._websocket.state is State.OPEN: return logger.debug("Connecting to Cartesia STT") - params = self._settings + params = { + "model": self._settings.model, + "language": self._settings.language, + "encoding": self._settings.encoding, + "sample_rate": str(self.sample_rate), + } ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 791c60a18..531aafdf7 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -9,8 +9,9 @@ import base64 import json import warnings +from dataclasses import dataclass, field from enum import Enum -from typing import AsyncGenerator, List, Literal, Optional +from typing import Any, AsyncGenerator, List, Literal, Optional from loguru import logger from pydantic import BaseModel, Field @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -191,6 +193,31 @@ class CartesiaEmotion(str, Enum): DETERMINED = "determined" +@dataclass +class CartesiaTTSSettings(TTSSettings): + """Typed settings for Cartesia TTS services. + + Parameters: + output_container: Audio container format (e.g. "raw"). + output_encoding: Audio encoding format (e.g. "pcm_s16le"). + output_sample_rate: Audio sample rate in Hz. + speed: Voice speed control for non-Sonic-3 models (literal values). + emotion: List of emotion controls for non-Sonic-3 models. + generation_config: Generation configuration for Sonic-3 models. Includes volume, + speed (numeric), and emotion (string) parameters. + pronunciation_dict_id: The ID of the pronunciation dictionary to use for + custom pronunciations. + """ + + output_container: str = field(default_factory=lambda: NOT_GIVEN) + output_encoding: str = field(default_factory=lambda: NOT_GIVEN) + output_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + speed: str = field(default_factory=lambda: NOT_GIVEN) + emotion: List[str] = field(default_factory=lambda: NOT_GIVEN) + generation_config: GenerationConfig = field(default_factory=lambda: NOT_GIVEN) + pronunciation_dict_id: str = field(default_factory=lambda: NOT_GIVEN) + + class CartesiaTTSService(AudioContextWordTTSService): """Cartesia TTS service with WebSocket streaming and word timestamps. @@ -289,22 +316,20 @@ class CartesiaTTSService(AudioContextWordTTSService): self._api_key = api_key self._cartesia_version = cartesia_version self._url = url - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings: CartesiaTTSSettings = CartesiaTTSSettings( + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - "speed": params.speed, - "emotion": params.emotion, - "generation_config": params.generation_config, - "pronunciation_dict_id": params.pronunciation_dict_id, - } + speed=params.speed, + emotion=params.emotion, + generation_config=params.generation_config, + pronunciation_dict_id=params.pronunciation_dict_id, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id self._context_id = None self._receive_task = None @@ -317,16 +342,6 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model. - - Args: - model: The model name to use for synthesis. - """ - self._model_id = model - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Cartesia language format. @@ -391,7 +406,7 @@ class CartesiaTTSService(AudioContextWordTTSService): Returns: List of (word, start_time) tuples processed for the language. """ - current_language = self._settings.get("language") + current_language = self._settings.language # Check if this is a CJK language (if language is None, treat as non-CJK) if current_language and self._is_cjk_language(current_language): @@ -414,7 +429,7 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_config["mode"] = "id" voice_config["id"] = self._voice_id - if self._settings["emotion"]: + if is_given(self._settings.emotion) and self._settings.emotion: with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( @@ -423,8 +438,7 @@ class CartesiaTTSService(AudioContextWordTTSService): stacklevel=2, ) voice_config["__experimental_controls"] = {} - if self._settings["emotion"]: - voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"] + voice_config["__experimental_controls"]["emotion"] = self._settings.emotion msg = { "transcript": text, @@ -432,24 +446,28 @@ class CartesiaTTSService(AudioContextWordTTSService): "context_id": self._context_id, "model_id": self.model_name, "voice": voice_config, - "output_format": self._settings["output_format"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, "add_timestamps": add_timestamps, "use_original_timestamps": False if self.model_name == "sonic" else True, } - if self._settings["language"]: - msg["language"] = self._settings["language"] + if is_given(self._settings.language) and self._settings.language: + msg["language"] = self._settings.language - if self._settings["speed"]: - msg["speed"] = self._settings["speed"] + if is_given(self._settings.speed) and self._settings.speed: + msg["speed"] = self._settings.speed - if self._settings["generation_config"]: - msg["generation_config"] = self._settings["generation_config"].model_dump( + if is_given(self._settings.generation_config) and self._settings.generation_config: + msg["generation_config"] = self._settings.generation_config.model_dump( exclude_none=True ) - if self._settings["pronunciation_dict_id"]: - msg["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"] + if is_given(self._settings.pronunciation_dict_id) and self._settings.pronunciation_dict_id: + msg["pronunciation_dict_id"] = self._settings.pronunciation_dict_id return json.dumps(msg) @@ -460,7 +478,7 @@ class CartesiaTTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -694,21 +712,21 @@ class CartesiaHttpTTSService(TTSService): self._api_key = api_key self._base_url = base_url self._cartesia_version = cartesia_version - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings: CartesiaTTSSettings = CartesiaTTSSettings( + model=model, + voice=voice_id, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - "speed": params.speed, - "emotion": params.emotion, - "generation_config": params.generation_config, - "pronunciation_dict_id": params.pronunciation_dict_id, - } - self.set_voice(voice_id) + speed=params.speed, + emotion=params.emotion, + generation_config=params.generation_config, + pronunciation_dict_id=params.pronunciation_dict_id, + ) + self._voice_id = voice_id self.set_model_name(model) self._client = AsyncCartesia( @@ -742,7 +760,7 @@ class CartesiaHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Cartesia HTTP TTS service. @@ -778,7 +796,7 @@ class CartesiaHttpTTSService(TTSService): try: voice_config = {"mode": "id", "id": self._voice_id} - if self._settings["emotion"]: + if is_given(self._settings.emotion) and self._settings.emotion: with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( @@ -786,30 +804,39 @@ class CartesiaHttpTTSService(TTSService): DeprecationWarning, stacklevel=2, ) - voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]} + voice_config["__experimental_controls"] = {"emotion": self._settings.emotion} await self.start_ttfb_metrics() + output_format = { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + } + payload = { "model_id": self._model_name, "transcript": text, "voice": voice_config, - "output_format": self._settings["output_format"], + "output_format": output_format, } - if self._settings["language"]: - payload["language"] = self._settings["language"] + if is_given(self._settings.language) and self._settings.language: + payload["language"] = self._settings.language - if self._settings["speed"]: - payload["speed"] = self._settings["speed"] + if is_given(self._settings.speed) and self._settings.speed: + payload["speed"] = self._settings.speed - if self._settings["generation_config"]: - payload["generation_config"] = self._settings["generation_config"].model_dump( + if is_given(self._settings.generation_config) and self._settings.generation_config: + payload["generation_config"] = self._settings.generation_config.model_dump( exclude_none=True ) - if self._settings["pronunciation_dict_id"]: - payload["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"] + if ( + is_given(self._settings.pronunciation_dict_id) + and self._settings.pronunciation_dict_id + ): + payload["pronunciation_dict_id"] = self._settings.pronunciation_dict_id yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/cerebras/llm.py b/src/pipecat/services/cerebras/llm.py index 54ea45ddb..01a8165f8 100644 --- a/src/pipecat/services/cerebras/llm.py +++ b/src/pipecat/services/cerebras/llm.py @@ -68,14 +68,14 @@ class CerebrasLLMService(OpenAILLMService): params = { "model": self.model_name, "stream": True, - "seed": self._settings["seed"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_completion_tokens": self._settings["max_completion_tokens"], + "seed": self._settings.seed, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_completion_tokens": self._settings.max_completion_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index 0f79499ba..91d4308cb 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -6,6 +6,7 @@ """Deepgram speech-to-text service implementation.""" +from dataclasses import dataclass, field from typing import AsyncGenerator, Dict, Optional from loguru import logger @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import DEEPGRAM_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -45,6 +47,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramSTTSettings(STTSettings): + """Typed settings for the Deepgram STT service. + + Parameters: + live_options: Deepgram ``LiveOptions`` for detailed configuration. + """ + + live_options: LiveOptions = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramSTTService(STTService): """Deepgram speech-to-text service. @@ -129,11 +142,17 @@ class DeepgramSTTService(STTService): merged_options["language"] = merged_options["language"].value self.set_model_name(merged_options["model"]) - self._settings = merged_options + merged_live_options = LiveOptions(**merged_options) + self._settings: DeepgramSTTSettings = DeepgramSTTSettings( + model=merged_options.get("model"), + language=merged_options.get("language"), + live_options=merged_live_options, + ) + self._addons = addons self._should_interrupt = should_interrupt - if merged_options.get("vad_events"): + if merged_live_options.vad_events: import warnings with warnings.catch_warnings(): @@ -164,7 +183,7 @@ class DeepgramSTTService(STTService): Returns: True if VAD events are enabled in the current settings. """ - return self._settings["vad_events"] + return self._settings.live_options.vad_events def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -174,28 +193,48 @@ class DeepgramSTTService(STTService): """ return True - async def set_model(self, model: str): - """Set the Deepgram model and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, keeping ``live_options`` in sync. - Args: - model: The Deepgram model name to use. + Top-level ``model`` and ``language`` are the source of truth. When + they are given in *update* their values are propagated into + ``live_options``. When only ``live_options`` is given, its ``model`` + and ``language`` are propagated *up* to the top-level fields. + + Any change triggers a WebSocket reconnect. """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._settings["model"] = model + # Determine which top-level fields are explicitly provided. + model_given = isinstance(update, DeepgramSTTSettings) and is_given( + getattr(update, "model", NOT_GIVEN) + ) + language_given = isinstance(update, DeepgramSTTSettings) and is_given( + getattr(update, "language", NOT_GIVEN) + ) + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # --- Sync model -------------------------------------------------- + if model_given: + # Top-level model wins → push into live_options. + self._settings.live_options.model = self._settings.model + elif "live_options" in changed and self._settings.live_options.model is not None: + # Only live_options was given → pull model up. + self._settings.model = self._settings.live_options.model + self.set_model_name(self._settings.model) + + # --- Sync language ----------------------------------------------- + if language_given: + self._settings.live_options.language = self._settings.language + elif "live_options" in changed and self._settings.live_options.language is not None: + self._settings.language = self._settings.live_options.language + await self._disconnect() await self._connect() - async def set_language(self, language: Language): - """Set the recognition language and reconnect. - - Args: - language: The language to use for speech recognition. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - await self._disconnect() - await self._connect() + return changed async def start(self, frame: StartFrame): """Start the Deepgram STT service. @@ -204,7 +243,7 @@ class DeepgramSTTService(STTService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.live_options.sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -257,7 +296,9 @@ class DeepgramSTTService(STTService): self._on_utterance_end, ) - if not await self._connection.start(options=self._settings, addons=self._addons): + if not await self._connection.start( + options=self._settings.live_options, addons=self._addons + ): await self.push_error(error_msg=f"Unable to connect to Deepgram") else: headers = { diff --git a/src/pipecat/services/deepgram/stt_sagemaker.py b/src/pipecat/services/deepgram/stt_sagemaker.py index 99f6cf487..95242ade6 100644 --- a/src/pipecat/services/deepgram/stt_sagemaker.py +++ b/src/pipecat/services/deepgram/stt_sagemaker.py @@ -14,6 +14,7 @@ languages, and various Deepgram features. import asyncio import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -31,6 +32,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import DEEPGRAM_SAGEMAKER_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -47,6 +49,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramSageMakerSTTSettings(STTSettings): + """Typed settings for the Deepgram SageMaker STT service. + + Parameters: + live_options: Deepgram ``LiveOptions`` for detailed configuration. + """ + + live_options: LiveOptions = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramSageMakerSTTService(STTService): """Deepgram speech-to-text service for AWS SageMaker. @@ -129,7 +142,12 @@ class DeepgramSageMakerSTTService(STTService): merged_options["language"] = merged_options["language"].value self.set_model_name(merged_options["model"]) - self._settings = merged_options + merged_live_options = LiveOptions(**merged_options) + self._settings: DeepgramSageMakerSTTSettings = DeepgramSageMakerSTTSettings( + model=merged_options.get("model"), + language=merged_options.get("language"), + live_options=merged_live_options, + ) self._client: Optional[SageMakerBidiClient] = None self._response_task: Optional[asyncio.Task] = None @@ -143,35 +161,40 @@ class DeepgramSageMakerSTTService(STTService): """ return True - async def set_model(self, model: str): - """Set the Deepgram model and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, keeping ``live_options`` in sync. - Disconnects from the current session, updates the model setting, and - establishes a new connection with the updated model. + Top-level ``model`` and ``language`` are the source of truth. When + they change their values are propagated into ``live_options``. - Args: - model: The Deepgram model name to use (e.g., "nova-3"). + Any change triggers a reconnect. """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._settings["model"] = model - await self._disconnect() - await self._connect() - - async def set_language(self, language: Language): - """Set the recognition language and reconnect. - - Disconnects from the current session, updates the language setting, and - establishes a new connection with the updated language. - - Args: - language: The language to use for speech recognition (e.g., Language.EN, - Language.ES). - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language + model_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given( + getattr(update, "model", NOT_GIVEN) + ) + language_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given( + getattr(update, "language", NOT_GIVEN) + ) + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # Sync model into live_options + if model_given and "model" in changed: + self._settings.live_options.model = self._settings.model + + # Sync language into live_options + if language_given and "language" in changed: + lang = self._settings.language + if isinstance(lang, Language): + lang = lang.value + self._settings.live_options.language = lang + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the Deepgram SageMaker STT service. @@ -180,7 +203,7 @@ class DeepgramSageMakerSTTService(STTService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.live_options.sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -226,12 +249,12 @@ class DeepgramSageMakerSTTService(STTService): """ logger.debug("Connecting to Deepgram on SageMaker...") - # Update sample rate in settings - self._settings["sample_rate"] = self.sample_rate + # Update sample rate in live_options + self._settings.live_options.sample_rate = self.sample_rate - # Build query string from settings, converting booleans to strings + # Build query string from live_options, converting booleans to strings query_params = {} - for key, value in self._settings.items(): + for key, value in self._settings.live_options.to_dict().items(): if value is not None: # Convert boolean values to lowercase strings for Deepgram API if isinstance(value, bool): diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index 12aba4905..4c698dcea 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -11,6 +11,7 @@ for generating speech from text using various voice models. """ import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -29,6 +30,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService, WebsocketTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -43,6 +45,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramTTSSettings(TTSSettings): + """Typed settings for Deepgram TTS service. + + Parameters: + encoding: Audio encoding format (linear16, mulaw, alaw). + """ + + encoding: str = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramTTSService(WebsocketTTSService): """Deepgram WebSocket-based text-to-speech service. @@ -91,10 +104,12 @@ class DeepgramTTSService(WebsocketTTSService): self._api_key = api_key self._base_url = base_url - self._settings = { - "encoding": encoding, - } - self.set_voice(voice) + self._settings: DeepgramTTSSettings = DeepgramTTSSettings( + model=voice, + voice=voice, + encoding=encoding, + ) + self._voice_id = voice self._receive_task = None self._context_id: Optional[str] = None @@ -177,7 +192,7 @@ class DeepgramTTSService(WebsocketTTSService): # Build WebSocket URL with query parameters params = [] params.append(f"model={self._voice_id}") - params.append(f"encoding={self._settings['encoding']}") + params.append(f"encoding={self._settings.encoding}") params.append(f"sample_rate={self.sample_rate}") url = f"{self._base_url}/v1/speak?{'&'.join(params)}" @@ -357,10 +372,12 @@ class DeepgramHttpTTSService(TTSService): self._api_key = api_key self._session = aiohttp_session self._base_url = base_url - self._settings = { - "encoding": encoding, - } - self.set_voice(voice) + self._settings: DeepgramTTSSettings = DeepgramTTSSettings( + model=voice, + voice=voice, + encoding=encoding, + ) + self._voice_id = voice def can_generate_metrics(self) -> bool: """Check if the service can generate metrics. @@ -390,7 +407,7 @@ class DeepgramHttpTTSService(TTSService): params = { "model": self._voice_id, - "encoding": self._settings["encoding"], + "encoding": self._settings.encoding, "sample_rate": self.sample_rate, "container": "none", } diff --git a/src/pipecat/services/deepseek/llm.py b/src/pipecat/services/deepseek/llm.py index 56f1ddd18..806dce13d 100644 --- a/src/pipecat/services/deepseek/llm.py +++ b/src/pipecat/services/deepseek/llm.py @@ -68,15 +68,15 @@ class DeepSeekLLMService(OpenAILLMService): "model": self.model_name, "stream": True, "stream_options": {"include_usage": True}, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 388f7146b..950dc5de9 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -14,6 +14,7 @@ transcription results directly. import base64 import io import json +from dataclasses import dataclass, field from enum import Enum from typing import AsyncGenerator, Optional @@ -33,6 +34,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import ELEVENLABS_REALTIME_TTFS_P99, ELEVENLABS_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -167,6 +169,44 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class ElevenLabsSTTSettings(STTSettings): + """Typed settings for the ElevenLabs file-based STT service. + + Parameters: + tag_audio_events: Whether to include audio event tags in transcription. + """ + + tag_audio_events: bool = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class ElevenLabsRealtimeSTTSettings(STTSettings): + """Typed settings for the ElevenLabs Realtime STT service. + + See ``ElevenLabsRealtimeSTTService.InputParams`` for detailed descriptions. + + Parameters: + commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD). + vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0). + vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive). + min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms). + min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms). + include_timestamps: Whether to include word-level timestamps in transcripts. + enable_logging: Whether to enable logging on ElevenLabs' side. + include_language_detection: Whether to include language detection in transcripts. + """ + + commit_strategy: CommitStrategy = field(default_factory=lambda: NOT_GIVEN) + vad_silence_threshold_secs: float = field(default_factory=lambda: NOT_GIVEN) + vad_threshold: float = field(default_factory=lambda: NOT_GIVEN) + min_speech_duration_ms: int = field(default_factory=lambda: NOT_GIVEN) + min_silence_duration_ms: int = field(default_factory=lambda: NOT_GIVEN) + include_timestamps: bool = field(default_factory=lambda: NOT_GIVEN) + enable_logging: bool = field(default_factory=lambda: NOT_GIVEN) + include_language_detection: bool = field(default_factory=lambda: NOT_GIVEN) + + class ElevenLabsSTTService(SegmentedSTTService): """Speech-to-text service using ElevenLabs' file-based API. @@ -223,13 +263,15 @@ class ElevenLabsSTTService(SegmentedSTTService): self._base_url = base_url self._session = aiohttp_session self._model_id = model - self._tag_audio_events = params.tag_audio_events - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: ElevenLabsSTTSettings = ElevenLabsSTTSettings( + model=model, + language=self.language_to_service_language(params.language) if params.language else "eng", - } + tag_audio_events=params.tag_audio_events, + ) + self.set_model_name(model) def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -250,27 +292,30 @@ class ElevenLabsSTTService(SegmentedSTTService): """ return language_to_elevenlabs_language(language) - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update. + + Converts language to ElevenLabs format before applying and keeps + ``_model_id`` in sync with the model setting. Args: - language: The language to use for speech-to-text transcription. + update: A :class:`STTSettings` (or ``ElevenLabsSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = self.language_to_service_language(language) + # Convert language to ElevenLabs format before applying + if is_given(update.language) and isinstance(update.language, Language): + converted = self.language_to_service_language(update.language) + if converted is not None: + update.language = converted - async def set_model(self, model: str): - """Set the STT model. + changed = await super()._update_settings_from_typed(update) - Args: - model: The model name to use for transcription. + if "model" in changed: + self._model_id = self._settings.model - Note: - ElevenLabs STT API does not currently support model selection. - This method is provided for interface compatibility. - """ - await super().set_model(model) - logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model") + return changed async def _transcribe_audio(self, audio_data: bytes) -> dict: """Upload audio data to ElevenLabs and get transcription result. @@ -298,8 +343,8 @@ class ElevenLabsSTTService(SegmentedSTTService): # Add required model_id, language_code, and tag_audio_events data.add_field("model_id", self._model_id) - data.add_field("language_code", self._settings["language"]) - data.add_field("tag_audio_events", str(self._tag_audio_events).lower()) + data.add_field("language_code", self._settings.language) + data.add_field("tag_audio_events", str(self._settings.tag_audio_events).lower()) async with self._session.post(url, data=data, headers=headers) as response: if response.status != 200: @@ -469,11 +514,22 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): self._api_key = api_key self._base_url = base_url self._model_id = model - self._params = params self._audio_format = "" # initialized in start() self._receive_task = None - self._settings = {"language": params.language_code} + self._settings: ElevenLabsRealtimeSTTSettings = ElevenLabsRealtimeSTTSettings( + model=model, + language=params.language_code, + commit_strategy=params.commit_strategy, + vad_silence_threshold_secs=params.vad_silence_threshold_secs, + vad_threshold=params.vad_threshold, + min_speech_duration_ms=params.min_speech_duration_ms, + min_silence_duration_ms=params.min_silence_duration_ms, + include_timestamps=params.include_timestamps, + enable_logging=params.enable_logging, + include_language_detection=params.include_language_detection, + ) + self.set_model_name(model) def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -483,42 +539,35 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and reconnect if anything changed. + + Converts language to ElevenLabs format before applying and keeps + ``_model_id`` in sync. Args: - language: The language to use for speech-to-text transcription. + update: A :class:`STTSettings` (or ``ElevenLabsRealtimeSTTSettings``) delta. - Note: - Changing language requires reconnecting to the WebSocket. + Returns: + Set of field names whose values actually changed. """ - logger.info(f"Switching STT language to: [{language}]") - new_language = ( - language_to_elevenlabs_language(language) - if isinstance(language, Language) - else language - ) - self._params.language_code = new_language - self._settings["language"] = new_language - # Reconnect with new settings - await self._disconnect() - await self._connect() - - async def set_model(self, model: str): - """Set the STT model. - - Args: - model: The model name to use for transcription. - - Note: - Changing model requires reconnecting to the WebSocket. - """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._model_id = model - # Reconnect with new settings + # Convert language to ElevenLabs format before applying + if is_given(update.language) and isinstance(update.language, Language): + converted = language_to_elevenlabs_language(update.language) + if converted is not None: + update.language = converted + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + if "model" in changed: + self._model_id = self._settings.model + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the STT service and establish WebSocket connection. @@ -566,7 +615,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): # Send commit when user stops speaking (manual commit mode) - if self._params.commit_strategy == CommitStrategy.MANUAL: + if self._settings.commit_strategy == CommitStrategy.MANUAL: if self._websocket and self._websocket.state is State.OPEN: try: commit_message = { @@ -656,36 +705,40 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): # Build query parameters params = [f"model_id={self._model_id}"] - if self._params.language_code: - params.append(f"language_code={self._params.language_code}") + if self._settings.language: + params.append(f"language_code={self._settings.language}") params.append(f"audio_format={self._audio_format}") - params.append(f"commit_strategy={self._params.commit_strategy.value}") + params.append(f"commit_strategy={self._settings.commit_strategy.value}") # Add optional parameters - if self._params.include_timestamps: - params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}") - - if self._params.enable_logging: - params.append(f"enable_logging={str(self._params.enable_logging).lower()}") - - if self._params.include_language_detection: + if self._settings.include_timestamps: params.append( - f"include_language_detection={str(self._params.include_language_detection).lower()}" + f"include_timestamps={str(self._settings.include_timestamps).lower()}" + ) + + if self._settings.enable_logging: + params.append(f"enable_logging={str(self._settings.enable_logging).lower()}") + + if self._settings.include_language_detection: + params.append( + f"include_language_detection={str(self._settings.include_language_detection).lower()}" ) # Add VAD parameters if using VAD commit strategy and values are specified - if self._params.commit_strategy == CommitStrategy.VAD: - if self._params.vad_silence_threshold_secs is not None: + if self._settings.commit_strategy == CommitStrategy.VAD: + if self._settings.vad_silence_threshold_secs is not None: params.append( - f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}" + f"vad_silence_threshold_secs={self._settings.vad_silence_threshold_secs}" + ) + if self._settings.vad_threshold is not None: + params.append(f"vad_threshold={self._settings.vad_threshold}") + if self._settings.min_speech_duration_ms is not None: + params.append(f"min_speech_duration_ms={self._settings.min_speech_duration_ms}") + if self._settings.min_silence_duration_ms is not None: + params.append( + f"min_silence_duration_ms={self._settings.min_silence_duration_ms}" ) - if self._params.vad_threshold is not None: - params.append(f"vad_threshold={self._params.vad_threshold}") - if self._params.min_speech_duration_ms is not None: - params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}") - if self._params.min_silence_duration_ms is not None: - params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}") ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}" @@ -817,7 +870,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): """ # If timestamps are enabled, skip this message and wait for the # committed_transcript_with_timestamps message which contains all the data - if self._params.include_timestamps: + if self._settings.include_timestamps: return text = data.get("text", "").strip() diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 4dab0c01a..b061383f3 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -13,7 +13,19 @@ with support for streaming audio, word timestamps, and voice customization. import asyncio import base64 import json -from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union +from dataclasses import dataclass, field +from typing import ( + Any, + AsyncGenerator, + ClassVar, + Dict, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) import aiohttp from loguru import logger @@ -32,6 +44,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import ( AudioContextWordTTSService, WordTTSService, @@ -136,12 +149,12 @@ def output_format_from_sample_rate(sample_rate: int) -> str: def build_elevenlabs_voice_settings( - settings: Dict[str, Any], + settings: Union[Dict[str, Any], "TTSSettings"], ) -> Optional[Dict[str, Union[float, bool]]]: """Build voice settings dictionary for ElevenLabs based on provided settings. Args: - settings: Dictionary containing voice settings parameters. + settings: Dictionary or typed settings containing voice settings parameters. Returns: Dictionary of voice settings or None if no valid settings are provided. @@ -150,8 +163,11 @@ def build_elevenlabs_voice_settings( voice_settings = {} for key in voice_setting_keys: - if key in settings and settings[key] is not None: - voice_settings[key] = settings[key] + val = ( + getattr(settings, key, None) if isinstance(settings, TTSSettings) else settings.get(key) + ) + if val is not None and is_given(val): + voice_settings[key] = val return voice_settings or None @@ -168,6 +184,75 @@ class PronunciationDictionaryLocator(BaseModel): version_id: str +@dataclass +class ElevenLabsTTSSettings(TTSSettings): + """Typed settings for the ElevenLabs WebSocket TTS service. + + Fields that appear in the WebSocket URL (``voice``, ``model``, + ``language``) require a full reconnect when changed. Fields that + affect the voice character (``stability``, ``similarity_boost``, + ``style``, ``use_speaker_boost``, ``speed``) can be applied by closing + the current audio context so a new one is opened with updated settings. + + Parameters: + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.7 to 1.2). + auto_mode: Whether to enable automatic mode optimization. + enable_ssml_parsing: Whether to parse SSML tags in text. + enable_logging: Whether to enable ElevenLabs logging. + apply_text_normalization: Text normalization mode ("auto", "on", "off"). + """ + + stability: float = field(default_factory=lambda: NOT_GIVEN) + similarity_boost: float = field(default_factory=lambda: NOT_GIVEN) + style: float = field(default_factory=lambda: NOT_GIVEN) + use_speaker_boost: bool = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + auto_mode: str = field(default_factory=lambda: NOT_GIVEN) + enable_ssml_parsing: bool = field(default_factory=lambda: NOT_GIVEN) + enable_logging: bool = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: str = field(default_factory=lambda: NOT_GIVEN) + + #: Fields in the WS URL — changing any of these requires a reconnect. + URL_FIELDS: ClassVar[frozenset[str]] = frozenset({"voice", "model", "language"}) + + #: Fields affecting voice character — changing these requires closing the + #: current audio context so the next one picks up new settings. + VOICE_SETTINGS_FIELDS: ClassVar[frozenset[str]] = frozenset( + {"stability", "similarity_boost", "style", "use_speaker_boost", "speed"} + ) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + +@dataclass +class ElevenLabsHttpTTSSettings(TTSSettings): + """Typed settings for the ElevenLabs HTTP TTS service. + + Parameters: + optimize_streaming_latency: Latency optimization level (0-4). + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.25 to 4.0). + apply_text_normalization: Text normalization mode ("auto", "on", "off"). + """ + + optimize_streaming_latency: int = field(default_factory=lambda: NOT_GIVEN) + stability: float = field(default_factory=lambda: NOT_GIVEN) + similarity_boost: float = field(default_factory=lambda: NOT_GIVEN) + style: float = field(default_factory=lambda: NOT_GIVEN) + use_speaker_boost: bool = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: str = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float, @@ -316,22 +401,25 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._api_key = api_key self._url = url - self._settings = { - "language": self.language_to_service_language(params.language) - if params.language - else None, - "stability": params.stability, - "similarity_boost": params.similarity_boost, - "style": params.style, - "use_speaker_boost": params.use_speaker_boost, - "speed": params.speed, - "auto_mode": str(params.auto_mode).lower(), - "enable_ssml_parsing": params.enable_ssml_parsing, - "enable_logging": params.enable_logging, - "apply_text_normalization": params.apply_text_normalization, - } + self._settings: ElevenLabsTTSSettings = ElevenLabsTTSSettings( + model=model, + voice=voice_id, + language=( + self.language_to_service_language(params.language) if params.language else None + ), + stability=params.stability, + similarity_boost=params.similarity_boost, + style=params.style, + use_speaker_boost=params.use_speaker_boost, + speed=params.speed, + auto_mode=str(params.auto_mode).lower(), + enable_ssml_parsing=params.enable_ssml_parsing, + enable_logging=params.enable_logging, + apply_text_normalization=params.apply_text_normalization, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id + self._output_format = "" # initialized in start() self._voice_settings = self._set_voice_settings() self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators @@ -366,54 +454,57 @@ class ElevenLabsTTSService(AudioContextWordTTSService): return language_to_elevenlabs_language(language) def _set_voice_settings(self): - return build_elevenlabs_voice_settings(self._settings) + ts = self._settings + voice_setting_keys = [ + "stability", + "similarity_boost", + "style", + "use_speaker_boost", + "speed", + ] + voice_settings = {} + for key in voice_setting_keys: + val = getattr(ts, key, None) + if val is not None and is_given(val): + voice_settings[key] = val + return voice_settings or None - async def set_model(self, model: str): - """Set the TTS model and reconnect. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update, reconnecting as needed. + + Uses the declarative ``URL_FIELDS`` and ``VOICE_SETTINGS_FIELDS`` + sets on :class:`ElevenLabsTTSSettings` to decide whether to + reconnect the WebSocket or close the current audio context. Args: - model: The model name to use for synthesis. + update: A :class:`TTSSettings` (or ``ElevenLabsTTSSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() + changed = await super()._update_settings_from_typed(update) - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice, model, or language changed.""" - # Track previous values for settings that require reconnection - prev_voice = self._voice_id - prev_model = self.model_name - prev_language = self._settings.get("language") - # Create snapshot of current voice settings to detect changes after update - prev_voice_settings = self._voice_settings.copy() if self._voice_settings else None + if not changed: + return changed - await super()._update_settings(settings) - - # Update voice settings for the next context creation + # Rebuild voice settings for next context self._voice_settings = self._set_voice_settings() - # Check if URL-level settings changed (these require reconnection) - url_changed = ( - prev_voice != self._voice_id - or prev_model != self.model_name - or prev_language != self._settings.get("language") - ) - - # Check if only voice settings changed (speed, stability, etc.) - voice_settings_changed = prev_voice_settings != self._voice_settings + url_changed = bool(changed & ElevenLabsTTSSettings.URL_FIELDS) + voice_settings_changed = bool(changed & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS) if url_changed: - # These settings are in the WebSocket URL, so we need to reconnect logger.debug( - f"URL-level setting changed (voice/model/language), reconnecting WebSocket" + f"URL-level setting changed ({changed & ElevenLabsTTSSettings.URL_FIELDS}), " + f"reconnecting WebSocket" ) await self._disconnect() await self._connect() elif voice_settings_changed and self._context_id: - # Voice settings can be updated by closing current context - # so new one gets created with updated voice settings - logger.debug(f"Voice settings changed, closing current context to apply changes") + logger.debug( + f"Voice settings changed ({changed & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS}), " + f"closing current context to apply changes" + ) try: if self._websocket: await self._websocket.send( @@ -423,6 +514,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._context_id = None + return changed + async def start(self, frame: StartFrame): """Start the ElevenLabs TTS service. @@ -505,19 +598,19 @@ class ElevenLabsTTSService(AudioContextWordTTSService): voice_id = self._voice_id model = self.model_name output_format = self._output_format - url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}" + url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings.auto_mode}" - if self._settings["enable_ssml_parsing"]: - url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}" + if self._settings.enable_ssml_parsing: + url += f"&enable_ssml_parsing={self._settings.enable_ssml_parsing}" - if self._settings["enable_logging"]: - url += f"&enable_logging={self._settings['enable_logging']}" + if self._settings.enable_logging: + url += f"&enable_logging={self._settings.enable_logging}" - if self._settings["apply_text_normalization"] is not None: - url += f"&apply_text_normalization={self._settings['apply_text_normalization']}" + if self._settings.apply_text_normalization is not None: + url += f"&apply_text_normalization={self._settings.apply_text_normalization}" # Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS - language = self._settings["language"] + language = self._settings.language if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None: url += f"&language_code={language}" logger.debug(f"Using language code: {language}") @@ -809,20 +902,22 @@ class ElevenLabsHttpTTSService(WordTTSService): self._params = params self._session = aiohttp_session - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: ElevenLabsHttpTTSSettings = ElevenLabsHttpTTSSettings( + model=model, + voice=voice_id, + language=self.language_to_service_language(params.language) if params.language else None, - "optimize_streaming_latency": params.optimize_streaming_latency, - "stability": params.stability, - "similarity_boost": params.similarity_boost, - "style": params.style, - "use_speaker_boost": params.use_speaker_boost, - "speed": params.speed, - "apply_text_normalization": params.apply_text_normalization, - } + optimize_streaming_latency=params.optimize_streaming_latency, + stability=params.stability, + similarity_boost=params.similarity_boost, + style=params.style, + use_speaker_boost=params.use_speaker_boost, + speed=params.speed, + apply_text_normalization=params.apply_text_normalization, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id self._output_format = "" # initialized in start() self._voice_settings = self._set_voice_settings() self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators @@ -859,10 +954,19 @@ class ElevenLabsHttpTTSService(WordTTSService): def _set_voice_settings(self): return build_elevenlabs_voice_settings(self._settings) - async def _update_settings(self, settings: Mapping[str, Any]): - await super()._update_settings(settings) - # Update voice settings for the next context creation - self._voice_settings = self._set_voice_settings() + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and rebuild voice settings. + + Args: + update: A :class:`TTSSettings` (or ``ElevenLabsHttpTTSSettings``) delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + if changed: + self._voice_settings = self._set_voice_settings() + return changed def _reset_state(self): """Reset internal state variables.""" @@ -999,10 +1103,13 @@ class ElevenLabsHttpTTSService(WordTTSService): locator.model_dump() for locator in self._pronunciation_dictionary_locators ] - if self._settings["apply_text_normalization"] is not None: - payload["apply_text_normalization"] = self._settings["apply_text_normalization"] + if ( + is_given(self._settings.apply_text_normalization) + and self._settings.apply_text_normalization is not None + ): + payload["apply_text_normalization"] = self._settings.apply_text_normalization - language = self._settings["language"] + language = self._settings.language if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language: payload["language_code"] = language logger.debug(f"Using language code: {language}") @@ -1020,8 +1127,11 @@ class ElevenLabsHttpTTSService(WordTTSService): params = { "output_format": self._output_format, } - if self._settings["optimize_streaming_latency"] is not None: - params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"] + if ( + is_given(self._settings.optimize_streaming_latency) + and self._settings.optimize_streaming_latency is not None + ): + params["optimize_streaming_latency"] = self._settings.optimize_streaming_latency try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index 4e8a655ec..eef0e0487 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -11,12 +11,14 @@ transcription using segmented audio processing. """ import os +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import FAL_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -146,6 +148,22 @@ def language_to_fal_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class FalSTTSettings(STTSettings): + """Typed settings for the Fal Wizper STT service. + + Parameters: + task: Task to perform ('transcribe' or 'translate'). Defaults to + 'transcribe'. + chunk_level: Level of chunking ('segment'). Defaults to 'segment'. + version: Version of Wizper model to use. Defaults to '3'. + """ + + task: str = field(default_factory=lambda: NOT_GIVEN) + chunk_level: str = field(default_factory=lambda: NOT_GIVEN) + version: str = field(default_factory=lambda: NOT_GIVEN) + + class FalSTTService(SegmentedSTTService): """Speech-to-text service using Fal's Wizper API. @@ -203,14 +221,14 @@ class FalSTTService(SegmentedSTTService): ) self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY")) - self._settings = { - "task": params.task, - "language": self.language_to_service_language(params.language) + self._settings: FalSTTSettings = FalSTTSettings( + language=self.language_to_service_language(params.language) if params.language else "en", - "chunk_level": params.chunk_level, - "version": params.version, - } + task=params.task, + chunk_level=params.chunk_level, + version=params.version, + ) def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -231,23 +249,17 @@ class FalSTTService(SegmentedSTTService): """ return language_to_fal_language(language) - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, converting language if changed.""" + changed = await super()._update_settings_from_typed(update) - Args: - language: The language to use for speech-to-text transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = self.language_to_service_language(language) + if "language" in changed: + # Convert the Language enum to a Fal language code. + lang = self._settings.language + if isinstance(lang, Language): + self._settings.language = self.language_to_service_language(lang) - async def set_model(self, model: str): - """Set the STT model. - - Args: - model: The model name to use for transcription. - """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") + return changed @traced_stt async def _handle_transcription( @@ -276,19 +288,19 @@ class FalSTTService(SegmentedSTTService): data_uri = fal_client.encode(audio, "audio/x-wav") response = await self._fal_client.run( "fal-ai/wizper", - arguments={"audio_url": data_uri, **self._settings}, + arguments={"audio_url": data_uri, **self._settings.given_fields()}, ) if response and "text" in response: text = response["text"].strip() if text: # Only yield non-empty text - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - Language(self._settings["language"]), + Language(self._settings.language), result=response, ) diff --git a/src/pipecat/services/fireworks/llm.py b/src/pipecat/services/fireworks/llm.py index d7bf57908..9338d8c5a 100644 --- a/src/pipecat/services/fireworks/llm.py +++ b/src/pipecat/services/fireworks/llm.py @@ -68,15 +68,15 @@ class FireworksLLMService(OpenAILLMService): params = { "model": self.model_name, "stream": True, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 93a718429..5517758ad 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -11,6 +11,7 @@ for streaming text-to-speech synthesis with customizable voice parameters. """ import uuid +from dataclasses import dataclass, field from typing import AsyncGenerator, Literal, Optional from loguru import logger @@ -28,6 +29,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleTTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -45,6 +47,29 @@ except ModuleNotFoundError as e: FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] +@dataclass +class FishAudioTTSSettings(TTSSettings): + """Typed settings for Fish Audio TTS service. + + Parameters: + fish_sample_rate: Audio sample rate sent to the API. + latency: Latency mode ("normal" or "balanced"). Defaults to "normal". + format: Audio output format. + normalize: Whether to normalize audio output. Defaults to True. + prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0. + prosody_volume: Volume adjustment in dB. Defaults to 0. + reference_id: Reference ID of the voice model. + """ + + fish_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + latency: str = field(default_factory=lambda: NOT_GIVEN) + format: str = field(default_factory=lambda: NOT_GIVEN) + normalize: bool = field(default_factory=lambda: NOT_GIVEN) + prosody_speed: float = field(default_factory=lambda: NOT_GIVEN) + prosody_volume: int = field(default_factory=lambda: NOT_GIVEN) + reference_id: str = field(default_factory=lambda: NOT_GIVEN) + + class FishAudioTTSService(InterruptibleTTSService): """Fish Audio text-to-speech service with WebSocket streaming. @@ -136,17 +161,16 @@ class FishAudioTTSService(InterruptibleTTSService): self._receive_task = None self._request_id = None - self._settings = { - "sample_rate": 0, - "latency": params.latency, - "format": output_format, - "normalize": params.normalize, - "prosody": { - "speed": params.prosody_speed, - "volume": params.prosody_volume, - }, - "reference_id": reference_id, - } + self._settings: FishAudioTTSSettings = FishAudioTTSSettings( + voice=reference_id, + fish_sample_rate=0, + latency=params.latency, + format=output_format, + normalize=params.normalize, + prosody_speed=params.prosody_speed, + prosody_volume=params.prosody_volume, + reference_id=reference_id, + ) self.set_model_name(model_id) @@ -158,16 +182,22 @@ class FishAudioTTSService(InterruptibleTTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model and reconnect. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if needed. + + Any change to voice or model triggers a WebSocket reconnect. Args: - model: The model name to use for synthesis. + update: A :class:`TTSSettings` (or ``FishAudioTTSSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() + changed = await super()._update_settings_from_typed(update) + if changed: + await self._disconnect() + await self._connect() + return changed async def start(self, frame: StartFrame): """Start the Fish Audio TTS service. @@ -176,7 +206,7 @@ class FishAudioTTSService(InterruptibleTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.fish_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -225,7 +255,18 @@ class FishAudioTTSService(InterruptibleTTSService): self._websocket = await websocket_connect(self._base_url, additional_headers=headers) # Send initial start message with ormsgpack - start_message = {"event": "start", "request": {"text": "", **self._settings}} + request_settings = { + "sample_rate": self._settings.fish_sample_rate, + "latency": self._settings.latency, + "format": self._settings.format, + "normalize": self._settings.normalize, + "prosody": { + "speed": self._settings.prosody_speed, + "volume": self._settings.prosody_volume, + }, + "reference_id": self._settings.reference_id, + } + start_message = {"event": "start", "request": {"text": "", **request_settings}} await self._websocket.send(ormsgpack.packb(start_message)) logger.debug("Sent start message to Fish Audio") diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index 475a7213e..76a1620e1 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -14,6 +14,7 @@ import asyncio import base64 import json import warnings +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, Literal, Optional import aiohttp @@ -32,6 +33,7 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.services.gladia.config import GladiaInputParams +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import GLADIA_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -178,6 +180,17 @@ class _InputParamsDescriptor: return GladiaInputParams +@dataclass +class GladiaSTTSettings(STTSettings): + """Typed settings for Gladia STT service. + + Parameters: + input_params: Gladia ``GladiaInputParams`` for detailed configuration. + """ + + input_params: GladiaInputParams = field(default_factory=lambda: NOT_GIVEN) + + class GladiaSTTService(WebsocketSTTService): """Speech-to-Text service using Gladia's API. @@ -265,9 +278,8 @@ class GladiaSTTService(WebsocketSTTService): self._region = region self._url = url self.set_model_name(model) - self._params = params self._receive_task = None - self._settings = {} + self._settings = GladiaSTTSettings(model=model, input_params=params) # Session management self._session_url = None @@ -307,31 +319,33 @@ class GladiaSTTService(WebsocketSTTService): return language_to_gladia_language(language) def _prepare_settings(self) -> Dict[str, Any]: + params = self._settings.input_params + settings = { - "encoding": self._params.encoding or "wav/pcm", - "bit_depth": self._params.bit_depth or 16, + "encoding": params.encoding or "wav/pcm", + "bit_depth": params.bit_depth or 16, "sample_rate": self.sample_rate, - "channels": self._params.channels or 1, + "channels": params.channels or 1, "model": self._model_name, } # Add custom_metadata if provided - settings["custom_metadata"] = dict(self._params.custom_metadata or {}) + settings["custom_metadata"] = dict(params.custom_metadata or {}) settings["custom_metadata"]["pipecat"] = pipecat_version() # Add endpointing parameters if provided - if self._params.endpointing is not None: - settings["endpointing"] = self._params.endpointing - if self._params.maximum_duration_without_endpointing is not None: + if params.endpointing is not None: + settings["endpointing"] = params.endpointing + if params.maximum_duration_without_endpointing is not None: settings["maximum_duration_without_endpointing"] = ( - self._params.maximum_duration_without_endpointing + params.maximum_duration_without_endpointing ) # Add language configuration (prioritize language_config over deprecated language) - if self._params.language_config: - settings["language_config"] = self._params.language_config.model_dump(exclude_none=True) - elif self._params.language: # Backward compatibility for deprecated parameter - language_code = self.language_to_service_language(self._params.language) + if params.language_config: + settings["language_config"] = params.language_config.model_dump(exclude_none=True) + elif params.language: # Backward compatibility for deprecated parameter + language_code = self.language_to_service_language(params.language) if language_code: settings["language_config"] = { "languages": [language_code], @@ -339,21 +353,18 @@ class GladiaSTTService(WebsocketSTTService): } # Add pre_processing configuration if provided - if self._params.pre_processing: - settings["pre_processing"] = self._params.pre_processing.model_dump(exclude_none=True) + if params.pre_processing: + settings["pre_processing"] = params.pre_processing.model_dump(exclude_none=True) # Add realtime_processing configuration if provided - if self._params.realtime_processing: - settings["realtime_processing"] = self._params.realtime_processing.model_dump( + if params.realtime_processing: + settings["realtime_processing"] = params.realtime_processing.model_dump( exclude_none=True ) # Add messages_config if provided - if self._params.messages_config: - settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True) - - # Store settings for tracing - self._settings = settings + if params.messages_config: + settings["messages_config"] = params.messages_config.model_dump(exclude_none=True) return settings @@ -366,6 +377,31 @@ class GladiaSTTService(WebsocketSTTService): await super().start(frame) await self._connect() + async def _update_settings_from_typed(self, update: GladiaSTTSettings) -> set[str]: + """Apply typed settings update. + + Gladia sessions are fixed at creation time, so any change requires + a full session teardown and reconnect. + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # Gladia sessions are fixed — need to tear down and recreate + self._session_url = None + self._session_id = None + await self._disconnect() + await self._connect() + + return changed + async def stop(self, frame: EndFrame): """Stop the Gladia STT websocket connection. @@ -522,7 +558,7 @@ class GladiaSTTService(WebsocketSTTService): Broadcasts UserStartedSpeakingFrame and optionally triggers interruption when VAD is enabled. """ - if not self._params.enable_vad or self._is_speaking: + if not self._settings.input_params.enable_vad or self._is_speaking: return logger.debug(f"{self} User started speaking") @@ -537,7 +573,7 @@ class GladiaSTTService(WebsocketSTTService): Broadcasts UserStoppedSpeakingFrame when VAD is enabled. """ - if not self._params.enable_vad or not self._is_speaking: + if not self._settings.input_params.enable_vad or not self._is_speaking: return self._is_speaking = False await self.broadcast_frame(UserStoppedSpeakingFrame) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index e209f3d0a..1edab5783 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -17,9 +17,9 @@ import io import time import uuid import warnings -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, ClassVar, Dict, List, Optional, Union from loguru import logger from PIL import Image @@ -47,7 +47,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, @@ -77,6 +76,7 @@ from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.string import match_endofsentence from pipecat.utils.time import time_now_iso8601 @@ -602,6 +602,31 @@ class InputParams(BaseModel): extra: Optional[Dict[str, Any]] = Field(default_factory=dict) +@dataclass +class GeminiLiveLLMSettings(LLMSettings): + """Typed settings for Gemini Live LLM services. + + Parameters: + modalities: Response modalities. + language: Language for generation. + media_resolution: Media resolution setting. + vad: Voice activity detection parameters. + context_window_compression: Context window compression configuration. + thinking: Thinking configuration. + enable_affective_dialog: Whether to enable affective dialog. + proactivity: Proactivity configuration. + """ + + modalities: Any = field(default_factory=lambda: NOT_GIVEN) + language: Any = field(default_factory=lambda: NOT_GIVEN) + media_resolution: Any = field(default_factory=lambda: NOT_GIVEN) + vad: Any = field(default_factory=lambda: NOT_GIVEN) + context_window_compression: Any = field(default_factory=lambda: NOT_GIVEN) + thinking: Any = field(default_factory=lambda: NOT_GIVEN) + enable_affective_dialog: Any = field(default_factory=lambda: NOT_GIVEN) + proactivity: Any = field(default_factory=lambda: NOT_GIVEN) + + class GeminiLiveLLMService(LLMService): """Provides access to Google's Gemini Live API. @@ -714,25 +739,26 @@ class GeminiLiveLLMService(LLMService): self._consecutive_failures = 0 self._connection_start_time = None - self._settings = { - "frequency_penalty": params.frequency_penalty, - "max_tokens": params.max_tokens, - "presence_penalty": params.presence_penalty, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "modalities": params.modalities, - "language": self._language_code, - "media_resolution": params.media_resolution, - "vad": params.vad, - "context_window_compression": params.context_window_compression.model_dump() + self._settings = GeminiLiveLLMSettings( + model=model, + frequency_penalty=params.frequency_penalty, + max_tokens=params.max_tokens, + presence_penalty=params.presence_penalty, + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + modalities=params.modalities, + language=self._language_code, + media_resolution=params.media_resolution, + vad=params.vad, + context_window_compression=params.context_window_compression.model_dump() if params.context_window_compression else {}, - "thinking": params.thinking or {}, - "enable_affective_dialog": params.enable_affective_dialog or False, - "proactivity": params.proactivity or {}, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + thinking=params.thinking or {}, + enable_affective_dialog=params.enable_affective_dialog or False, + proactivity=params.proactivity or {}, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) self._file_api_base_url = file_api_base_url self._file_api: Optional[GeminiFileAPI] = None @@ -798,7 +824,7 @@ class GeminiLiveLLMService(LLMService): Args: modalities: The modalities to use for responses. """ - self._settings["modalities"] = modalities + self._settings.modalities = modalities def set_language(self, language: Language): """Set the language for generation. @@ -808,7 +834,7 @@ class GeminiLiveLLMService(LLMService): """ self._language = language self._language_code = language_to_gemini_language(language) or "en-US" - self._settings["language"] = self._language_code + self._settings.language = self._language_code logger.info(f"Set Gemini language to: {self._language_code}") async def set_context(self, context: OpenAILLMContext): @@ -866,7 +892,7 @@ class GeminiLiveLLMService(LLMService): async def _handle_interruption(self): if self._bot_is_responding: await self._set_bot_is_responding(False) - if self._settings.get("modalities") == GeminiModalities.AUDIO: + if self._settings.modalities == GeminiModalities.AUDIO: await self.push_frame(TTSStoppedFrame()) # Do not send LLMFullResponseEndFrame here - an interruption # already tells the assistant context aggregator that the response @@ -947,8 +973,6 @@ class GeminiLiveLLMService(LLMService): # uses this frame *without* a user context aggregator still works # (we have an example that does just that, actually). await self._create_single_response(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() else: @@ -1074,20 +1098,20 @@ class GeminiLiveLLMService(LLMService): # Assemble basic configuration config = LiveConnectConfig( generation_config=GenerationConfig( - frequency_penalty=self._settings["frequency_penalty"], - max_output_tokens=self._settings["max_tokens"], - presence_penalty=self._settings["presence_penalty"], - temperature=self._settings["temperature"], - top_k=self._settings["top_k"], - top_p=self._settings["top_p"], - response_modalities=[Modality(self._settings["modalities"].value)], + frequency_penalty=self._settings.frequency_penalty, + max_output_tokens=self._settings.max_tokens, + presence_penalty=self._settings.presence_penalty, + temperature=self._settings.temperature, + top_k=self._settings.top_k, + top_p=self._settings.top_p, + response_modalities=[Modality(self._settings.modalities.value)], speech_config=SpeechConfig( voice_config=VoiceConfig( prebuilt_voice_config={"voice_name": self._voice_id} ), - language_code=self._settings["language"], + language_code=self._settings.language, ), - media_resolution=MediaResolution(self._settings["media_resolution"].value), + media_resolution=MediaResolution(self._settings.media_resolution.value), ), input_audio_transcription=AudioTranscriptionConfig(), output_audio_transcription=AudioTranscriptionConfig(), @@ -1095,37 +1119,36 @@ class GeminiLiveLLMService(LLMService): ) # Add context window compression to configuration, if enabled - if self._settings.get("context_window_compression", {}).get("enabled", False): + cwc = self._settings.context_window_compression or {} + if cwc.get("enabled", False): compression_config = ContextWindowCompressionConfig() # Add sliding window (always true if compression is enabled) compression_config.sliding_window = SlidingWindow() # Add trigger_tokens if specified - trigger_tokens = self._settings.get("context_window_compression", {}).get( - "trigger_tokens" - ) + trigger_tokens = cwc.get("trigger_tokens") if trigger_tokens is not None: compression_config.trigger_tokens = trigger_tokens config.context_window_compression = compression_config # Add thinking configuration to configuration, if provided - if self._settings.get("thinking"): - config.thinking_config = self._settings["thinking"] + if self._settings.thinking: + config.thinking_config = self._settings.thinking # Add affective dialog setting, if provided - if self._settings.get("enable_affective_dialog", False): - config.enable_affective_dialog = self._settings["enable_affective_dialog"] + if self._settings.enable_affective_dialog: + config.enable_affective_dialog = self._settings.enable_affective_dialog # Add proactivity configuration to configuration, if provided - if self._settings.get("proactivity"): - config.proactivity = self._settings["proactivity"] + if self._settings.proactivity: + config.proactivity = self._settings.proactivity # Add VAD configuration to configuration, if provided - if self._settings.get("vad"): + if self._settings.vad: vad_config = AutomaticActivityDetection() - vad_params = self._settings["vad"] + vad_params = self._settings.vad has_vad_settings = False # Only add parameters that are explicitly set @@ -1604,7 +1627,7 @@ class GeminiLiveLLMService(LLMService): text: The transcription text to push result: Optional LiveServerMessage that triggered this transcription """ - await self._handle_user_transcription(text, True, self._settings["language"]) + await self._handle_user_transcription(text, True, self._settings.language) await self.push_frame( TranscriptionFrame( text=text, diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 563acadb3..692106241 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -15,8 +15,8 @@ import io import json import os import uuid -from dataclasses import dataclass -from typing import Any, AsyncIterator, Dict, List, Literal, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncIterator, ClassVar, Dict, List, Literal, Optional from loguru import logger from PIL import Image @@ -39,7 +39,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, ) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext @@ -59,6 +58,7 @@ from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm # Suppress gRPC fork warnings @@ -673,6 +673,17 @@ class GoogleLLMContext(OpenAILLMContext): self._messages = [m for m in self._messages if m.parts] +@dataclass +class GoogleLLMSettings(LLMSettings): + """Typed settings for Google LLM services. + + Parameters: + thinking: Thinking configuration. + """ + + thinking: Any = field(default_factory=lambda: NOT_GIVEN) + + class GoogleLLMService(LLMService): """Google AI (Gemini) LLM service implementation. @@ -773,14 +784,15 @@ class GoogleLLMService(LLMService): self._system_instruction = system_instruction self._http_options = update_google_client_http_options(http_options) - self._settings = { - "max_tokens": params.max_tokens, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "thinking": params.thinking, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + self._settings = GoogleLLMSettings( + model=model, + max_tokens=params.max_tokens, + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + thinking=params.thinking, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) self._tools = tools self._tool_config = tool_config @@ -874,10 +886,10 @@ class GoogleLLMService(LLMService): k: v for k, v in { "system_instruction": system_instruction, - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "top_k": self._settings["top_k"], - "max_output_tokens": self._settings["max_tokens"], + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "top_k": self._settings.top_k, + "max_output_tokens": self._settings.max_tokens, "tools": tools, "tool_config": tool_config, }.items() @@ -885,13 +897,13 @@ class GoogleLLMService(LLMService): } # Add thinking parameters if configured - if self._settings["thinking"]: - generation_params["thinking_config"] = self._settings["thinking"].model_dump( + if self._settings.thinking: + generation_params["thinking_config"] = self._settings.thinking.model_dump( exclude_unset=True ) - if self._settings["extra"]: - generation_params.update(self._settings["extra"]) + if self._settings.extra: + generation_params.update(self._settings.extra) return generation_params @@ -1190,8 +1202,6 @@ class GoogleLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = GoogleLLMContext(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 23396b0b8..8f762da9d 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -15,13 +15,15 @@ import asyncio import json import os import time +import warnings +from dataclasses import dataclass, field from pipecat.utils.tracing.service_decorators import traced_stt # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" -from typing import AsyncGenerator, List, Optional, Union +from typing import Any, AsyncGenerator, List, Optional, Union from loguru import logger from pydantic import BaseModel, Field, field_validator @@ -34,6 +36,7 @@ from pipecat.frames.frames import ( StartFrame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import GOOGLE_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -355,6 +358,44 @@ def language_to_google_stt_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class GoogleSTTSettings(STTSettings): + """Typed settings for Google Cloud Speech-to-Text V2. + + Parameters: + languages: List of ``Language`` enums for recognition + (e.g. ``[Language.EN_US]``). Preferred over ``language_codes``. + language_codes: List of Google STT language code strings + (e.g. ``["en-US"]``). + + .. deprecated:: 0.0.103 + Use ``languages`` instead. If both are provided, ``languages`` + takes precedence. This field is here just for backward + compatibility with dict-based settings updates. + use_separate_recognition_per_channel: Process each audio channel separately. + enable_automatic_punctuation: Add punctuation to transcripts. + enable_spoken_punctuation: Include spoken punctuation in transcript. + enable_spoken_emojis: Include spoken emojis in transcript. + profanity_filter: Filter profanity from transcript. + enable_word_time_offsets: Include timing information for each word. + enable_word_confidence: Include confidence scores for each word. + enable_interim_results: Stream partial recognition results. + enable_voice_activity_events: Detect voice activity in audio. + """ + + languages: Any = field(default_factory=lambda: NOT_GIVEN) + language_codes: Any = field(default_factory=lambda: NOT_GIVEN) + use_separate_recognition_per_channel: Any = field(default_factory=lambda: NOT_GIVEN) + enable_automatic_punctuation: Any = field(default_factory=lambda: NOT_GIVEN) + enable_spoken_punctuation: Any = field(default_factory=lambda: NOT_GIVEN) + enable_spoken_emojis: Any = field(default_factory=lambda: NOT_GIVEN) + profanity_filter: Any = field(default_factory=lambda: NOT_GIVEN) + enable_word_time_offsets: Any = field(default_factory=lambda: NOT_GIVEN) + enable_word_confidence: Any = field(default_factory=lambda: NOT_GIVEN) + enable_interim_results: Any = field(default_factory=lambda: NOT_GIVEN) + enable_voice_activity_events: Any = field(default_factory=lambda: NOT_GIVEN) + + class GoogleSTTService(STTService): """Google Cloud Speech-to-Text V2 service implementation. @@ -508,21 +549,19 @@ class GoogleSTTService(STTService): self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options) - self._settings = { - "language_codes": [ - self.language_to_service_language(lang) for lang in params.language_list - ], - "model": params.model, - "use_separate_recognition_per_channel": params.use_separate_recognition_per_channel, - "enable_automatic_punctuation": params.enable_automatic_punctuation, - "enable_spoken_punctuation": params.enable_spoken_punctuation, - "enable_spoken_emojis": params.enable_spoken_emojis, - "profanity_filter": params.profanity_filter, - "enable_word_time_offsets": params.enable_word_time_offsets, - "enable_word_confidence": params.enable_word_confidence, - "enable_interim_results": params.enable_interim_results, - "enable_voice_activity_events": params.enable_voice_activity_events, - } + self._settings = GoogleSTTSettings( + languages=list(params.language_list), + model=params.model, + use_separate_recognition_per_channel=params.use_separate_recognition_per_channel, + enable_automatic_punctuation=params.enable_automatic_punctuation, + enable_spoken_punctuation=params.enable_spoken_punctuation, + enable_spoken_emojis=params.enable_spoken_emojis, + profanity_filter=params.profanity_filter, + enable_word_time_offsets=params.enable_word_time_offsets, + enable_word_confidence=params.enable_word_confidence, + enable_interim_results=params.enable_interim_results, + enable_voice_activity_events=params.enable_voice_activity_events, + ) def can_generate_metrics(self) -> bool: """Check if the service can generate metrics. @@ -545,6 +584,23 @@ class GoogleSTTService(STTService): return [language_to_google_stt_language(lang) or "en-US" for lang in language] return language_to_google_stt_language(language) or "en-US" + def _get_language_codes(self) -> List[str]: + """Resolve the current language settings to Google STT language code strings. + + Prefers ``languages`` (``Language`` enums) over the deprecated + ``language_codes`` (raw strings). Falls back to ``["en-US"]``. + + Returns: + List[str]: Google STT language code strings. + """ + from pipecat.services.settings import is_given + + if is_given(self._settings.languages): + return [self.language_to_service_language(lang) for lang in self._settings.languages] + if is_given(self._settings.language_codes): + return list(self._settings.language_codes) + return ["en-US"] + async def _reconnect_if_needed(self): """Reconnect the stream if it's currently active.""" if self._streaming_task: @@ -552,41 +608,65 @@ class GoogleSTTService(STTService): await self._disconnect() await self._connect() - async def set_language(self, language: Language): - """Update the service's recognition language. - - A convenience method for setting a single language. - - Args: - language: New language for recognition. - """ - logger.debug(f"Switching STT language to: {language}") - await self.set_languages([language]) - async def set_languages(self, languages: List[Language]): """Update the service's recognition languages. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(languages=...)`` + instead. + Args: languages: List of languages for recognition. First language is primary. """ + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "set_languages() is deprecated. Use STTUpdateSettingsFrame with " + "GoogleSTTSettings(languages=...) instead.", + DeprecationWarning, + ) logger.debug(f"Switching STT languages to: {languages}") - self._settings["language_codes"] = [ - self.language_to_service_language(lang) for lang in languages - ] - # Recreate stream with new languages - await self._reconnect_if_needed() + await self._update_settings_from_typed(GoogleSTTSettings(languages=list(languages))) - async def set_model(self, model: str): - """Update the service's recognition model. + async def _update_settings_from_typed(self, update: GoogleSTTSettings) -> set[str]: + """Apply typed settings update and reconnect if anything changed. + + Handles ``language`` from base ``set_language`` by converting it to + ``languages``. Emits a deprecation warning if ``language_codes`` is + used. All other fields (model, boolean flags) are applied directly. + Reconnects the stream on any change. Args: - model: The new recognition model to use. + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. """ - logger.debug(f"Switching STT model to: {model}") - await super().set_model(model) - self._settings["model"] = model - # Recreate stream with new model - await self._reconnect_if_needed() + from pipecat.services.settings import is_given + + # If base set_language sent a Language value, convert to languages list + if is_given(update.language): + update.languages = [update.language] + # Clear language so the base class doesn't try to store it + update.language = NOT_GIVEN + + # Warn on deprecated language_codes usage + if is_given(update.language_codes): + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "GoogleSTTSettings.language_codes is deprecated. " + "Use GoogleSTTSettings.languages (List[Language]) instead.", + DeprecationWarning, + stacklevel=2, + ) + + changed = await super()._update_settings_from_typed(update) + + if changed: + await self._reconnect_if_needed() + + return changed async def start(self, frame: StartFrame): """Start the STT service and establish connection. @@ -632,6 +712,10 @@ class GoogleSTTService(STTService): ) -> None: """Update service options dynamically. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(...)`` + instead. + Args: languages: New list of recognition languages. model: New recognition model. @@ -649,55 +733,42 @@ class GoogleSTTService(STTService): Changes that affect the streaming configuration will cause the stream to be reconnected. """ - # Update settings with new values + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "update_options() is deprecated. Use STTUpdateSettingsFrame with " + "GoogleSTTSettings(...) instead.", + DeprecationWarning, + ) + # Build a typed settings delta from the provided options + update = GoogleSTTSettings() + if languages is not None: - logger.debug(f"Updating language to: {languages}") - self._settings["language_codes"] = [ - self.language_to_service_language(lang) for lang in languages - ] - + update.languages = list(languages) if model is not None: - logger.debug(f"Updating model to: {model}") - self._settings["model"] = model - + update.model = model if enable_automatic_punctuation is not None: - logger.debug(f"Updating automatic punctuation to: {enable_automatic_punctuation}") - self._settings["enable_automatic_punctuation"] = enable_automatic_punctuation - + update.enable_automatic_punctuation = enable_automatic_punctuation if enable_spoken_punctuation is not None: - logger.debug(f"Updating spoken punctuation to: {enable_spoken_punctuation}") - self._settings["enable_spoken_punctuation"] = enable_spoken_punctuation - + update.enable_spoken_punctuation = enable_spoken_punctuation if enable_spoken_emojis is not None: - logger.debug(f"Updating spoken emojis to: {enable_spoken_emojis}") - self._settings["enable_spoken_emojis"] = enable_spoken_emojis - + update.enable_spoken_emojis = enable_spoken_emojis if profanity_filter is not None: - logger.debug(f"Updating profanity filter to: {profanity_filter}") - self._settings["profanity_filter"] = profanity_filter - + update.profanity_filter = profanity_filter if enable_word_time_offsets is not None: - logger.debug(f"Updating word time offsets to: {enable_word_time_offsets}") - self._settings["enable_word_time_offsets"] = enable_word_time_offsets - + update.enable_word_time_offsets = enable_word_time_offsets if enable_word_confidence is not None: - logger.debug(f"Updating word confidence to: {enable_word_confidence}") - self._settings["enable_word_confidence"] = enable_word_confidence - + update.enable_word_confidence = enable_word_confidence if enable_interim_results is not None: - logger.debug(f"Updating interim results to: {enable_interim_results}") - self._settings["enable_interim_results"] = enable_interim_results - + update.enable_interim_results = enable_interim_results if enable_voice_activity_events is not None: - logger.debug(f"Updating voice activity events to: {enable_voice_activity_events}") - self._settings["enable_voice_activity_events"] = enable_voice_activity_events + update.enable_voice_activity_events = enable_voice_activity_events if location is not None: logger.debug(f"Updating location to: {location}") self._location = location - # Reconnect the stream for updates - await self._reconnect_if_needed() + await self._update_settings_from_typed(update) async def _connect(self): """Initialize streaming recognition config and stream.""" @@ -714,20 +785,20 @@ class GoogleSTTService(STTService): sample_rate_hertz=self.sample_rate, audio_channel_count=1, ), - language_codes=self._settings["language_codes"], - model=self._settings["model"], + language_codes=self._get_language_codes(), + model=self._settings.model, features=cloud_speech.RecognitionFeatures( - enable_automatic_punctuation=self._settings["enable_automatic_punctuation"], - enable_spoken_punctuation=self._settings["enable_spoken_punctuation"], - enable_spoken_emojis=self._settings["enable_spoken_emojis"], - profanity_filter=self._settings["profanity_filter"], - enable_word_time_offsets=self._settings["enable_word_time_offsets"], - enable_word_confidence=self._settings["enable_word_confidence"], + enable_automatic_punctuation=self._settings.enable_automatic_punctuation, + enable_spoken_punctuation=self._settings.enable_spoken_punctuation, + enable_spoken_emojis=self._settings.enable_spoken_emojis, + profanity_filter=self._settings.profanity_filter, + enable_word_time_offsets=self._settings.enable_word_time_offsets, + enable_word_confidence=self._settings.enable_word_confidence, ), ), streaming_features=cloud_speech.StreamingRecognitionFeatures( - enable_voice_activity_events=self._settings["enable_voice_activity_events"], - interim_results=self._settings["enable_interim_results"], + enable_voice_activity_events=self._settings.enable_voice_activity_events, + interim_results=self._settings.enable_interim_results, ), ) @@ -857,7 +928,7 @@ class GoogleSTTService(STTService): if not transcript: continue - primary_language = self._settings["language_codes"][0] + primary_language = self._get_language_codes()[0] if result.is_final: self._last_transcript_was_final = True diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 4016286df..d015571d0 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -23,7 +23,8 @@ from pipecat.utils.tracing.service_decorators import traced_tts # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" -from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, List, Literal, Optional from loguru import logger from pydantic import BaseModel @@ -36,6 +37,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language @@ -474,6 +476,63 @@ def language_to_gemini_tts_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class GoogleHttpTTSSettings(TTSSettings): + """Typed settings for Google HTTP TTS service. + + Parameters: + pitch: Voice pitch adjustment (e.g., "+2st", "-50%"). + rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). Used for + SSML prosody tags (non-Chirp voices). + speaking_rate: Speaking rate for AudioConfig (Chirp/Journey voices). + Range [0.25, 2.0]. + volume: Volume adjustment (e.g., "loud", "soft", "+6dB"). + emphasis: Emphasis level for the text. + language: Language for synthesis. Defaults to English. + gender: Voice gender preference. + google_style: Google-specific voice style. + """ + + pitch: str = field(default_factory=lambda: NOT_GIVEN) + rate: str = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float = field(default_factory=lambda: NOT_GIVEN) + volume: str = field(default_factory=lambda: NOT_GIVEN) + emphasis: str = field(default_factory=lambda: NOT_GIVEN) + language: str = field(default_factory=lambda: NOT_GIVEN) + gender: str = field(default_factory=lambda: NOT_GIVEN) + google_style: str = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class GoogleStreamTTSSettings(TTSSettings): + """Typed settings for Google streaming TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + speaking_rate: The speaking rate, in the range [0.25, 2.0]. + """ + + language: str = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class GeminiTTSSettings(TTSSettings): + """Typed settings for Gemini TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + prompt: Optional style instructions for how to synthesize the content. + multi_speaker: Whether to enable multi-speaker support. + speaker_configs: List of speaker configurations for multi-speaker mode. + """ + + language: str = field(default_factory=lambda: NOT_GIVEN) + prompt: str = field(default_factory=lambda: NOT_GIVEN) + multi_speaker: bool = field(default_factory=lambda: NOT_GIVEN) + speaker_configs: List[dict] = field(default_factory=lambda: NOT_GIVEN) + + class GoogleHttpTTSService(TTSService): """Google Cloud Text-to-Speech HTTP service with SSML support. @@ -538,19 +597,19 @@ class GoogleHttpTTSService(TTSService): params = params or GoogleHttpTTSService.InputParams() self._location = location - self._settings = { - "pitch": params.pitch, - "rate": params.rate, - "speaking_rate": params.speaking_rate, - "volume": params.volume, - "emphasis": params.emphasis, - "language": self.language_to_service_language(params.language) + self._settings: GoogleHttpTTSSettings = GoogleHttpTTSSettings( + pitch=params.pitch, + rate=params.rate, + speaking_rate=params.speaking_rate, + volume=params.volume, + emphasis=params.emphasis, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "gender": params.gender, - "google_style": params.google_style, - } - self.set_voice(voice_id) + gender=params.gender, + google_style=params.google_style, + ) + self._voice_id = voice_id self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path ) @@ -619,21 +678,20 @@ class GoogleHttpTTSService(TTSService): """ return language_to_google_tts_language(language) - async def _update_settings(self, settings: Mapping[str, Any]): - """Override to handle speaking_rate updates for Chirp/Journey voices. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Override to handle speaking_rate validation. Args: - settings: Dictionary of settings to update. Can include 'speaking_rate' (float) + update: Typed settings delta. Can include 'speaking_rate' (float). """ - if "speaking_rate" in settings: - rate_value = float(settings["speaking_rate"]) - if 0.25 <= rate_value <= 2.0: - self._settings["speaking_rate"] = rate_value - else: + if isinstance(update, GoogleHttpTTSSettings) and is_given(update.speaking_rate): + rate_value = float(update.speaking_rate) + if not (0.25 <= rate_value <= 2.0): logger.warning( f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0" ) - await super()._update_settings(settings) + update.speaking_rate = NOT_GIVEN + return await super()._update_settings_from_typed(update) def _construct_ssml(self, text: str) -> str: ssml = "" @@ -641,39 +699,39 @@ class GoogleHttpTTSService(TTSService): # Voice tag voice_attrs = [f"name='{self._voice_id}'"] - language = self._settings["language"] + language = self._settings.language voice_attrs.append(f"language='{language}'") - if self._settings["gender"]: - voice_attrs.append(f"gender='{self._settings['gender']}'") + if self._settings.gender: + voice_attrs.append(f"gender='{self._settings.gender}'") ssml += f"" # Prosody tag prosody_attrs = [] - if self._settings["pitch"]: - prosody_attrs.append(f"pitch='{self._settings['pitch']}'") - if self._settings["rate"]: - prosody_attrs.append(f"rate='{self._settings['rate']}'") - if self._settings["volume"]: - prosody_attrs.append(f"volume='{self._settings['volume']}'") + if self._settings.pitch: + prosody_attrs.append(f"pitch='{self._settings.pitch}'") + if self._settings.rate: + prosody_attrs.append(f"rate='{self._settings.rate}'") + if self._settings.volume: + prosody_attrs.append(f"volume='{self._settings.volume}'") if prosody_attrs: ssml += f"" # Emphasis tag - if self._settings["emphasis"]: - ssml += f"" + if self._settings.emphasis: + ssml += f"" # Google style tag - if self._settings["google_style"]: - ssml += f"" + if self._settings.google_style: + ssml += f"" ssml += text # Close tags - if self._settings["google_style"]: + if self._settings.google_style: ssml += "" - if self._settings["emphasis"]: + if self._settings.emphasis: ssml += "" if prosody_attrs: ssml += "" @@ -710,7 +768,7 @@ class GoogleHttpTTSService(TTSService): synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], name=self._voice_id + language_code=self._settings.language, name=self._voice_id ) # Build audio config with conditional speaking_rate audio_config_params = { @@ -719,8 +777,8 @@ class GoogleHttpTTSService(TTSService): } # For Chirp and Journey voices, include speaking_rate in AudioConfig - if (is_chirp_voice or is_journey_voice) and self._settings["speaking_rate"] is not None: - audio_config_params["speaking_rate"] = self._settings["speaking_rate"] + if (is_chirp_voice or is_journey_voice) and self._settings.speaking_rate is not None: + audio_config_params["speaking_rate"] = self._settings.speaking_rate audio_config = texttospeech_v1.AudioConfig(**audio_config_params) @@ -950,33 +1008,32 @@ class GoogleTTSService(GoogleBaseTTSService): params = params or GoogleTTSService.InputParams() self._location = location - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: GoogleStreamTTSSettings = GoogleStreamTTSSettings( + language=self.language_to_service_language(params.language) if params.language else "en-US", - "speaking_rate": params.speaking_rate, - } - self.set_voice(voice_id) + speaking_rate=params.speaking_rate, + ) + self._voice_id = voice_id self._voice_cloning_key = voice_cloning_key self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path ) - async def _update_settings(self, settings: Mapping[str, Any]): - """Override to handle speaking_rate updates for streaming API. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Override to handle speaking_rate validation. Args: - settings: Dictionary of settings to update. Can include 'speaking_rate' (float) + update: Typed settings delta. Can include 'speaking_rate' (float). """ - if "speaking_rate" in settings: - rate_value = float(settings["speaking_rate"]) - if 0.25 <= rate_value <= 2.0: - self._settings["speaking_rate"] = rate_value - else: + if isinstance(update, GoogleStreamTTSSettings) and is_given(update.speaking_rate): + rate_value = float(update.speaking_rate) + if not (0.25 <= rate_value <= 2.0): logger.warning( f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0" ) - await super()._update_settings(settings) + update.speaking_rate = NOT_GIVEN + return await super()._update_settings_from_typed(update) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -1000,11 +1057,11 @@ class GoogleTTSService(GoogleBaseTTSService): voice_cloning_key=self._voice_cloning_key ) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], voice_clone=voice_clone_params + language_code=self._settings.language, voice_clone=voice_clone_params ) else: voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], name=self._voice_id + language_code=self._settings.language, name=self._voice_id ) # Create streaming config @@ -1013,7 +1070,7 @@ class GoogleTTSService(GoogleBaseTTSService): streaming_audio_config=texttospeech_v1.StreamingAudioConfig( audio_encoding=texttospeech_v1.AudioEncoding.PCM, sample_rate_hertz=self.sample_rate, - speaking_rate=self._settings["speaking_rate"], + speaking_rate=self._settings.speaking_rate, ), ) @@ -1159,14 +1216,14 @@ class GeminiTTSService(GoogleBaseTTSService): self._location = location self._model = model self._voice_id = voice_id - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: GeminiTTSSettings = GeminiTTSSettings( + language=self.language_to_service_language(params.language) if params.language else "en-US", - "prompt": params.prompt, - "multi_speaker": params.multi_speaker, - "speaker_configs": params.speaker_configs, - } + prompt=params.prompt, + multi_speaker=params.multi_speaker, + speaker_configs=params.speaker_configs, + ) self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path @@ -1183,7 +1240,7 @@ class GeminiTTSService(GoogleBaseTTSService): """ return language_to_gemini_tts_language(language) - def set_voice(self, voice_id: str): + async def set_voice(self, voice_id: str): """Set the voice for TTS generation. Args: @@ -1206,15 +1263,13 @@ class GeminiTTSService(GoogleBaseTTSService): f"Current rate of {self.sample_rate}Hz may cause issues." ) - async def _update_settings(self, settings: Mapping[str, Any]): + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: """Override to handle prompt updates. Args: - settings: Dictionary of settings to update. Can include 'prompt' (str) + update: Typed settings delta. Can include 'prompt' (str). """ - if "prompt" in settings: - self._settings["prompt"] = settings["prompt"] - await super()._update_settings(settings) + return await super()._update_settings_from_typed(update) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -1234,10 +1289,10 @@ class GeminiTTSService(GoogleBaseTTSService): await self.start_ttfb_metrics() # Build voice selection params - if self._settings["multi_speaker"] and self._settings["speaker_configs"]: + if self._settings.multi_speaker and self._settings.speaker_configs: # Multi-speaker mode speaker_voice_configs = [] - for speaker_config in self._settings["speaker_configs"]: + for speaker_config in self._settings.speaker_configs: speaker_voice_configs.append( texttospeech_v1.MultispeakerPrebuiltVoice( speaker_alias=speaker_config["speaker_alias"], @@ -1250,14 +1305,14 @@ class GeminiTTSService(GoogleBaseTTSService): ) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], + language_code=self._settings.language, model_name=self._model, multi_speaker_voice_config=multi_speaker_voice_config, ) else: # Single speaker mode voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], + language_code=self._settings.language, name=self._voice_id, model_name=self._model, ) @@ -1273,7 +1328,7 @@ class GeminiTTSService(GoogleBaseTTSService): # Use base class streaming logic with prompt support async for frame in self._stream_tts( - streaming_config, text, context_id, self._settings["prompt"] + streaming_config, text, context_id, self._settings.prompt ): yield frame diff --git a/src/pipecat/services/gradium/stt.py b/src/pipecat/services/gradium/stt.py index 7433c2549..2bad8cf30 100644 --- a/src/pipecat/services/gradium/stt.py +++ b/src/pipecat/services/gradium/stt.py @@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription. import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import GRADIUM_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -64,6 +66,18 @@ def language_to_gradium_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class GradiumSTTSettings(STTSettings): + """Typed settings for the Gradium STT service. + + Parameters: + delay_in_frames: Delay in audio frames (80ms each) before text is + generated. Higher delays allow more context but increase latency. + """ + + delay_in_frames: int = field(default_factory=lambda: NOT_GIVEN) + + class GradiumSTTService(WebsocketSTTService): """Gradium real-time speech-to-text service. @@ -127,9 +141,15 @@ class GradiumSTTService(WebsocketSTTService): self._api_key = api_key self._api_endpoint_base_url = api_endpoint_base_url self._websocket = None - self._params = params or GradiumSTTService.InputParams() self._json_config = json_config + params = params or GradiumSTTService.InputParams() + + self._settings: GradiumSTTSettings = GradiumSTTSettings( + language=params.language, + delay_in_frames=params.delay_in_frames if params.delay_in_frames else NOT_GIVEN, + ) + self._receive_task = None self._audio_buffer = bytearray() @@ -149,16 +169,22 @@ class GradiumSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the recognition language and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, sync params, and reconnect. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``GradiumSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - logger.info(f"Switching STT language to: [{language}]") - self._params.language = language + changed = await super()._update_settings_from_typed(update) + if not changed: + return changed + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the speech-to-text service. @@ -298,12 +324,12 @@ class GradiumSTTService(WebsocketSTTService): json_config = {} if self._json_config: json_config = json.loads(self._json_config) - if self._params.language: - gradium_language = language_to_gradium_language(self._params.language) + if is_given(self._settings.language) and self._settings.language: + gradium_language = language_to_gradium_language(self._settings.language) if gradium_language: json_config["language"] = gradium_language - if self._params.delay_in_frames: - json_config["delay_in_frames"] = self._params.delay_in_frames + if is_given(self._settings.delay_in_frames) and self._settings.delay_in_frames: + json_config["delay_in_frames"] = self._settings.delay_in_frames if json_config: setup_msg["json_config"] = json_config await self._websocket.send(json.dumps(setup_msg)) diff --git a/src/pipecat/services/gradium/tts.py b/src/pipecat/services/gradium/tts.py index 0e9865cf0..e129fba68 100644 --- a/src/pipecat/services/gradium/tts.py +++ b/src/pipecat/services/gradium/tts.py @@ -6,7 +6,8 @@ import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleWordTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -37,6 +39,17 @@ except ModuleNotFoundError as e: SAMPLE_RATE = 48000 +@dataclass +class GradiumTTSSettings(TTSSettings): + """Typed settings for the Gradium TTS service. + + Parameters: + output_format: Audio output format. + """ + + output_format: str = field(default_factory=lambda: NOT_GIVEN) + + class GradiumTTSService(InterruptibleWordTTSService): """Text-to-Speech service using Gradium's websocket API.""" @@ -86,12 +99,11 @@ class GradiumTTSService(InterruptibleWordTTSService): self._url = url self._voice_id = voice_id self._json_config = json_config - self._model = model - self._settings = { - "voice_id": voice_id, - "model_name": model, - "output_format": "pcm", - } + self._settings: GradiumTTSSettings = GradiumTTSSettings( + model=model, + voice=voice_id, + output_format="pcm", + ) # State tracking self._receive_task = None @@ -105,24 +117,21 @@ class GradiumTTSService(InterruptibleWordTTSService): """ return True - async def set_model(self, model: str): - """Update the TTS model. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if voice changed. Args: - model: The model name to use for synthesis. - """ - self._model = model - await super().set_model(model) + update: A :class:`TTSSettings` (or ``GradiumTTSSettings``) delta. - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" + Returns: + Set of field names whose values actually changed. + """ prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - self._settings["voice_id"] = self._voice_id - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + changed = await super()._update_settings_from_typed(update) + if self._voice_id != prev_voice: await self._disconnect() await self._connect() + return changed def _build_msg(self, text: str = "") -> dict: """Build JSON message for Gradium API.""" diff --git a/src/pipecat/services/grok/realtime/llm.py b/src/pipecat/services/grok/realtime/llm.py index e1355ce31..7cb619a7d 100644 --- a/src/pipecat/services/grok/realtime/llm.py +++ b/src/pipecat/services/grok/realtime/llm.py @@ -13,8 +13,8 @@ https://docs.x.ai/docs/guides/voice/agent import base64 import json import time -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger @@ -56,6 +56,7 @@ from pipecat.processors.aggregators.llm_response_universal import ( from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.time import time_now_iso8601 from . import events @@ -85,6 +86,17 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class GrokRealtimeLLMSettings(LLMSettings): + """Typed settings for Grok Realtime LLM services. + + Parameters: + session_properties: Grok Realtime session configuration. + """ + + session_properties: Any = field(default_factory=lambda: NOT_GIVEN) + + class GrokRealtimeLLMService(LLMService): """Grok Realtime Voice Agent LLM service providing real-time audio and text communication. @@ -134,9 +146,8 @@ class GrokRealtimeLLMService(LLMService): self.api_key = api_key self.base_url = base_url - # Initialize session_properties - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = GrokRealtimeLLMSettings( + session_properties=session_properties or events.SessionProperties(), ) self._audio_input_paused = start_audio_paused @@ -186,13 +197,13 @@ class GrokRealtimeLLMService(LLMService): Configured sample rate or None if not manually configured. For PCMU/PCMA formats, returns 8000 Hz (G.711 standard). """ - if not self._session_properties.audio: + if not self._settings.session_properties.audio: return None audio_config = ( - self._session_properties.audio.input + self._settings.session_properties.audio.input if direction == "input" - else self._session_properties.audio.output + else self._settings.session_properties.audio.output ) if audio_config and audio_config.format: @@ -222,8 +233,8 @@ class GrokRealtimeLLMService(LLMService): def _is_turn_detection_enabled(self) -> bool: """Check if server-side VAD is enabled.""" - if self._session_properties.turn_detection: - return self._session_properties.turn_detection.type == "server_vad" + if self._settings.session_properties.turn_detection: + return self._settings.session_properties.turn_detection.type == "server_vad" return False async def _handle_interruption(self): @@ -290,18 +301,18 @@ class GrokRealtimeLLMService(LLMService): await super().start(frame) # Ensure audio configuration exists with both input and output - if not self._session_properties.audio: - self._session_properties.audio = events.AudioConfiguration() + if not self._settings.session_properties.audio: + self._settings.session_properties.audio = events.AudioConfiguration() # Fill in missing input configuration - if not self._session_properties.audio.input: - self._session_properties.audio.input = events.AudioInput( + if not self._settings.session_properties.audio.input: + self._settings.session_properties.audio.input = events.AudioInput( format=events.PCMAudioFormat(rate=frame.audio_in_sample_rate) ) # Fill in missing output configuration - if not self._session_properties.audio.output: - self._session_properties.audio.output = events.AudioOutput( + if not self._settings.session_properties.audio.output: + self._settings.session_properties.audio.output = events.AudioOutput( format=events.PCMAudioFormat(rate=frame.audio_out_sample_rate) ) @@ -336,6 +347,16 @@ class GrokRealtimeLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Legacy dict path: frame.settings contains SessionProperties fields, + # not our Settings fields, so we construct SessionProperties directly. + # The new typed path (frame.update) falls through to super, which calls + # _update_settings_from_typed → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._update_settings() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -355,9 +376,6 @@ class GrokRealtimeLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() @@ -436,9 +454,16 @@ class GrokRealtimeLLMService(LLMService): return await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) + async def _update_settings_from_typed(self, update): + """Apply a typed settings update, sending a session update if needed.""" + changed = await super()._update_settings_from_typed(update) + if "session_properties" in changed: + await self._update_settings() + return changed + async def _update_settings(self): """Update session settings on the server.""" - settings = self._session_properties + settings = self._settings.session_properties adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter() if self._context: diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 331af8eb7..678a2426d 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -8,6 +8,7 @@ import io import wave +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -20,6 +21,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -32,6 +34,21 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class GroqTTSSettings(TTSSettings): + """Typed settings for the Groq TTS service. + + Parameters: + output_format: Audio output format. + speed: Speech speed multiplier. Defaults to 1.0. + groq_sample_rate: Audio sample rate. + """ + + output_format: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + groq_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class GroqTTSService(TTSService): """Groq text-to-speech service implementation. @@ -92,14 +109,14 @@ class GroqTTSService(TTSService): self._voice_id = voice_id self._params = params - self._settings = { - "model": model_name, - "voice_id": voice_id, - "output_format": output_format, - "language": str(params.language) if params.language else "en", - "speed": params.speed, - "sample_rate": sample_rate, - } + self._settings: GroqTTSSettings = GroqTTSSettings( + model=model_name, + voice=voice_id, + language=str(params.language) if params.language else "en", + output_format=output_format, + speed=params.speed, + groq_sample_rate=sample_rate, + ) self._client = AsyncGroq(api_key=self._api_key) diff --git a/src/pipecat/services/hathora/stt.py b/src/pipecat/services/hathora/stt.py index defdc355d..b0e3beead 100644 --- a/src/pipecat/services/hathora/stt.py +++ b/src/pipecat/services/hathora/stt.py @@ -8,6 +8,7 @@ import base64 import os +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -18,6 +19,7 @@ from pipecat.frames.frames import ( Frame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import HATHORA_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language @@ -27,6 +29,19 @@ from pipecat.utils.tracing.service_decorators import traced_stt from .utils import ConfigOption +@dataclass +class HathoraSTTSettings(STTSettings): + """Typed settings for the Hathora STT service. + + Parameters: + config: Some models support additional config, refer to + `docs `_ for each model to see + what is supported. + """ + + config: Optional[list] = field(default_factory=lambda: NOT_GIVEN) + + class HathoraSTTService(SegmentedSTTService): """This service supports several different speech-to-text models hosted by Hathora. @@ -83,10 +98,11 @@ class HathoraSTTService(SegmentedSTTService): params = params or HathoraSTTService.InputParams() - self._settings = { - "language": params.language, - "config": params.config, - } + self._settings: HathoraSTTSettings = HathoraSTTSettings( + model=model, + language=params.language, + config=params.config, + ) self.set_model_name(model) @@ -123,12 +139,11 @@ class HathoraSTTService(SegmentedSTTService): "model": self._model, } - if self._settings["language"] is not None: - payload["language"] = self._settings["language"] - if self._settings["config"] is not None: + if self._settings.language is not None: + payload["language"] = self._settings.language + if self._settings.config is not None: payload["model_config"] = [ - {"name": option.name, "value": option.value} - for option in self._settings["config"] + {"name": option.name, "value": option.value} for option in self._settings.config ] base64_audio = base64.b64encode(audio).decode("utf-8") @@ -147,7 +162,7 @@ class HathoraSTTService(SegmentedSTTService): if text: # Only yield non-empty text # Hathora's API currently doesn't return language info # so we default to the requested language or "en" - response_language = self._settings["language"] or "en" + response_language = self._settings.language or "en" await self._handle_transcription(text, True, response_language) yield TranscriptionFrame( text, diff --git a/src/pipecat/services/hathora/tts.py b/src/pipecat/services/hathora/tts.py index 80cbd4fe8..b821b1e05 100644 --- a/src/pipecat/services/hathora/tts.py +++ b/src/pipecat/services/hathora/tts.py @@ -9,6 +9,7 @@ import io import os import wave +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional, Tuple import aiohttp @@ -21,6 +22,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -45,6 +47,21 @@ def _decode_audio_payload( return audio_bytes, fallback_sample_rate, fallback_channels +@dataclass +class HathoraTTSSettings(TTSSettings): + """Typed settings for Hathora TTS service. + + Parameters: + speed: Speech speed multiplier (if supported by model). + config: Some models support additional config, refer to + [docs](https://models.hathora.dev) for each model to see + what is supported. + """ + + speed: float = field(default_factory=lambda: NOT_GIVEN) + config: list = field(default_factory=lambda: NOT_GIVEN) + + class HathoraTTSService(TTSService): """This service supports several different text-to-speech models hosted by Hathora. @@ -98,13 +115,15 @@ class HathoraTTSService(TTSService): params = params or HathoraTTSService.InputParams() - self._settings = { - "speed": params.speed, - "config": params.config, - } + self._settings: HathoraTTSSettings = HathoraTTSSettings( + model=model, + voice=voice_id, + speed=params.speed, + config=params.config, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -135,12 +154,11 @@ class HathoraTTSService(TTSService): if self._voice_id is not None: payload["voice"] = self._voice_id - if self._settings["speed"] is not None: - payload["speed"] = self._settings["speed"] - if self._settings["config"] is not None: + if self._settings.speed is not None: + payload["speed"] = self._settings.speed + if self._settings.config is not None: payload["model_config"] = [ - {"name": option.name, "value": option.value} - for option in self._settings["config"] + {"name": option.name, "value": option.value} for option in self._settings.config ] yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 2d98e1f8c..3b45cc249 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -117,7 +117,7 @@ class HumeTTSService(WordTTSService): self._params = params or HumeTTSService.InputParams() # Store voice in the base class (mirrors other services) - self.set_voice(voice_id) + self._voice_id = voice_id self._audio_bytes = b"" @@ -196,7 +196,7 @@ class HumeTTSService(WordTTSService): key_l = (key or "").lower() if key_l == "voice_id": - self.set_voice(str(value)) + await self.set_voice(str(value)) logger.debug(f"HumeTTSService voice_id set to: {self.voice}") elif key_l == "description": self._params.description = None if value is None else str(value) diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index 2ea94399b..68c140187 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -16,6 +16,7 @@ Inworld’s text-to-speech (TTS) models offer ultra-realistic, context-aware spe import asyncio import base64 import json +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple import aiohttp @@ -23,6 +24,8 @@ import websockets from loguru import logger from pydantic import BaseModel +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given + try: from websockets.asyncio.client import connect as websocket_connect from websockets.protocol import State @@ -47,6 +50,31 @@ from pipecat.services.tts_service import AudioContextWordTTSService, WordTTSServ from pipecat.utils.tracing.service_decorators import traced_tts +@dataclass +class InworldTTSSettings(TTSSettings): + """Typed settings for Inworld TTS services. + + Parameters: + audio_encoding: Audio encoding format (e.g. LINEAR16). + audio_sample_rate: Audio sample rate in Hz. + speaking_rate: Speaking rate for speech synthesis. + temperature: Temperature for speech synthesis. + auto_mode: Whether to use auto mode. Recommended when texts are sent + in full sentences/phrases. When enabled, the server controls + flushing of buffered text to achieve minimal latency while + maintaining high quality audio output. If None (default), + automatically set based on aggregate_sentences. + apply_text_normalization: Whether to apply text normalization. + """ + + audio_encoding: str = field(default_factory=lambda: NOT_GIVEN) + audio_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + auto_mode: bool = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: str = field(default_factory=lambda: NOT_GIVEN) + + class InworldHttpTTSService(WordTTSService): """Inworld AI HTTP-based TTS service. @@ -110,23 +138,21 @@ class InworldHttpTTSService(WordTTSService): else: self._base_url = "https://api.inworld.ai/tts/v1/voice" - self._settings = { - "voiceId": voice_id, - "modelId": model, - "audioConfig": { - "audioEncoding": encoding, - "sampleRateHertz": 0, - }, - } + self._settings: InworldTTSSettings = InworldTTSSettings( + model=model, + voice=voice_id, + audio_encoding=encoding, + audio_sample_rate=0, + ) if params.temperature is not None: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature if params.speaking_rate is not None: - self._settings["audioConfig"]["speakingRate"] = params.speaking_rate + self._settings.speaking_rate = params.speaking_rate self._cumulative_time = 0.0 - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) def can_generate_metrics(self) -> bool: @@ -144,7 +170,7 @@ class InworldHttpTTSService(WordTTSService): frame: The start frame. """ await super().start(frame) - self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Inworld TTS service. @@ -223,15 +249,22 @@ class InworldHttpTTSService(WordTTSService): """ logger.debug(f"{self}: Generating TTS [{text}] (streaming={self._streaming})") + audio_config = { + "audioEncoding": self._settings.audio_encoding, + "sampleRateHertz": self._settings.audio_sample_rate, + } + if is_given(self._settings.speaking_rate): + audio_config["speakingRate"] = self._settings.speaking_rate + payload = { "text": text, - "voiceId": self._settings["voiceId"], - "modelId": self._settings["modelId"], - "audioConfig": self._settings["audioConfig"], + "voiceId": self._settings.voice, + "modelId": self._settings.model, + "audioConfig": audio_config, } - if "temperature" in self._settings: - payload["temperature"] = self._settings["temperature"] + if is_given(self._settings.temperature): + payload["temperature"] = self._settings.temperature # Use WORD timestamps for simplicity and correct spacing/capitalization payload["timestampType"] = self._timestamp_type @@ -470,27 +503,25 @@ class InworldTTSService(AudioContextWordTTSService): self._api_key = api_key self._url = url - self._settings: Dict[str, Any] = { - "voiceId": voice_id, - "modelId": model, - "audioConfig": { - "audioEncoding": encoding, - "sampleRateHertz": 0, - }, - } + self._settings: InworldTTSSettings = InworldTTSSettings( + model=model, + voice=voice_id, + audio_encoding=encoding, + audio_sample_rate=0, + ) self._timestamp_type = "WORD" if params.temperature is not None: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature if params.speaking_rate is not None: - self._settings["audioConfig"]["speakingRate"] = params.speaking_rate + self._settings.speaking_rate = params.speaking_rate if params.apply_text_normalization is not None: - self._settings["applyTextNormalization"] = params.apply_text_normalization + self._settings.apply_text_normalization = params.apply_text_normalization if params.auto_mode is not None: - self._settings["autoMode"] = params.auto_mode + self._settings.auto_mode = params.auto_mode else: - self._settings["autoMode"] = aggregate_sentences + self._settings.auto_mode = aggregate_sentences self._buffer_settings = { "maxBufferDelayMs": params.max_buffer_delay_ms, @@ -509,7 +540,7 @@ class InworldTTSService(AudioContextWordTTSService): # Track the end time of the last word in the current generation self._generation_end_time = 0.0 - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) def can_generate_metrics(self) -> bool: @@ -527,7 +558,7 @@ class InworldTTSService(AudioContextWordTTSService): frame: The start frame. """ await super().start(frame) - self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -859,18 +890,25 @@ class InworldTTSService(AudioContextWordTTSService): Args: context_id: The context ID. """ + audio_config = { + "audioEncoding": self._settings.audio_encoding, + "sampleRateHertz": self._settings.audio_sample_rate, + } + if is_given(self._settings.speaking_rate): + audio_config["speakingRate"] = self._settings.speaking_rate + create_config: Dict[str, Any] = { - "voiceId": self._settings["voiceId"], - "modelId": self._settings["modelId"], - "audioConfig": self._settings["audioConfig"], + "voiceId": self._settings.voice, + "modelId": self._settings.model, + "audioConfig": audio_config, } - if "temperature" in self._settings: - create_config["temperature"] = self._settings["temperature"] - if "applyTextNormalization" in self._settings: - create_config["applyTextNormalization"] = self._settings["applyTextNormalization"] - if "autoMode" in self._settings: - create_config["autoMode"] = self._settings["autoMode"] + if is_given(self._settings.temperature): + create_config["temperature"] = self._settings.temperature + if is_given(self._settings.apply_text_normalization): + create_config["applyTextNormalization"] = self._settings.apply_text_normalization + if is_given(self._settings.auto_mode): + create_config["autoMode"] = self._settings.auto_mode # Set buffer settings for timely audio generation. # Use provided values or defaults that work well for streaming LLM output. diff --git a/src/pipecat/services/kokoro/tts.py b/src/pipecat/services/kokoro/tts.py index 49ede2409..242446de9 100644 --- a/src/pipecat/services/kokoro/tts.py +++ b/src/pipecat/services/kokoro/tts.py @@ -7,6 +7,7 @@ """Kokoro TTS service implementation using kokoro-onnx.""" import os +from dataclasses import dataclass, field from pathlib import Path from typing import AsyncGenerator, Optional @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -87,6 +89,17 @@ def language_to_kokoro_language(language: Language) -> str: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class KokoroTTSSettings(TTSSettings): + """Typed settings for the Kokoro TTS service. + + Parameters: + lang_code: Kokoro language code for synthesis. + """ + + lang_code: str = field(default_factory=lambda: NOT_GIVEN) + + class KokoroTTSService(TTSService): """Kokoro TTS service implementation. @@ -129,6 +142,12 @@ class KokoroTTSService(TTSService): self._voice_id = voice_id self._lang_code = language_to_kokoro_language(params.language) + self._settings: KokoroTTSSettings = KokoroTTSSettings( + voice=voice_id, + language=language_to_kokoro_language(params.language), + lang_code=language_to_kokoro_language(params.language), + ) + model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx" voices = Path(voices_path) if voices_path else KOKORO_CACHE_DIR / "voices-v1.0.bin" diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index af7e691b0..77af50f15 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -44,6 +44,7 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, + LLMUpdateSettingsFrame, StartFrame, UserImageRequestFrame, ) @@ -58,6 +59,7 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import ServiceSettings from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin from pipecat.utils.context.llm_context_summarization import ( LLMContextSummarizationUtil, @@ -351,6 +353,17 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): await self._handle_interruptions(frame) elif isinstance(frame, LLMConfigureOutputFrame): self._skip_tts = frame.skip_tts + elif isinstance(frame, LLMUpdateSettingsFrame): + # New path: typed settings update object. + if frame.update is not None: + await self._update_settings_from_typed(frame.update) + # Legacy path: plain dict, but service uses typed settings — convert. + elif isinstance(self._settings, ServiceSettings): + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings_from_typed(update) + # Legacy path: plain dict, service still uses dict-based settings. + else: + await self._update_settings(frame.settings) elif isinstance(frame, LLMContextSummaryRequestFrame): await self._handle_summary_request(frame) diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index 4c34e28d5..97569fa1d 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -7,6 +7,7 @@ """LMNT text-to-speech service implementation.""" import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleTTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -71,6 +73,17 @@ def language_to_lmnt_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class LmntTTSSettings(TTSSettings): + """Typed settings for LMNT TTS service. + + Parameters: + format: Audio output format. Defaults to "raw". + """ + + format: str = field(default_factory=lambda: NOT_GIVEN) + + class LmntTTSService(InterruptibleTTSService): """LMNT real-time text-to-speech service. @@ -107,12 +120,14 @@ class LmntTTSService(InterruptibleTTSService): ) self._api_key = api_key - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) - self._settings = { - "language": self.language_to_service_language(language), - "format": "raw", # Use raw format for direct PCM data - } + self._settings: LmntTTSSettings = LmntTTSSettings( + model=model, + voice=voice_id, + language=self.language_to_service_language(language), + format="raw", + ) self._receive_task = None self._context_id: Optional[str] = None @@ -202,9 +217,9 @@ class LmntTTSService(InterruptibleTTSService): init_msg = { "X-API-Key": self._api_key, "voice": self._voice_id, - "format": self._settings["format"], + "format": self._settings.format, "sample_rate": self.sample_rate, - "language": self._settings["language"], + "language": self._settings.language, "model": self.model_name, } diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 7284d9630..6ce3e4b45 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -11,6 +11,7 @@ for streaming text-to-speech synthesis. """ import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -85,6 +87,40 @@ def language_to_minimax_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class MiniMaxTTSSettings(TTSSettings): + """Typed settings for MiniMax TTS service. + + Parameters: + stream: Whether to use streaming mode. + speed: Speech speed (range: 0.5 to 2.0). + volume: Speech volume (range: 0 to 10). + pitch: Pitch adjustment (range: -12 to 12). + emotion: Emotional tone (options: "happy", "sad", "angry", "fearful", + "disgusted", "surprised", "calm", "fluent"). + text_normalization: Enable text normalization (Chinese/English). + latex_read: Enable LaTeX formula reading. + audio_bitrate: Audio bitrate in bps. + audio_format: Audio output format. + audio_channel: Number of audio channels. + audio_sample_rate: Audio sample rate in Hz. + language_boost: Language boost string for multilingual support. + """ + + stream: bool = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + volume: float = field(default_factory=lambda: NOT_GIVEN) + pitch: int = field(default_factory=lambda: NOT_GIVEN) + emotion: str = field(default_factory=lambda: NOT_GIVEN) + text_normalization: bool = field(default_factory=lambda: NOT_GIVEN) + latex_read: bool = field(default_factory=lambda: NOT_GIVEN) + audio_bitrate: int = field(default_factory=lambda: NOT_GIVEN) + audio_format: str = field(default_factory=lambda: NOT_GIVEN) + audio_channel: int = field(default_factory=lambda: NOT_GIVEN) + audio_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + language_boost: str = field(default_factory=lambda: NOT_GIVEN) + + class MiniMaxHttpTTSService(TTSService): """Text-to-speech service using MiniMax's T2A (Text-to-Audio) API. @@ -172,29 +208,27 @@ class MiniMaxHttpTTSService(TTSService): self._voice_id = voice_id # Create voice settings - self._settings = { - "stream": True, - "voice_setting": { - "speed": params.speed, - "vol": params.volume, - "pitch": params.pitch, - }, - "audio_setting": { - "bitrate": 128000, - "format": "pcm", - "channel": 1, - }, - } + self._settings: MiniMaxTTSSettings = MiniMaxTTSSettings( + model=model, + voice=voice_id, + stream=True, + speed=params.speed, + volume=params.volume, + pitch=params.pitch, + audio_bitrate=128000, + audio_format="pcm", + audio_channel=1, + ) # Set voice and model - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) # Add language boost if provided if params.language: service_lang = self.language_to_service_language(params.language) if service_lang: - self._settings["language_boost"] = service_lang + self._settings.language_boost = service_lang # Add optional emotion if provided if params.emotion: @@ -210,7 +244,7 @@ class MiniMaxHttpTTSService(TTSService): "fluent", ] if params.emotion in supported_emotions: - self._settings["voice_setting"]["emotion"] = params.emotion + self._settings.emotion = params.emotion else: logger.warning( f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}" @@ -226,15 +260,15 @@ class MiniMaxHttpTTSService(TTSService): "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.", DeprecationWarning, ) - self._settings["voice_setting"]["text_normalization"] = params.english_normalization + self._settings.text_normalization = params.english_normalization # Add text_normalization if provided (corrected parameter name) if params.text_normalization is not None: - self._settings["voice_setting"]["text_normalization"] = params.text_normalization + self._settings.text_normalization = params.text_normalization # Add latex_read if provided if params.latex_read is not None: - self._settings["voice_setting"]["latex_read"] = params.latex_read + self._settings.latex_read = params.latex_read def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -263,16 +297,6 @@ class MiniMaxHttpTTSService(TTSService): """ self._model_name = model - def set_voice(self, voice: str): - """Set the voice to use. - - Args: - voice: The voice identifier to use for synthesis. - """ - self._voice_id = voice - if "voice_setting" in self._settings: - self._settings["voice_setting"]["voice_id"] = voice - async def start(self, frame: StartFrame): """Start the MiniMax TTS service. @@ -280,7 +304,7 @@ class MiniMaxHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["audio_setting"]["sample_rate"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}") @traced_tts @@ -302,10 +326,38 @@ class MiniMaxHttpTTSService(TTSService): "Authorization": f"Bearer {self._api_key}", } + # Build voice_setting dict for API + voice_setting = { + "voice_id": self._voice_id, + "speed": self._settings.speed, + "vol": self._settings.volume, + "pitch": self._settings.pitch, + } + if is_given(self._settings.emotion): + voice_setting["emotion"] = self._settings.emotion + if is_given(self._settings.text_normalization): + voice_setting["text_normalization"] = self._settings.text_normalization + if is_given(self._settings.latex_read): + voice_setting["latex_read"] = self._settings.latex_read + + # Build audio_setting dict for API + audio_setting = { + "bitrate": self._settings.audio_bitrate, + "format": self._settings.audio_format, + "channel": self._settings.audio_channel, + "sample_rate": self._settings.audio_sample_rate, + } + # Create payload from settings - payload = self._settings.copy() - payload["model"] = self._model_name - payload["text"] = text + payload = { + "stream": self._settings.stream, + "voice_setting": voice_setting, + "audio_setting": audio_setting, + "model": self._model_name, + "text": text, + } + if is_given(self._settings.language_boost): + payload["language_boost"] = self._settings.language_boost try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/mistral/llm.py b/src/pipecat/services/mistral/llm.py index 54361ef28..7a8f5b71a 100644 --- a/src/pipecat/services/mistral/llm.py +++ b/src/pipecat/services/mistral/llm.py @@ -185,19 +185,19 @@ class MistralLLMService(OpenAILLMService): "messages": fixed_messages, "tools": params_from_context["tools"], "tool_choice": params_from_context["tool_choice"], - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Handle Mistral-specific parameter mapping # Mistral uses "random_seed" instead of "seed" - if self._settings["seed"]: - params["random_seed"] = self._settings["seed"] + if self._settings.seed: + params["random_seed"] = self._settings.seed # Add any extra parameters - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 24eb05bd3..b7019e6d6 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -13,7 +13,8 @@ text-to-speech API for real-time audio synthesis. import asyncio import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Optional import aiohttp from loguru import logger @@ -34,6 +35,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -72,6 +74,23 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class NeuphonicTTSSettings(TTSSettings): + """Typed settings for Neuphonic TTS service. + + Parameters: + lang_code: Neuphonic language code. + speed: Speech speed multiplier. Defaults to 1.0. + encoding: Audio encoding format. + sampling_rate: Audio sample rate. + """ + + lang_code: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + encoding: str = field(default_factory=lambda: NOT_GIVEN) + sampling_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class NeuphonicTTSService(InterruptibleTTSService): """Neuphonic real-time text-to-speech service using WebSocket streaming. @@ -127,13 +146,13 @@ class NeuphonicTTSService(InterruptibleTTSService): self._api_key = api_key self._url = url - self._settings = { - "lang_code": self.language_to_service_language(params.language), - "speed": params.speed, - "encoding": encoding, - "sampling_rate": sample_rate, - } - self.set_voice(voice_id) + self._settings: NeuphonicTTSSettings = NeuphonicTTSSettings( + lang_code=self.language_to_service_language(params.language), + speed=params.speed, + encoding=encoding, + sampling_rate=sample_rate, + ) + self._voice_id = voice_id self._cumulative_time = 0 @@ -160,15 +179,14 @@ class NeuphonicTTSService(InterruptibleTTSService): """ return language_to_neuphonic_lang_code(language) - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect with new configuration.""" - if "voice_id" in settings: - self.set_voice(settings["voice_id"]) - - await super()._update_settings(settings) - await self._disconnect() - await self._connect() - logger.info(f"Switching TTS to settings: [{self._settings}]") + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect with new configuration.""" + changed = await super()._update_settings_from_typed(update) + if changed: + await self._disconnect() + await self._connect() + logger.info(f"Switching TTS to settings: [{self._settings}]") + return changed async def start(self, frame: StartFrame): """Start the Neuphonic TTS service. @@ -266,7 +284,10 @@ class NeuphonicTTSService(InterruptibleTTSService): logger.debug("Connecting to Neuphonic") tts_config = { - **self._settings, + "lang_code": self._settings.lang_code, + "speed": self._settings.speed, + "encoding": self._settings.encoding, + "sampling_rate": self._settings.sampling_rate, "voice_id": self._voice_id, } @@ -275,7 +296,7 @@ class NeuphonicTTSService(InterruptibleTTSService): if value is not None: query_params.append(f"{key}={value}") - url = f"{self._url}/speak/{self._settings['lang_code']}" + url = f"{self._url}/speak/{self._settings.lang_code}" if query_params: url += f"?{'&'.join(query_params)}" @@ -429,7 +450,7 @@ class NeuphonicHttpTTSService(TTSService): self._lang_code = self.language_to_service_language(params.language) or "en" self._speed = params.speed self._encoding = encoding - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. diff --git a/src/pipecat/services/nvidia/stt.py b/src/pipecat/services/nvidia/stt.py index 8eb6d7bb5..c65d6da62 100644 --- a/src/pipecat/services/nvidia/stt.py +++ b/src/pipecat/services/nvidia/stt.py @@ -8,6 +8,7 @@ import asyncio from concurrent.futures import CancelledError as FuturesCancelledError +from dataclasses import dataclass, field from typing import AsyncGenerator, List, Mapping, Optional from loguru import logger @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( StartFrame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import NVIDIA_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService, STTService from pipecat.transcriptions.language import Language, resolve_language @@ -89,6 +91,32 @@ def language_to_nvidia_riva_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class NvidiaSTTSettings(STTSettings): + """Typed settings for the NVIDIA Riva streaming STT service.""" + + pass + + +@dataclass +class NvidiaSegmentedSTTSettings(STTSettings): + """Typed settings for the NVIDIA Riva segmented STT service. + + Parameters: + profanity_filter: Whether to filter profanity from results. + automatic_punctuation: Whether to add automatic punctuation. + verbatim_transcripts: Whether to return verbatim transcripts. + boosted_lm_words: List of words to boost in language model. + boosted_lm_score: Score boost for specified words. + """ + + profanity_filter: bool = field(default_factory=lambda: NOT_GIVEN) + automatic_punctuation: bool = field(default_factory=lambda: NOT_GIVEN) + verbatim_transcripts: bool = field(default_factory=lambda: NOT_GIVEN) + boosted_lm_words: Optional[List[str]] = field(default_factory=lambda: NOT_GIVEN) + boosted_lm_score: float = field(default_factory=lambda: NOT_GIVEN) + + class NvidiaSTTService(STTService): """Real-time speech-to-text service using NVIDIA Riva streaming ASR. @@ -141,12 +169,6 @@ class NvidiaSTTService(STTService): self._server = server self._api_key = api_key self._use_ssl = use_ssl - self._profanity_filter = False - self._automatic_punctuation = True - self._no_verbatim_transcripts = False - self._language_code = params.language - self._boosted_lm_words = None - self._boosted_lm_score = 4.0 self._start_history = -1 self._start_threshold = -1.0 self._stop_history = -1 @@ -156,14 +178,9 @@ class NvidiaSTTService(STTService): self._custom_configuration = "" self._function_id = model_function_map.get("function_id") - self._settings = { - "language": str(params.language), - "profanity_filter": self._profanity_filter, - "automatic_punctuation": self._automatic_punctuation, - "verbatim_transcripts": not self._no_verbatim_transcripts, - "boosted_lm_words": self._boosted_lm_words, - "boosted_lm_score": self._boosted_lm_score, - } + self._settings: NvidiaSTTSettings = NvidiaSTTSettings( + language=params.language, + ) self.set_model_name(model_function_map.get("model_name")) @@ -186,22 +203,18 @@ class NvidiaSTTService(STTService): config = riva.client.StreamingRecognitionConfig( config=riva.client.RecognitionConfig( encoding=riva.client.AudioEncoding.LINEAR_PCM, - language_code=self._language_code, + language_code=self._settings.language, model="", max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=not self._no_verbatim_transcripts, + profanity_filter=False, + enable_automatic_punctuation=True, + verbatim_transcripts=True, sample_rate_hertz=self.sample_rate, audio_channel_count=1, ), interim_results=True, ) - riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score - ) - riva.client.add_endpoint_parameters_to_config( config, self._start_history, @@ -318,14 +331,14 @@ class NvidiaSTTService(STTService): transcript, self._user_id, time_now_iso8601(), - self._language_code, + self._settings.language, result=result, ) ) await self._handle_transcription( transcript=transcript, is_final=result.is_final, - language=self._language_code, + language=self._settings.language, ) else: await self.push_frame( @@ -333,7 +346,7 @@ class NvidiaSTTService(STTService): transcript, self._user_id, time_now_iso8601(), - self._language_code, + self._settings.language, result=result, ) ) @@ -445,18 +458,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): self._server = server self._use_ssl = use_ssl self._function_id = model_function_map.get("function_id") - self._model_name = model_function_map.get("model_name") - - # Store the language as a Language enum and as a string - self._language_enum = params.language or Language.EN_US - self._language = self.language_to_service_language(self._language_enum) or "en-US" - - # Configure transcription parameters - self._profanity_filter = params.profanity_filter - self._automatic_punctuation = params.automatic_punctuation - self._verbatim_transcripts = params.verbatim_transcripts - self._boosted_lm_words = params.boosted_lm_words - self._boosted_lm_score = params.boosted_lm_score # Voice activity detection thresholds (use NVIDIA Riva defaults) self._start_history = -1 @@ -467,10 +468,16 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): self._stop_threshold_eou = -1.0 self._custom_configuration = "" - # Create NVIDIA Riva client self._config = None self._asr_service = None - self._settings = {"language": self._language_enum} + self._settings: NvidiaSegmentedSTTSettings = NvidiaSegmentedSTTSettings( + language=params.language or Language.EN_US, + profanity_filter=params.profanity_filter, + automatic_punctuation=params.automatic_punctuation, + verbatim_transcripts=params.verbatim_transcripts, + boosted_lm_words=params.boosted_lm_words, + boosted_lm_score=params.boosted_lm_score, + ) def language_to_service_language(self, language: Language) -> Optional[str]: """Convert pipecat Language enum to NVIDIA Riva's language code. @@ -498,21 +505,25 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): auth = riva.client.Auth(None, self._use_ssl, self._server, metadata) self._asr_service = riva.client.ASRService(auth) + def _get_language_code(self) -> str: + """Resolve the current language enum to an NVIDIA Riva language code string.""" + return self.language_to_service_language(self._settings.language) or "en-US" + def _create_recognition_config(self): """Create the NVIDIA Riva ASR recognition configuration.""" # Create base configuration config = riva.client.RecognitionConfig( - language_code=self._language, # Now using the string, not a tuple + language_code=self._get_language_code(), max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=self._verbatim_transcripts, + profanity_filter=self._settings.profanity_filter, + enable_automatic_punctuation=self._settings.automatic_punctuation, + verbatim_transcripts=self._settings.verbatim_transcripts, ) # Add word boosting if specified - if self._boosted_lm_words: + if self._settings.boosted_lm_words: riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score + config, self._settings.boosted_lm_words, self._settings.boosted_lm_score ) # Add voice activity detection parameters @@ -567,20 +578,21 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): self._config = self._create_recognition_config() logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}") - async def set_language(self, language: Language): - """Set the language for the STT service. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and sync internal state. Args: - language: Target language for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._language_enum = language - self._language = self.language_to_service_language(language) or "en-US" - self._settings["language"] = language + update: A :class:`STTSettings` (or ``NvidiaSegmentedSTTSettings``) delta. - # Update configuration with new language - if self._config: - self._config.language_code = self._language + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if changed: + self._config = self._create_recognition_config() + + return changed @traced_stt async def _handle_transcription( @@ -633,11 +645,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): text, self._user_id, time_now_iso8601(), - self._language_enum, + self._settings.language, ) transcription_found = True - await self._handle_transcription(text, True, self._language_enum) + await self._handle_transcription(text, True, self._settings.language) if not transcription_found: logger.debug(f"{self}: No transcription results found in NVIDIA Riva response") diff --git a/src/pipecat/services/nvidia/tts.py b/src/pipecat/services/nvidia/tts.py index 6bac54e3a..8a018d6aa 100644 --- a/src/pipecat/services/nvidia/tts.py +++ b/src/pipecat/services/nvidia/tts.py @@ -100,7 +100,7 @@ class NvidiaTTSService(TTSService): self._function_id = model_function_map.get("function_id") self._use_ssl = use_ssl self.set_model_name(model_function_map.get("model_name")) - self.set_voice(voice_id) + self._voice_id = voice_id self._service = None self._config = None diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 2cdde51ea..2ac53794c 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -10,7 +10,8 @@ import asyncio import base64 import json from contextlib import asynccontextmanager -from typing import Any, Dict, List, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Mapping, Optional import httpx from loguru import logger @@ -32,7 +33,6 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, - LLMUpdateSettingsFrame, ) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext @@ -42,9 +42,24 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN +from pipecat.services.settings import LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm +@dataclass +class OpenAILLMSettings(LLMSettings): + """Typed settings for OpenAI-compatible LLM services. + + Parameters: + max_completion_tokens: Maximum completion tokens to generate. + service_tier: Service tier to use (e.g., "auto", "flex", "priority"). + """ + + max_completion_tokens: Any = field(default_factory=lambda: _NOT_GIVEN) + service_tier: Any = field(default_factory=lambda: _NOT_GIVEN) + + class BaseOpenAILLMService(LLMService): """Base class for all services that use the AsyncOpenAI client. @@ -120,17 +135,18 @@ class BaseOpenAILLMService(LLMService): params = params or BaseOpenAILLMService.InputParams() - self._settings = { - "frequency_penalty": params.frequency_penalty, - "presence_penalty": params.presence_penalty, - "seed": params.seed, - "temperature": params.temperature, - "top_p": params.top_p, - "max_tokens": params.max_tokens, - "max_completion_tokens": params.max_completion_tokens, - "service_tier": params.service_tier, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + self._settings = OpenAILLMSettings( + model=model, + frequency_penalty=params.frequency_penalty, + presence_penalty=params.presence_penalty, + seed=params.seed, + temperature=params.temperature, + top_p=params.top_p, + max_tokens=params.max_tokens, + max_completion_tokens=params.max_completion_tokens, + service_tier=params.service_tier, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout self.set_model_name(model) @@ -250,20 +266,20 @@ class BaseOpenAILLMService(LLMService): "model": self.model_name, "stream": True, "stream_options": {"include_usage": True}, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "seed": self._settings["seed"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], - "max_completion_tokens": self._settings["max_completion_tokens"], - "service_tier": self._settings["service_tier"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "seed": self._settings.seed, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, + "max_completion_tokens": self._settings.max_completion_tokens, + "service_tier": self._settings.service_tier, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params async def run_inference( @@ -508,8 +524,6 @@ class BaseOpenAILLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = OpenAILLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index cf249408c..abd66963b 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -10,8 +10,8 @@ import base64 import io import json import time -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger from PIL import Image @@ -59,6 +59,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt @@ -90,6 +91,17 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class OpenAIRealtimeLLMSettings(LLMSettings): + """Typed settings for OpenAI Realtime LLM services. + + Parameters: + session_properties: OpenAI Realtime session configuration. + """ + + session_properties: Any = field(default_factory=lambda: NOT_GIVEN) + + class OpenAIRealtimeLLMService(LLMService): """OpenAI Realtime LLM service providing real-time audio and text communication. @@ -161,9 +173,9 @@ class OpenAIRealtimeLLMService(LLMService): self.base_url = full_url self.set_model_name(model) - # Initialize session_properties - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = OpenAIRealtimeLLMSettings( + model=model, + session_properties=session_properties or events.SessionProperties(), ) self._audio_input_paused = start_audio_paused self._video_input_paused = start_video_paused @@ -227,12 +239,12 @@ class OpenAIRealtimeLLMService(LLMService): def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" - modalities = self._session_properties.output_modalities or ["audio", "text"] + modalities = self._settings.session_properties.output_modalities or ["audio", "text"] return modality in modalities def _get_enabled_modalities(self) -> list[str]: """Get the list of enabled modalities.""" - modalities = self._session_properties.output_modalities or ["audio", "text"] + modalities = self._settings.session_properties.output_modalities or ["audio", "text"] # API only supports single modality responses: either ["text"] or ["audio"] if "audio" in modalities: return ["audio"] @@ -305,9 +317,9 @@ class OpenAIRealtimeLLMService(LLMService): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. turn_detection_disabled = ( - self._session_properties.audio - and self._session_properties.audio.input - and self._session_properties.audio.input.turn_detection is False + self._settings.session_properties.audio + and self._settings.session_properties.audio.input + and self._settings.session_properties.audio.input.turn_detection is False ) if turn_detection_disabled: await self.send_client_event(events.InputAudioBufferClearEvent()) @@ -327,9 +339,9 @@ class OpenAIRealtimeLLMService(LLMService): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. turn_detection_disabled = ( - self._session_properties.audio - and self._session_properties.audio.input - and self._session_properties.audio.input.turn_detection is False + self._settings.session_properties.audio + and self._settings.session_properties.audio.input + and self._settings.session_properties.audio.input.turn_detection is False ) if turn_detection_disabled: await self.send_client_event(events.InputAudioBufferCommitEvent()) @@ -397,6 +409,16 @@ class OpenAIRealtimeLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Legacy dict path: frame.settings contains SessionProperties fields, + # not our Settings fields, so we construct SessionProperties directly. + # The new typed path (frame.update) falls through to super, which calls + # _update_settings_from_typed → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._update_settings() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -424,9 +446,6 @@ class OpenAIRealtimeLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() @@ -513,8 +532,15 @@ class OpenAIRealtimeLLMService(LLMService): # treat a send-side error as fatal. await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) + async def _update_settings_from_typed(self, update): + """Apply a typed settings update, sending a session update if needed.""" + changed = await super()._update_settings_from_typed(update) + if "session_properties" in changed: + await self._update_settings() + return changed + async def _update_settings(self): - settings = self._session_properties + settings = self._settings.session_properties adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() if self._context: diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index 4dd16be6e..12eada24e 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -16,6 +16,7 @@ Provides two STT services: import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Literal, Optional, Union from loguru import logger @@ -34,6 +35,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -123,6 +125,17 @@ class OpenAISTTService(BaseWhisperSTTService): _OPENAI_SAMPLE_RATE = 24000 +@dataclass +class OpenAIRealtimeSTTSettings(STTSettings): + """Typed settings for the OpenAI Realtime STT service. + + Parameters: + prompt: Optional prompt text to guide transcription style. + """ + + prompt: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + + class OpenAIRealtimeSTTService(WebsocketSTTService): """OpenAI Realtime Speech-to-Text service using WebSocket transcription sessions. @@ -213,12 +226,17 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): self._base_url = base_url self.set_model_name(model) - self._language_code = self._language_to_code(language) if language else None self._prompt = prompt self._turn_detection = turn_detection self._noise_reduction = noise_reduction self._should_interrupt = should_interrupt + self._settings: OpenAIRealtimeSTTSettings = OpenAIRealtimeSTTSettings( + model=model, + language=language, + prompt=prompt, + ) + self._receive_task = None self._session_ready = False self._resampler = create_stream_resampler() @@ -248,19 +266,31 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the language for speech recognition. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and send session update if needed. - If the session is already active, sends an updated configuration - to the server. + Keeps ``_language_code`` and ``_prompt`` in sync with typed settings + and sends a ``session.update`` to the server when the session is active. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``OpenAIRealtimeSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - self._language_code = self._language_to_code(language) + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + if "prompt" in changed and isinstance(self._settings, OpenAIRealtimeSTTSettings): + self._prompt = self._settings.prompt + if self._session_ready: await self._send_session_update() + return changed + async def start(self, frame: StartFrame): """Start the service and establish WebSocket connection. @@ -407,8 +437,11 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): """Send ``session.update`` to configure the transcription session.""" transcription: dict = {"model": self.model_name} - if self._language_code: - transcription["language"] = self._language_code + language_code = ( + self._language_to_code(self._settings.language) if self._settings.language else None + ) + if language_code: + transcription["language"] = language_code if self._prompt: transcription["prompt"] = self._prompt diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index f59f0b31b..ee1e34316 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -10,6 +10,7 @@ This module provides integration with OpenAI's text-to-speech API for generating high-quality synthetic speech from text input. """ +from dataclasses import dataclass, field from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -60,6 +62,19 @@ VALID_VOICES: Dict[str, ValidVoice] = { } +@dataclass +class OpenAITTSSettings(TTSSettings): + """Typed settings for OpenAI TTS service. + + Parameters: + instructions: Instructions to guide voice synthesis behavior. + speed: Voice speed control (0.25 to 4.0, default 1.0). + """ + + instructions: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + + class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. @@ -118,7 +133,7 @@ class OpenAITTSService(TTSService): super().__init__(sample_rate=sample_rate, **kwargs) self.set_model_name(model) - self.set_voice(voice) + self._voice_id = voice self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) if instructions or speed: @@ -132,10 +147,12 @@ class OpenAITTSService(TTSService): stacklevel=2, ) - self._settings = { - "instructions": params.instructions if params else instructions, - "speed": params.speed if params else speed, - } + self._settings: OpenAITTSSettings = OpenAITTSSettings( + model=model, + voice=voice, + instructions=params.instructions if params else instructions, + speed=params.speed if params else speed, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -145,15 +162,6 @@ class OpenAITTSService(TTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model to use. - - Args: - model: The model name to use for text-to-speech synthesis. - """ - logger.info(f"Switching TTS model to: [{model}]") - self.set_model_name(model) - async def start(self, frame: StartFrame): """Start the OpenAI TTS service. @@ -190,11 +198,11 @@ class OpenAITTSService(TTSService): "response_format": "pcm", } - if self._settings["instructions"]: - create_params["instructions"] = self._settings["instructions"] + if self._settings.instructions: + create_params["instructions"] = self._settings.instructions - if self._settings["speed"]: - create_params["speed"] = self._settings["speed"] + if self._settings.speed: + create_params["speed"] = self._settings.speed async with self._client.audio.speech.with_streaming_response.create( **create_params diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 1199d8556..d37b1434e 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -10,8 +10,8 @@ import base64 import json import time import warnings -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger @@ -54,6 +54,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.openai.llm import OpenAIContextAggregatorPair +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt @@ -91,6 +92,17 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class OpenAIRealtimeBetaLLMSettings(LLMSettings): + """Typed settings for OpenAI Realtime Beta LLM services. + + Parameters: + session_properties: OpenAI Realtime session configuration. + """ + + session_properties: Any = field(default_factory=lambda: NOT_GIVEN) + + class OpenAIRealtimeBetaLLMService(LLMService): """OpenAI Realtime Beta LLM service providing real-time audio and text communication. @@ -146,8 +158,9 @@ class OpenAIRealtimeBetaLLMService(LLMService): self.base_url = full_url self.set_model_name(model) - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = OpenAIRealtimeBetaLLMSettings( + model=model, + session_properties=session_properties or events.SessionProperties(), ) self._audio_input_paused = start_audio_paused self._send_transcription_frames = send_transcription_frames @@ -187,12 +200,12 @@ class OpenAIRealtimeBetaLLMService(LLMService): def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" - modalities = self._session_properties.modalities or ["audio", "text"] + modalities = self._settings.session_properties.modalities or ["audio", "text"] return modality in modalities def _get_enabled_modalities(self) -> list[str]: """Get the list of enabled modalities.""" - return self._session_properties.modalities or ["audio", "text"] + return self._settings.session_properties.modalities or ["audio", "text"] async def retrieve_conversation_item(self, item_id: str): """Retrieve a conversation item by ID from the server. @@ -259,7 +272,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_interruption(self): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. - if self._session_properties.turn_detection is False: + if self._settings.session_properties.turn_detection is False: await self.send_client_event(events.InputAudioBufferClearEvent()) await self.send_client_event(events.ResponseCancelEvent()) await self._truncate_current_audio_response() @@ -276,7 +289,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_user_stopped_speaking(self, frame): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. - if self._session_properties.turn_detection is False: + if self._settings.session_properties.turn_detection is False: await self.send_client_event(events.InputAudioBufferCommitEvent()) await self.send_client_event(events.ResponseCreateEvent()) @@ -342,6 +355,16 @@ class OpenAIRealtimeBetaLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Legacy dict path: frame.settings contains SessionProperties fields, + # not our Settings fields, so we construct SessionProperties directly. + # The new typed path (frame.update) falls through to super, which calls + # _update_settings_from_typed → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._update_settings() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -377,9 +400,6 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_messages_append(frame) elif isinstance(frame, RealtimeMessagesUpdateFrame): self._context = frame.context - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() elif isinstance(frame, RealtimeFunctionCallResultFrame): @@ -456,8 +476,15 @@ class OpenAIRealtimeBetaLLMService(LLMService): # treat a send-side error as fatal. await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) + async def _update_settings_from_typed(self, update): + """Apply a typed settings update, sending a session update if needed.""" + changed = await super()._update_settings_from_typed(update) + if "session_properties" in changed: + await self._update_settings() + return changed + async def _update_settings(self): - settings = self._session_properties + settings = self._settings.session_properties # tools given in the context override the tools in the session properties if self._context and self._context.tools: settings.tools = self._context.tools diff --git a/src/pipecat/services/perplexity/llm.py b/src/pipecat/services/perplexity/llm.py index 4ea23aa82..d2dd40a57 100644 --- a/src/pipecat/services/perplexity/llm.py +++ b/src/pipecat/services/perplexity/llm.py @@ -72,16 +72,16 @@ class PerplexityLLMService(OpenAILLMService): } # Add OpenAI-compatible parameters if they're set - if self._settings["frequency_penalty"] is not NOT_GIVEN: - params["frequency_penalty"] = self._settings["frequency_penalty"] - if self._settings["presence_penalty"] is not NOT_GIVEN: - params["presence_penalty"] = self._settings["presence_penalty"] - if self._settings["temperature"] is not NOT_GIVEN: - params["temperature"] = self._settings["temperature"] - if self._settings["top_p"] is not NOT_GIVEN: - params["top_p"] = self._settings["top_p"] - if self._settings["max_tokens"] is not NOT_GIVEN: - params["max_tokens"] = self._settings["max_tokens"] + if self._settings.frequency_penalty is not NOT_GIVEN: + params["frequency_penalty"] = self._settings.frequency_penalty + if self._settings.presence_penalty is not NOT_GIVEN: + params["presence_penalty"] = self._settings.presence_penalty + if self._settings.temperature is not NOT_GIVEN: + params["temperature"] = self._settings.temperature + if self._settings.top_p is not NOT_GIVEN: + params["top_p"] = self._settings.top_p + if self._settings.max_tokens is not NOT_GIVEN: + params["max_tokens"] = self._settings.max_tokens return params diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 2d4cd0427..f79f54560 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -14,6 +14,7 @@ import io import json import struct import warnings +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -32,6 +33,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -97,6 +99,25 @@ def language_to_playht_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class PlayHTTTSSettings(TTSSettings): + """Typed settings for PlayHT TTS services. + + Parameters: + output_format: Audio output format. + voice_engine: Voice engine to use. + speed: Speech speed multiplier. Defaults to 1.0. + seed: Random seed for voice consistency. + playht_sample_rate: Audio sample rate sent to the API. + """ + + output_format: str = field(default_factory=lambda: NOT_GIVEN) + voice_engine: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + seed: int = field(default_factory=lambda: NOT_GIVEN) + playht_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class PlayHTTTSService(InterruptibleTTSService): """PlayHT WebSocket-based text-to-speech service. @@ -170,17 +191,19 @@ class PlayHTTTSService(InterruptibleTTSService): self._receive_task = None self._context_id = None - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: PlayHTTTSSettings = PlayHTTTSSettings( + model=voice_engine, + voice=voice_url, + language=self.language_to_service_language(params.language) if params.language else "english", - "output_format": output_format, - "voice_engine": voice_engine, - "speed": params.speed, - "seed": params.seed, - } + output_format=output_format, + voice_engine=voice_engine, + speed=params.speed, + seed=params.seed, + ) self.set_model_name(voice_engine) - self.set_voice(voice_url) + self._voice_id = voice_url def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -304,13 +327,13 @@ class PlayHTTTSService(InterruptibleTTSService): # Handle the new response format with multiple URLs if "websocket_urls" in data: # Select URL based on voice_engine - if self._settings["voice_engine"] in data["websocket_urls"]: + if self._settings.voice_engine in data["websocket_urls"]: self._websocket_url = data["websocket_urls"][ - self._settings["voice_engine"] + self._settings.voice_engine ] else: raise ValueError( - f"Unsupported voice engine: {self._settings['voice_engine']}" + f"Unsupported voice engine: {self._settings.voice_engine}" ) else: raise ValueError("Invalid response: missing websocket_urls") @@ -382,12 +405,12 @@ class PlayHTTTSService(InterruptibleTTSService): tts_command = { "text": text, "voice": self._voice_id, - "voice_engine": self._settings["voice_engine"], - "output_format": self._settings["output_format"], + "voice_engine": self._settings.voice_engine, + "output_format": self._settings.output_format, "sample_rate": self.sample_rate, - "language": self._settings["language"], - "speed": self._settings["speed"], - "seed": self._settings["seed"], + "language": self._settings.language, + "speed": self._settings.speed, + "seed": self._settings.seed, "request_id": self._context_id, } @@ -499,17 +522,18 @@ class PlayHTHttpTTSService(TTSService): # Extract the base engine name voice_engine = voice_engine.replace("-ws", "") - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: PlayHTTTSSettings = PlayHTTTSSettings( + voice=voice_url, + language=self.language_to_service_language(params.language) if params.language else "english", - "output_format": output_format, - "voice_engine": voice_engine, - "speed": params.speed, - "seed": params.seed, - } + output_format=output_format, + voice_engine=voice_engine, + speed=params.speed, + seed=params.seed, + ) self.set_model_name(voice_engine) - self.set_voice(voice_url) + self._voice_id = voice_url async def start(self, frame: StartFrame): """Start the PlayHT HTTP TTS service. @@ -518,7 +542,7 @@ class PlayHTHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.playht_sample_rate = self.sample_rate def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -559,17 +583,17 @@ class PlayHTHttpTTSService(TTSService): payload = { "text": text, "voice": self._voice_id, - "voice_engine": self._settings["voice_engine"], - "output_format": self._settings["output_format"], + "voice_engine": self._settings.voice_engine, + "output_format": self._settings.output_format, "sample_rate": self.sample_rate, - "language": self._settings["language"], + "language": self._settings.language, } # Add optional parameters if they exist - if self._settings["speed"] is not None: - payload["speed"] = self._settings["speed"] - if self._settings["seed"] is not None: - payload["seed"] = self._settings["seed"] + if self._settings.speed is not None: + payload["speed"] = self._settings.speed + if self._settings.seed is not None: + payload["seed"] = self._settings.seed headers = { "Authorization": f"Bearer {self._api_key}", diff --git a/src/pipecat/services/resembleai/tts.py b/src/pipecat/services/resembleai/tts.py index 964b9fa18..08f9b81bd 100644 --- a/src/pipecat/services/resembleai/tts.py +++ b/src/pipecat/services/resembleai/tts.py @@ -8,6 +8,7 @@ import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import AudioContextWordTTSService from pipecat.transcriptions.language import Language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -38,6 +40,21 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class ResembleAITTSSettings(TTSSettings): + """Typed settings for Resemble AI TTS service. + + Parameters: + precision: PCM bit depth (PCM_32, PCM_24, PCM_16, or MULAW). + output_format: Audio format (wav or mp3). + resemble_sample_rate: Audio sample rate sent to the API. + """ + + precision: str = field(default_factory=lambda: NOT_GIVEN) + output_format: str = field(default_factory=lambda: NOT_GIVEN) + resemble_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class ResembleAITTSService(AudioContextWordTTSService): """Resemble AI TTS service with WebSocket streaming and word timestamps. @@ -76,11 +93,12 @@ class ResembleAITTSService(AudioContextWordTTSService): self._api_key = api_key self._voice_id = voice_id self._url = url - self._settings = { - "precision": precision, - "output_format": output_format, - "sample_rate": sample_rate, - } + self._settings: ResembleAITTSSettings = ResembleAITTSSettings( + voice=voice_id, + precision=precision, + output_format=output_format, + resemble_sample_rate=sample_rate, + ) self._websocket = None self._request_id_counter = 0 @@ -101,7 +119,7 @@ class ResembleAITTSService(AudioContextWordTTSService): self._jitter_buffer_bytes = 44100 # ~1000ms at 22050Hz to handle 400ms+ network gaps self._playback_started: dict[str, bool] = {} # Track if we've started playback per request - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -125,9 +143,9 @@ class ResembleAITTSService(AudioContextWordTTSService): "data": text, "binary_response": False, # Use JSON frames to get timestamps "request_id": self._request_id_counter, # ResembleAI only accepts number - "output_format": self._settings["output_format"], - "sample_rate": self._settings["sample_rate"], - "precision": self._settings["precision"], + "output_format": self._settings.output_format, + "sample_rate": self._settings.resemble_sample_rate, + "precision": self._settings.precision, "no_audio_header": True, } @@ -141,7 +159,7 @@ class ResembleAITTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.resemble_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index e38e840e6..5a3ed67a2 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -12,7 +12,8 @@ using Rime's API for streaming and batch audio synthesis. import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Optional import aiohttp from loguru import logger @@ -30,6 +31,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import ( AudioContextWordTTSService, InterruptibleTTSService, @@ -68,6 +70,62 @@ def language_to_rime_language(language: Language) -> str: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class RimeTTSSettings(TTSSettings): + """Typed settings for Rime WS JSON and HTTP TTS services. + + Parameters: + speaker: Voice speaker ID. + modelId: Rime model identifier. + audioFormat: Audio output format. + samplingRate: Audio sample rate. + lang: Rime language code. + speedAlpha: Speech speed multiplier. Defaults to 1.0. + reduceLatency: Whether to reduce latency at potential quality cost. + pauseBetweenBrackets: Whether to add pauses between bracketed content. + phonemizeBetweenBrackets: Whether to phonemize bracketed content. + inlineSpeedAlpha: Inline speed control markup. + """ + + speaker: str = field(default_factory=lambda: NOT_GIVEN) + modelId: str = field(default_factory=lambda: NOT_GIVEN) + audioFormat: str = field(default_factory=lambda: NOT_GIVEN) + samplingRate: int = field(default_factory=lambda: NOT_GIVEN) + lang: str = field(default_factory=lambda: NOT_GIVEN) + speedAlpha: float = field(default_factory=lambda: NOT_GIVEN) + reduceLatency: bool = field(default_factory=lambda: NOT_GIVEN) + pauseBetweenBrackets: bool = field(default_factory=lambda: NOT_GIVEN) + phonemizeBetweenBrackets: bool = field(default_factory=lambda: NOT_GIVEN) + inlineSpeedAlpha: str = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class RimeNonJsonTTSSettings(TTSSettings): + """Typed settings for Rime non-JSON WS TTS service. + + Parameters: + speaker: Voice speaker ID. + modelId: Rime model identifier. + audioFormat: Audio output format. + samplingRate: Audio sample rate. + lang: Rime language code. + segment: Text segmentation mode ("immediate", "bySentence", "never"). + repetition_penalty: Token repetition penalty (1.0-2.0). + temperature: Sampling temperature (0.0-1.0). + top_p: Cumulative probability threshold (0.0-1.0). + """ + + speaker: str = field(default_factory=lambda: NOT_GIVEN) + modelId: str = field(default_factory=lambda: NOT_GIVEN) + audioFormat: str = field(default_factory=lambda: NOT_GIVEN) + samplingRate: int = field(default_factory=lambda: NOT_GIVEN) + lang: str = field(default_factory=lambda: NOT_GIVEN) + segment: str = field(default_factory=lambda: NOT_GIVEN) + repetition_penalty: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + top_p: float = field(default_factory=lambda: NOT_GIVEN) + + class RimeTTSService(AudioContextWordTTSService): """Text-to-Speech service using Rime's websocket API. @@ -149,19 +207,17 @@ class RimeTTSService(AudioContextWordTTSService): self._url = url self._voice_id = voice_id self._model = model - self._settings = { - "speaker": voice_id, - "modelId": model, - "audioFormat": "pcm", - "samplingRate": 0, - "lang": self.language_to_service_language(params.language) - if params.language - else "eng", - "speedAlpha": params.speed_alpha, - "reduceLatency": params.reduce_latency, - "pauseBetweenBrackets": json.dumps(params.pause_between_brackets), - "phonemizeBetweenBrackets": json.dumps(params.phonemize_between_brackets), - } + self._settings: RimeTTSSettings = RimeTTSSettings( + speaker=voice_id, + modelId=model, + audioFormat="pcm", + samplingRate=0, + lang=self.language_to_service_language(params.language) if params.language else "eng", + speedAlpha=params.speed_alpha, + reduceLatency=params.reduce_latency, + pauseBetweenBrackets=json.dumps(params.pause_between_brackets), + phonemizeBetweenBrackets=json.dumps(params.phonemize_between_brackets), + ) # State tracking self._context_id = None # Tracks current turn @@ -188,15 +244,6 @@ class RimeTTSService(AudioContextWordTTSService): """ return language_to_rime_language(language) - async def set_model(self, model: str): - """Update the TTS model. - - Args: - model: The model name to use for synthesis. - """ - self._model = model - await super().set_model(model) - # A set of Rime-specific helpers for text transformations def SPELL(text: str) -> str: """Wrap text in Rime spell function.""" @@ -222,15 +269,15 @@ class RimeTTSService(AudioContextWordTTSService): self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)]) return f"[{text}]" - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if voice changed.""" prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - self._settings["speaker"] = self._voice_id - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + changed = await super()._update_settings_from_typed(update) + if "voice" in changed: + self._settings.speaker = self._voice_id await self._disconnect() await self._connect() + return changed def _build_msg(self, text: str = "") -> dict: """Build JSON message for Rime API.""" @@ -255,7 +302,7 @@ class RimeTTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["samplingRate"] = self.sample_rate + self._settings.samplingRate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -301,7 +348,20 @@ class RimeTTSService(AudioContextWordTTSService): if self._websocket and self._websocket.state is State.OPEN: return - params = "&".join(f"{k}={v}" for k, v in self._settings.items()) + params = "&".join( + f"{k}={v}" + for k, v in { + "speaker": self._settings.speaker, + "modelId": self._settings.modelId, + "audioFormat": self._settings.audioFormat, + "samplingRate": self._settings.samplingRate, + "lang": self._settings.lang, + "speedAlpha": self._settings.speedAlpha, + "reduceLatency": self._settings.reduceLatency, + "pauseBetweenBrackets": self._settings.pauseBetweenBrackets, + "phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets, + }.items() + ) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect(url, additional_headers=headers) @@ -525,21 +585,17 @@ class RimeHttpTTSService(TTSService): self._api_key = api_key self._session = aiohttp_session self._base_url = "https://users.rime.ai/v1/rime-tts" - self._settings = { - "lang": self.language_to_service_language(params.language) - if params.language - else "eng", - "speedAlpha": params.speed_alpha, - "reduceLatency": params.reduce_latency, - "pauseBetweenBrackets": params.pause_between_brackets, - "phonemizeBetweenBrackets": params.phonemize_between_brackets, - } - self.set_voice(voice_id) + self._settings: RimeTTSSettings = RimeTTSSettings( + lang=self.language_to_service_language(params.language) if params.language else "eng", + speedAlpha=params.speed_alpha, + reduceLatency=params.reduce_latency, + pauseBetweenBrackets=params.pause_between_brackets, + phonemizeBetweenBrackets=params.phonemize_between_brackets, + inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else NOT_GIVEN, + ) + self._voice_id = voice_id self.set_model_name(model) - if params.inline_speed_alpha: - self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha - def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -578,7 +634,15 @@ class RimeHttpTTSService(TTSService): "Content-Type": "application/json", } - payload = self._settings.copy() + payload = { + "lang": self._settings.lang, + "speedAlpha": self._settings.speedAlpha, + "reduceLatency": self._settings.reduceLatency, + "pauseBetweenBrackets": self._settings.pauseBetweenBrackets, + "phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets, + } + if is_given(self._settings.inlineSpeedAlpha): + payload["inlineSpeedAlpha"] = self._settings.inlineSpeedAlpha payload["text"] = text payload["speaker"] = self._voice_id payload["modelId"] = self._model_name @@ -699,26 +763,24 @@ class RimeNonJsonTTSService(InterruptibleTTSService): self._url = url self._voice_id = voice_id self._model = model - self._settings = { - "speaker": voice_id, - "modelId": model, - "audioFormat": audio_format, - "samplingRate": sample_rate, - } - - if params.language: - self._settings["lang"] = self.language_to_service_language(params.language) - if params.segment is not None: - self._settings["segment"] = params.segment - if params.repetition_penalty is not None: - self._settings["repetition_penalty"] = params.repetition_penalty - if params.temperature is not None: - self._settings["temperature"] = params.temperature - if params.top_p is not None: - self._settings["top_p"] = params.top_p + self._settings: RimeNonJsonTTSSettings = RimeNonJsonTTSSettings( + speaker=voice_id, + modelId=model, + audioFormat=audio_format, + samplingRate=sample_rate, + lang=self.language_to_service_language(params.language) + if params.language + else NOT_GIVEN, + segment=params.segment if params.segment is not None else NOT_GIVEN, + repetition_penalty=params.repetition_penalty + if params.repetition_penalty is not None + else NOT_GIVEN, + temperature=params.temperature if params.temperature is not None else NOT_GIVEN, + top_p=params.top_p if params.top_p is not None else NOT_GIVEN, + ) # Add any extra parameters for future compatibility if params.extra: - self._settings.update(params.extra) + self._settings.extra.update(params.extra) self._receive_task = None self._context_id: Optional[str] = None @@ -750,7 +812,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["samplingRate"] = self.sample_rate + self._settings.samplingRate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -794,8 +856,26 @@ class RimeNonJsonTTSService(InterruptibleTTSService): try: if self._websocket and self._websocket.state is State.OPEN: return - # Build URL with query parameters (only non-None values) - params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None) + # Build URL with query parameters (only given, non-None values) + settings_dict = { + "speaker": self._settings.speaker, + "modelId": self._settings.modelId, + "audioFormat": self._settings.audioFormat, + "samplingRate": self._settings.samplingRate, + } + if is_given(self._settings.lang): + settings_dict["lang"] = self._settings.lang + if is_given(self._settings.segment): + settings_dict["segment"] = self._settings.segment + if is_given(self._settings.repetition_penalty): + settings_dict["repetition_penalty"] = self._settings.repetition_penalty + if is_given(self._settings.temperature): + settings_dict["temperature"] = self._settings.temperature + if is_given(self._settings.top_p): + settings_dict["top_p"] = self._settings.top_p + # Include extras + settings_dict.update(self._settings.extra) + params = "&".join(f"{k}={v}" for k, v in settings_dict.items() if v is not None) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect( @@ -889,68 +969,23 @@ class RimeNonJsonTTSService(InterruptibleTTSService): except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if necessary. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if necessary. Since all settings are WebSocket URL query parameters, any setting change requires reconnecting to apply the new values. """ - needs_reconnect = False + changed = await super()._update_settings_from_typed(update) - # Track previous values from self._settings only - prev_settings = self._settings.copy() + # Sync voice and model to settings dict fields + if "voice" in changed: + self._settings.speaker = self._voice_id + if "model" in changed: + self._settings.modelId = self._model_name - # Let parent class handle standard settings (voice, model, language) - await super()._update_settings(settings) - - # Check if voice changed and update settings dict - if "voice" in settings or "voice_id" in settings: - self._settings["speaker"] = self._voice_id - if prev_settings.get("speaker") != self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") - needs_reconnect = True - - # Check if model changed and update settings dict - if "model" in settings: - self._settings["modelId"] = self._model - if prev_settings.get("modelId") != self._model: - logger.info(f"Switching TTS model to: [{self._model}]") - needs_reconnect = True - - # Handle language explicitly - if "language" in settings: - new_lang = self.language_to_service_language(settings["language"]) - if new_lang and new_lang != prev_settings.get("lang"): - logger.info(f"Updating language to: [{new_lang}]") - self._settings["lang"] = new_lang - needs_reconnect = True - - # Check other parameters - for key in ["segment", "repetition_penalty", "temperature", "top_p"]: - if key in settings and settings[key] != prev_settings.get(key): - logger.info(f"Updating {key} to: [{settings[key]}]") - self._settings[key] = settings[key] - needs_reconnect = True - - # Handle extra parameters - for key, value in settings.items(): - if key not in [ - "voice", - "voice_id", - "model", - "language", - "segment", - "repetition_penalty", - "temperature", - "top_p", - ]: - if value != prev_settings.get(key): - logger.info(f"Updating extra parameter {key} to: [{value}]") - self._settings[key] = value - needs_reconnect = True - - # Reconnect if any setting changed - if needs_reconnect: + if changed: logger.debug("Settings changed, reconnecting WebSocket with new parameters") await self._disconnect() await self._connect() + + return changed diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index 047ce0e6c..99c7bca2c 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -87,16 +87,16 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore "model": self.model_name, "stream": True, "stream_options": {"include_usage": True}, - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], - "max_completion_tokens": self._settings["max_completion_tokens"], + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, + "max_completion_tokens": self._settings.max_completion_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params @traced_llm # type: ignore diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index 998597956..e2bc6a08f 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -12,7 +12,7 @@ can handle multiple audio formats for Indian language speech recognition. """ import base64 -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger @@ -32,6 +32,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.sarvam._sdk import sdk_headers +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import SARVAM_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -130,6 +131,23 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = { } +@dataclass +class SarvamSTTSettings(STTSettings): + """Typed settings for the Sarvam STT service. + + Parameters: + prompt: Optional prompt to guide transcription/translation style. + mode: Mode of operation (transcribe, translate, verbatim, etc.). + vad_signals: Enable VAD signals in response. + high_vad_sensitivity: Enable high VAD sensitivity. + """ + + prompt: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + mode: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + vad_signals: Optional[bool] = field(default_factory=lambda: NOT_GIVEN) + high_vad_sensitivity: Optional[bool] = field(default_factory=lambda: NOT_GIVEN) + + class SarvamSTTService(STTService): """Sarvam speech-to-text service. @@ -207,22 +225,8 @@ class SarvamSTTService(STTService): self.set_model_name(model) self._api_key = api_key - self._language_code: Optional[Language] = params.language - - # Set language string: use provided language or model's default - if params.language: - self._language_string = language_to_sarvam_language(params.language) - else: - self._language_string = self._config.default_language - - self._prompt = params.prompt - - # Set mode: use provided mode or model's default - self._mode = params.mode if params.mode is not None else self._config.default_mode # Store connection parameters - self._vad_signals = params.vad_signals - self._high_vad_sensitivity = params.high_vad_sensitivity self._input_audio_codec = input_audio_codec # Initialize Sarvam SDK client @@ -240,7 +244,19 @@ class SarvamSTTService(STTService): self._socket_client = None self._receive_task = None - if self._vad_signals: + # Resolve mode default from model config + mode = params.mode if params.mode is not None else self._config.default_mode + + self._settings: SarvamSTTSettings = SarvamSTTSettings( + model=model, + language=params.language, + prompt=params.prompt if params.prompt is not None else NOT_GIVEN, + mode=mode if mode is not None else NOT_GIVEN, + vad_signals=params.vad_signals, + high_vad_sensitivity=params.high_vad_sensitivity, + ) + + if params.vad_signals: self._register_event_handler("on_speech_started") self._register_event_handler("on_speech_stopped") self._register_event_handler("on_utterance_end") @@ -258,6 +274,12 @@ class SarvamSTTService(STTService): """ return language_to_sarvam_language(language) + def _get_language_string(self) -> Optional[str]: + """Resolve the current language setting to a Sarvam language code string.""" + if self._settings.language: + return language_to_sarvam_language(self._settings.language) + return self._config.default_language + def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -275,42 +297,74 @@ class SarvamSTTService(STTService): await super().process_frame(frame, direction) # Only handle VAD frames when not using Sarvam's VAD signals - if not self._vad_signals: + if not self._settings.vad_signals: if isinstance(frame, VADUserStartedSpeakingFrame): await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): if self._socket_client: await self._socket_client.flush() - async def set_language(self, language: Language): - """Set the recognition language and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, validate, sync state, and reconnect. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``SarvamSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. Raises: - ValueError: If called on a model that auto-detects language. + ValueError: If a setting is not supported by the current model. """ - if not self._config.supports_language: - raise ValueError( - f"Model '{self.model_name}' does not support language parameter " - "(auto-detects language)." - ) + # Validate against model capabilities before applying + if is_given(update.language) and update.language is not None: + if not self._config.supports_language: + raise ValueError( + f"Model '{self.model_name}' does not support language parameter " + "(auto-detects language)." + ) + + if isinstance(update, SarvamSTTSettings): + if is_given(update.prompt) and update.prompt is not None: + if not self._config.supports_prompt: + raise ValueError( + f"Model '{self.model_name}' does not support prompt parameter." + ) + if is_given(update.mode) and update.mode is not None: + if not self._config.supports_mode: + raise ValueError(f"Model '{self.model_name}' does not support mode parameter.") + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed - logger.info(f"Switching STT language to: [{language}]") - self._language_code = language - self._language_string = language_to_sarvam_language(language) await self._disconnect() await self._connect() + return changed async def set_prompt(self, prompt: Optional[str]): """Set the transcription/translation prompt and reconnect. + .. deprecated:: + Use ``STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...))`` instead. + Args: prompt: Prompt text to guide transcription/translation style/context. Pass None to clear/disable prompt. Only applicable to models that support prompts. """ + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + f"{self.__class__.__name__}.set_prompt() is deprecated. " + "Use STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...)) instead.", + DeprecationWarning, + stacklevel=2, + ) + if not self._config.supports_prompt: if prompt is not None: raise ValueError(f"Model '{self.model_name}' does not support prompt parameter.") @@ -318,7 +372,7 @@ class SarvamSTTService(STTService): return logger.info(f"Updating {self.model_name} prompt.") - self._prompt = prompt + self._settings.prompt = prompt await self._disconnect() await self._connect() @@ -405,24 +459,25 @@ class SarvamSTTService(STTService): # Enable flush signal when using Pipecat's VAD (not Sarvam's) so that # the flush() call on user-stopped-speaking is honored by the server. - if not self._vad_signals: + if not self._settings.vad_signals: connect_kwargs["flush_signal"] = "true" # Only send vad parameters when explicitly set (avoid overriding server defaults) - if self._vad_signals is not None: - connect_kwargs["vad_signals"] = "true" if self._vad_signals else "false" - if self._high_vad_sensitivity is not None: + if self._settings.vad_signals is not None: + connect_kwargs["vad_signals"] = "true" if self._settings.vad_signals else "false" + if self._settings.high_vad_sensitivity is not None: connect_kwargs["high_vad_sensitivity"] = ( - "true" if self._high_vad_sensitivity else "false" + "true" if self._settings.high_vad_sensitivity else "false" ) # Add language_code for models that support it - if self._language_string is not None: - connect_kwargs["language_code"] = self._language_string + language_string = self._get_language_string() + if language_string is not None: + connect_kwargs["language_code"] = language_string # Add mode for models that support it - if self._config.supports_mode and self._mode is not None: - connect_kwargs["mode"] = self._mode + if self._config.supports_mode and is_given(self._settings.mode): + connect_kwargs["mode"] = self._settings.mode def _connect_with_sdk_headers(connect_fn, **kwargs): # Different SDK versions may use different kwarg names. @@ -449,8 +504,8 @@ class SarvamSTTService(STTService): self._socket_client = await self._websocket_context.__aenter__() # Set prompt if provided (only for models that support prompts) - if self._prompt is not None and self._config.supports_prompt: - await self._socket_client.set_prompt(self._prompt) + if is_given(self._settings.prompt) and self._config.supports_prompt: + await self._socket_client.set_prompt(self._settings.prompt) # Register event handler for incoming messages def _message_handler(message): @@ -544,10 +599,12 @@ class SarvamSTTService(STTService): # Prefer language from message (auto-detected for translate models). Fallback to configured. if language_code: language = self._map_language_code_to_enum(language_code) - elif self._language_string: - language = self._map_language_code_to_enum(self._language_string) else: - language = Language.HI_IN + language_string = self._get_language_string() + if language_string: + language = self._map_language_code_to_enum(language_string) + else: + language = Language.HI_IN # Emit utterance end event await self._call_event_handler("on_utterance_end") diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 753293c75..e28914b4c 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -40,9 +40,9 @@ See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for full API import asyncio import base64 import json -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple +from typing import AsyncGenerator, Dict, List, Optional, Tuple import aiohttp from loguru import logger @@ -62,6 +62,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.sarvam._sdk import sdk_headers +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -244,6 +245,80 @@ def language_to_sarvam_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class SarvamHttpTTSSettings(TTSSettings): + """Typed settings for Sarvam HTTP TTS service. + + Parameters: + language: Sarvam language code. + enable_preprocessing: Whether to enable text preprocessing. Defaults to False. + **Note:** Always enabled for bulbul:v3-beta (cannot be disabled). + pace: Speech pace multiplier. Defaults to 1.0. + - bulbul:v2: Range 0.3 to 3.0 + - bulbul:v3-beta: Range 0.5 to 2.0 + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0). + Lower values = more deterministic, higher = more random. Defaults to 0.6. + **Note:** Only supported for bulbul:v3-beta. Ignored for v2. + sample_rate: Audio sample rate. + """ + + language: str = field(default_factory=lambda: NOT_GIVEN) + enable_preprocessing: bool = field(default_factory=lambda: NOT_GIVEN) + pace: float = field(default_factory=lambda: NOT_GIVEN) + pitch: float = field(default_factory=lambda: NOT_GIVEN) + loudness: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + sarvam_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class SarvamWSTTSSettings(TTSSettings): + """Typed settings for Sarvam WebSocket TTS service. + + Parameters: + target_language_code: Sarvam language code. + speaker: Voice speaker ID. + speech_sample_rate: Audio sample rate as string. + enable_preprocessing: Enable text preprocessing. Defaults to False. + **Note:** Always enabled for bulbul:v3-beta. + min_buffer_size: Minimum characters to buffer before generating audio. + Lower values reduce latency but may affect quality. Defaults to 50. + max_chunk_length: Maximum characters processed in a single chunk. + Controls memory usage and processing efficiency. Defaults to 150. + output_audio_codec: Audio codec format. Options: linear16, mulaw, alaw, + opus, flac, aac, wav, mp3. Defaults to "linear16". + output_audio_bitrate: Audio bitrate (32k, 64k, 96k, 128k, 192k). + Defaults to "128k". + pace: Speech pace multiplier. Defaults to 1.0. + - bulbul:v2: Range 0.3 to 3.0 + - bulbul:v3-beta: Range 0.5 to 2.0 + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0). + Lower = more deterministic, higher = more random. Defaults to 0.6. + **Note:** Only supported for bulbul:v3-beta. Ignored for v2. + """ + + target_language_code: str = field(default_factory=lambda: NOT_GIVEN) + speaker: str = field(default_factory=lambda: NOT_GIVEN) + speech_sample_rate: str = field(default_factory=lambda: NOT_GIVEN) + enable_preprocessing: bool = field(default_factory=lambda: NOT_GIVEN) + min_buffer_size: int = field(default_factory=lambda: NOT_GIVEN) + max_chunk_length: int = field(default_factory=lambda: NOT_GIVEN) + output_audio_codec: str = field(default_factory=lambda: NOT_GIVEN) + output_audio_bitrate: str = field(default_factory=lambda: NOT_GIVEN) + pace: float = field(default_factory=lambda: NOT_GIVEN) + pitch: float = field(default_factory=lambda: NOT_GIVEN) + loudness: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + + class SarvamHttpTTSService(TTSService): """Text-to-Speech service using Sarvam AI's API. @@ -403,35 +478,35 @@ class SarvamHttpTTSService(TTSService): pace = max(pace_min, min(pace_max, pace)) # Build base settings - self._settings = { - "language": ( + self._settings: SarvamHttpTTSSettings = SarvamHttpTTSSettings( + language=( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "enable_preprocessing": ( + enable_preprocessing=( True if self._config.preprocessing_always_enabled else params.enable_preprocessing ), - "pace": pace, - "model": model, - } + pace=pace, + model=model, + ) # Add parameters based on model support if self._config.supports_pitch: - self._settings["pitch"] = params.pitch + self._settings.pitch = params.pitch elif params.pitch != 0.0: logger.warning(f"pitch parameter is ignored for {model}") if self._config.supports_loudness: - self._settings["loudness"] = params.loudness + self._settings.loudness = params.loudness elif params.loudness != 1.0: logger.warning(f"loudness parameter is ignored for {model}") if self._config.supports_temperature: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature elif params.temperature != 0.6: logger.warning(f"temperature parameter is ignored for {model}") self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -459,7 +534,7 @@ class SarvamHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.sarvam_sample_rate = self.sample_rate @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -480,21 +555,25 @@ class SarvamHttpTTSService(TTSService): # Build payload with common parameters payload = { "text": text, - "target_language_code": self._settings["language"], + "target_language_code": self._settings.language, "speaker": self._voice_id, "sample_rate": self.sample_rate, - "enable_preprocessing": self._settings["enable_preprocessing"], + "enable_preprocessing": self._settings.enable_preprocessing, "model": self._model_name, - "pace": self._settings.get("pace", 1.0), + "pace": self._settings.pace if is_given(self._settings.pace) else 1.0, } # Add model-specific parameters based on config if self._config.supports_pitch: - payload["pitch"] = self._settings.get("pitch", 0.0) + payload["pitch"] = self._settings.pitch if is_given(self._settings.pitch) else 0.0 if self._config.supports_loudness: - payload["loudness"] = self._settings.get("loudness", 1.0) + payload["loudness"] = ( + self._settings.loudness if is_given(self._settings.loudness) else 1.0 + ) if self._config.supports_temperature: - payload["temperature"] = self._settings.get("temperature", 0.6) + payload["temperature"] = ( + self._settings.temperature if is_given(self._settings.temperature) else 0.6 + ) headers = { "api-subscription-key": self._api_key, @@ -748,7 +827,7 @@ class SarvamTTSService(InterruptibleTTSService): self._websocket_url = f"{url}?model={model}" self._api_key = api_key self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id # Validate and clamp pace to model's valid range pace = params.pace @@ -758,36 +837,36 @@ class SarvamTTSService(InterruptibleTTSService): pace = max(pace_min, min(pace_max, pace)) # Build base settings - self._settings = { - "target_language_code": ( + self._settings: SarvamWSTTSSettings = SarvamWSTTSSettings( + target_language_code=( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "speaker": voice_id, - "speech_sample_rate": str(sample_rate), - "enable_preprocessing": ( + speaker=voice_id, + speech_sample_rate=str(sample_rate), + enable_preprocessing=( True if self._config.preprocessing_always_enabled else params.enable_preprocessing ), - "min_buffer_size": params.min_buffer_size, - "max_chunk_length": params.max_chunk_length, - "output_audio_codec": params.output_audio_codec, - "output_audio_bitrate": params.output_audio_bitrate, - "pace": pace, - "model": model, - } + min_buffer_size=params.min_buffer_size, + max_chunk_length=params.max_chunk_length, + output_audio_codec=params.output_audio_codec, + output_audio_bitrate=params.output_audio_bitrate, + pace=pace, + model=model, + ) # Add parameters based on model support if self._config.supports_pitch: - self._settings["pitch"] = params.pitch + self._settings.pitch = params.pitch elif params.pitch != 0.0: logger.warning(f"pitch parameter is ignored for {model}") if self._config.supports_loudness: - self._settings["loudness"] = params.loudness + self._settings.loudness = params.loudness elif params.loudness != 1.0: logger.warning(f"loudness parameter is ignored for {model}") if self._config.supports_temperature: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature elif params.temperature != 0.6: logger.warning(f"temperature parameter is ignored for {model}") @@ -823,7 +902,7 @@ class SarvamTTSService(InterruptibleTTSService): await super().start(frame) # WebSocket API expects sample rate as string - self._settings["speech_sample_rate"] = str(self.sample_rate) + self._settings.speech_sample_rate = str(self.sample_rate) await self._connect() async def stop(self, frame: EndFrame): @@ -870,13 +949,12 @@ class SarvamTTSService(InterruptibleTTSService): if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): await self.flush_audio() - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" - prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and resend config if voice changed.""" + changed = await super()._update_settings_from_typed(update) + if "voice" in changed: await self._send_config() + return changed async def _connect(self): """Connect to Sarvam WebSocket and start background tasks.""" @@ -934,9 +1012,28 @@ class SarvamTTSService(InterruptibleTTSService): """Send initial configuration message.""" if not self._websocket: raise Exception("WebSocket not connected") - self._settings["speaker"] = self._voice_id - logger.debug(f"Config being sent is {self._settings}") - config_message = {"type": "config", "data": self._settings} + self._settings.speaker = self._voice_id + # Build config dict for the API + config_data = { + "target_language_code": self._settings.target_language_code, + "speaker": self._settings.speaker, + "speech_sample_rate": self._settings.speech_sample_rate, + "enable_preprocessing": self._settings.enable_preprocessing, + "min_buffer_size": self._settings.min_buffer_size, + "max_chunk_length": self._settings.max_chunk_length, + "output_audio_codec": self._settings.output_audio_codec, + "output_audio_bitrate": self._settings.output_audio_bitrate, + "pace": self._settings.pace, + "model": self._settings.model, + } + if is_given(self._settings.pitch): + config_data["pitch"] = self._settings.pitch + if is_given(self._settings.loudness): + config_data["loudness"] = self._settings.loudness + if is_given(self._settings.temperature): + config_data["temperature"] = self._settings.temperature + logger.debug(f"Config being sent is {config_data}") + config_message = {"type": "config", "data": config_data} try: await self._websocket.send(json.dumps(config_message)) diff --git a/src/pipecat/services/settings.py b/src/pipecat/services/settings.py new file mode 100644 index 000000000..fbec5cdf8 --- /dev/null +++ b/src/pipecat/services/settings.py @@ -0,0 +1,297 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Typed settings infrastructure for Pipecat AI services. + +This module provides typed dataclass-based settings objects that replace the +stringly-typed ``Mapping[str, Any]`` dictionaries previously used for service +configuration. Each service type has a corresponding settings class (e.g. +``TTSSettings``, ``LLMSettings``) whose fields use the ``NOT_GIVEN`` sentinel +to distinguish "leave unchanged" from an explicit ``None``. + +Key concepts: + +- **NOT_GIVEN sentinel**: A value meaning "this field was not provided in the + update". Distinct from ``None`` (which may be a valid value for a setting). +- **Settings as both state and delta**: The same class is used for the + service's current settings *and* for update objects. Fields set to + ``NOT_GIVEN`` are simply skipped when applying an update. +- **apply_update**: Applies a delta onto a target settings object and returns + the set of field names that actually changed. +- **from_mapping**: Constructs a typed settings object from a plain dict, + supporting field aliases (e.g. ``"voice_id"`` → ``"voice"``). +- **Extras**: Unknown keys land in the ``extra`` dict so services that have + non-standard settings don't lose data. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass, field, fields +from typing import Any, ClassVar, Dict, Mapping, Optional, Set, Type, TypeVar + +from loguru import logger + +# --------------------------------------------------------------------------- +# NOT_GIVEN sentinel +# --------------------------------------------------------------------------- + + +class _NotGiven: + """Sentinel indicating a settings field was not provided. + + ``NOT_GIVEN`` means "the caller did not supply this value" — distinct from + ``None``, which may be a legitimate setting value. It is used as the + default for every settings field so that ``apply_update`` can tell which + fields the caller actually wants to change. + """ + + _instance: Optional[_NotGiven] = None + + def __new__(cls) -> _NotGiven: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __repr__(self) -> str: + return "NOT_GIVEN" + + def __bool__(self) -> bool: + return False + + +NOT_GIVEN: _NotGiven = _NotGiven() +"""Singleton sentinel meaning "this field was not included in the update".""" + + +def is_given(value: Any) -> bool: + """Check whether a value was explicitly provided (i.e. is not ``NOT_GIVEN``). + + Args: + value: The value to check. + + Returns: + ``True`` if *value* is anything other than ``NOT_GIVEN``. + """ + return not isinstance(value, _NotGiven) + + +# --------------------------------------------------------------------------- +# Base ServiceSettings +# --------------------------------------------------------------------------- + +_S = TypeVar("_S", bound="ServiceSettings") + + +@dataclass +class ServiceSettings: + """Base class for typed service settings. + + Every AI service type (LLM, TTS, STT) extends this with its own fields. + Fields default to ``NOT_GIVEN`` so that an instance can represent either + the full current state **or** a sparse update delta. + + Parameters: + model: The model identifier used by the service. + extra: Overflow dict for service-specific keys that don't map to a + declared field. + """ + + # -- common fields ------------------------------------------------------- + + model: Any = field(default_factory=lambda: NOT_GIVEN) + """AI model identifier (e.g. ``"gpt-4o"``, ``"eleven_turbo_v2_5"``).""" + + extra: Dict[str, Any] = field(default_factory=dict) + """Catch-all for service-specific keys that have no declared field.""" + + # -- class-level configuration ------------------------------------------- + + _aliases: ClassVar[Dict[str, str]] = {} + """Map of alternative key names to canonical field names. + + For example ``{"voice_id": "voice"}`` lets callers use either spelling. + Subclasses should override this as needed. + """ + + # -- public API ---------------------------------------------------------- + + def given_fields(self) -> Dict[str, Any]: + """Return a dict of only the fields that were explicitly provided. + + Skips ``NOT_GIVEN`` values and the ``extra`` field itself. Entries + from ``extra`` are included at the top level. + + Returns: + Dictionary mapping field names to their provided values. + """ + result: Dict[str, Any] = {} + for f in fields(self): + if f.name == "extra": + continue + val = getattr(self, f.name) + if is_given(val): + result[f.name] = val + result.update(self.extra) + return result + + def apply_update(self: _S, update: _S) -> Set[str]: + """Apply *update* onto this settings object, returning changed field names. + + Only fields in *update* that are **given** (i.e. not ``NOT_GIVEN``) + are considered. A field is "changed" if its new value differs from + the current value. + + The ``extra`` dicts are merged: keys present in the update overwrite + keys in the target. + + Args: + update: A settings object of the same type containing the delta. + + Returns: + The set of field names whose values actually changed. + + Examples:: + + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="bob") + changed = current.apply_update(delta) + # changed == {"voice"} + # current.voice == "bob", current.language == "en" + """ + changed: Set[str] = set() + for f in fields(self): + if f.name == "extra": + continue + new_val = getattr(update, f.name) + if not is_given(new_val): + continue + old_val = getattr(self, f.name) + if old_val != new_val: + setattr(self, f.name, new_val) + changed.add(f.name) + + # Merge extra + for key, new_val in update.extra.items(): + old_val = self.extra.get(key, NOT_GIVEN) + if old_val != new_val: + self.extra[key] = new_val + changed.add(key) + + return changed + + @classmethod + def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S: + """Construct a typed settings object from a plain dictionary. + + Keys are matched to dataclass fields by name. Keys listed in + ``_aliases`` are translated to their canonical name first. Any + remaining unrecognized keys are placed into ``extra``. + + Args: + settings: A dictionary of setting names to values. + + Returns: + A new settings instance with the corresponding fields populated. + + Examples:: + + update = TTSSettings.from_mapping({"voice_id": "alice", "speed": 1.2}) + # update.voice == "alice" (via alias) + # update.extra == {"speed": 1.2} + """ + field_names = {f.name for f in fields(cls)} - {"extra"} + kwargs: Dict[str, Any] = {} + extra: Dict[str, Any] = {} + + for key, value in settings.items(): + # Resolve aliases first + canonical = cls._aliases.get(key, key) + if canonical in field_names: + kwargs[canonical] = value + else: + extra[key] = value + + instance = cls(**kwargs) + instance.extra = extra + return instance + + def to_dict(self) -> Dict[str, Any]: + """Serialize to a flat dictionary, including extra. + + Only given (non-``NOT_GIVEN``) values are included. This is the + inverse of ``from_mapping`` and useful for passing settings to APIs + that expect plain dicts. + + Returns: + A flat dictionary of all given settings. + """ + return self.given_fields() + + def copy(self: _S) -> _S: + """Return a deep copy of this settings instance. + + Returns: + A new settings object with the same field values. + """ + return copy.deepcopy(self) + + +# --------------------------------------------------------------------------- +# Service-specific settings +# --------------------------------------------------------------------------- + + +@dataclass +class LLMSettings(ServiceSettings): + """Typed settings for LLM services. + + Parameters: + model: LLM model identifier. + temperature: Sampling temperature. + max_tokens: Maximum tokens to generate. + top_p: Nucleus sampling probability. + top_k: Top-k sampling parameter. + frequency_penalty: Frequency penalty. + presence_penalty: Presence penalty. + seed: Random seed for reproducibility. + """ + + temperature: Any = field(default_factory=lambda: NOT_GIVEN) + max_tokens: Any = field(default_factory=lambda: NOT_GIVEN) + top_p: Any = field(default_factory=lambda: NOT_GIVEN) + top_k: Any = field(default_factory=lambda: NOT_GIVEN) + frequency_penalty: Any = field(default_factory=lambda: NOT_GIVEN) + presence_penalty: Any = field(default_factory=lambda: NOT_GIVEN) + seed: Any = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class TTSSettings(ServiceSettings): + """Typed settings for TTS services. + + Parameters: + model: TTS model identifier. + voice: Voice identifier or name. + language: Language for speech synthesis. + """ + + voice: Any = field(default_factory=lambda: NOT_GIVEN) + language: Any = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + +@dataclass +class STTSettings(ServiceSettings): + """Typed settings for STT services. + + Parameters: + model: STT model identifier. + language: Language for speech recognition. + """ + + language: Any = field(default_factory=lambda: NOT_GIVEN) diff --git a/src/pipecat/services/soniox/stt.py b/src/pipecat/services/soniox/stt.py index c9184ba4c..9d732a356 100644 --- a/src/pipecat/services/soniox/stt.py +++ b/src/pipecat/services/soniox/stt.py @@ -8,7 +8,8 @@ import json import time -from typing import AsyncGenerator, List, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, List, Optional from loguru import logger from pydantic import BaseModel @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import SONIOX_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -134,6 +136,17 @@ def _prepare_language_hints( return list(set(prepared_languages)) +@dataclass +class SonioxSTTSettings(STTSettings): + """Typed settings for Soniox STT service. + + Parameters: + input_params: Soniox ``SonioxInputParams`` for detailed configuration. + """ + + input_params: SonioxInputParams = field(default_factory=lambda: NOT_GIVEN) + + class SonioxSTTService(WebsocketSTTService): """Speech-to-Text service using Soniox's WebSocket API. @@ -181,9 +194,13 @@ class SonioxSTTService(WebsocketSTTService): self._api_key = api_key self._url = url self.set_model_name(params.model) - self._params = params self._vad_force_turn_endpoint = vad_force_turn_endpoint + self._settings = SonioxSTTSettings( + model=params.model, + input_params=params, + ) + self._final_transcription_buffer = [] self._last_tokens_received: Optional[float] = None @@ -198,6 +215,43 @@ class SonioxSTTService(WebsocketSTTService): await super().start(frame) await self._connect() + async def _update_settings_from_typed(self, update: SonioxSTTSettings) -> set[str]: + """Apply a typed settings update, keeping ``input_params`` in sync. + + Top-level ``model`` is the source of truth. When it is given in + *update* its value is propagated into ``input_params``. When only + ``input_params`` is given, its ``model`` is propagated *up* to the + top-level field. + + Any change triggers a WebSocket reconnect. + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + model_given = is_given(getattr(update, "model", NOT_GIVEN)) + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # --- Sync model -------------------------------------------------- + if model_given: + # Top-level model wins → push into input_params. + self._settings.input_params.model = self._settings.model + elif "input_params" in changed and self._settings.input_params.model is not None: + # Only input_params was given → pull model up. + self._settings.model = self._settings.input_params.model + self.set_model_name(self._settings.model) + + await self._disconnect() + await self._connect() + + return changed + async def stop(self, frame: EndFrame): """Stop the Soniox STT websocket connection. @@ -311,7 +365,9 @@ class SonioxSTTService(WebsocketSTTService): # Either one or the other is required. enable_endpoint_detection = not self._vad_force_turn_endpoint - context = self._params.context + params = self._settings.input_params + + context = params.context if isinstance(context, SonioxContextObject): context = context.model_dump() @@ -319,16 +375,16 @@ class SonioxSTTService(WebsocketSTTService): config = { "api_key": self._api_key, "model": self._model_name, - "audio_format": self._params.audio_format, - "num_channels": self._params.num_channels or 1, + "audio_format": params.audio_format, + "num_channels": params.num_channels or 1, "enable_endpoint_detection": enable_endpoint_detection, "sample_rate": self.sample_rate, - "language_hints": _prepare_language_hints(self._params.language_hints), - "language_hints_strict": self._params.language_hints_strict, + "language_hints": _prepare_language_hints(params.language_hints), + "language_hints_strict": params.language_hints_strict, "context": context, - "enable_speaker_diarization": self._params.enable_speaker_diarization, - "enable_language_identification": self._params.enable_language_identification, - "client_reference_id": self._params.client_reference_id, + "enable_speaker_diarization": params.enable_speaker_diarization, + "enable_language_identification": params.enable_language_identification, + "client_reference_id": params.client_reference_id, } # Send the configuration message. diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index ca949a9fd..d04bb564d 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -8,8 +8,10 @@ import asyncio import os +import warnings +from dataclasses import dataclass, field from enum import Enum -from typing import Any, AsyncGenerator +from typing import Any, AsyncGenerator, ClassVar from dotenv import load_dotenv from loguru import logger @@ -31,6 +33,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import SPEECHMATICS_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -80,6 +83,81 @@ class TurnDetectionMode(str, Enum): SMART_TURN = "smart_turn" +@dataclass +class SpeechmaticsSTTSettings(STTSettings): + """Typed settings for Speechmatics STT service. + + See ``SpeechmaticsSTTService.InputParams`` for detailed descriptions of each field. + + Parameters: + model: The operating point / model name. + domain: Domain for Speechmatics API. + turn_detection_mode: Endpoint handling mode. + speaker_active_format: Formatter for active speaker ID. + speaker_passive_format: Formatter for passive speaker ID. + focus_speakers: List of speaker IDs to focus on. + ignore_speakers: List of speaker IDs to ignore. + focus_mode: Speaker focus mode for diarization. + known_speakers: List of known speaker labels and identifiers. + additional_vocab: List of additional vocabulary entries. + audio_encoding: Audio encoding format. + operating_point: Operating point for accuracy vs. latency. + max_delay: Maximum delay in seconds for transcription. + end_of_utterance_silence_trigger: Maximum delay for end of utterance trigger. + end_of_utterance_max_delay: Maximum delay for end of utterance. + punctuation_overrides: Punctuation overrides. + include_partials: Include partial segment fragments. + split_sentences: Emit finalized sentences mid-turn. + enable_diarization: Enable speaker diarization. + speaker_sensitivity: Diarization sensitivity. + max_speakers: Maximum number of speakers to detect. + prefer_current_speaker: Prefer current speaker ID. + extra_params: Extra parameters for the STT engine. + """ + + domain: str = field(default_factory=lambda: NOT_GIVEN) + turn_detection_mode: TurnDetectionMode = field(default_factory=lambda: NOT_GIVEN) + speaker_active_format: str = field(default_factory=lambda: NOT_GIVEN) + speaker_passive_format: str = field(default_factory=lambda: NOT_GIVEN) + focus_speakers: list = field(default_factory=lambda: NOT_GIVEN) + ignore_speakers: list = field(default_factory=lambda: NOT_GIVEN) + focus_mode: Any = field(default_factory=lambda: NOT_GIVEN) + known_speakers: list = field(default_factory=lambda: NOT_GIVEN) + additional_vocab: list = field(default_factory=lambda: NOT_GIVEN) + audio_encoding: Any = field(default_factory=lambda: NOT_GIVEN) + operating_point: Any = field(default_factory=lambda: NOT_GIVEN) + max_delay: float = field(default_factory=lambda: NOT_GIVEN) + end_of_utterance_silence_trigger: float = field(default_factory=lambda: NOT_GIVEN) + end_of_utterance_max_delay: float = field(default_factory=lambda: NOT_GIVEN) + punctuation_overrides: dict = field(default_factory=lambda: NOT_GIVEN) + include_partials: bool = field(default_factory=lambda: NOT_GIVEN) + split_sentences: bool = field(default_factory=lambda: NOT_GIVEN) + enable_diarization: bool = field(default_factory=lambda: NOT_GIVEN) + speaker_sensitivity: float = field(default_factory=lambda: NOT_GIVEN) + max_speakers: int = field(default_factory=lambda: NOT_GIVEN) + prefer_current_speaker: bool = field(default_factory=lambda: NOT_GIVEN) + extra_params: dict = field(default_factory=lambda: NOT_GIVEN) + + #: Fields that can be updated on a live connection via the Speechmatics + #: diarization-config API — no reconnect needed. + HOT_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "focus_speakers", + "ignore_speakers", + "focus_mode", + } + ) + + #: Fields that are purely local (formatting templates) — no reconnect + #: and no API call needed. + LOCAL_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "speaker_active_format", + "speaker_passive_format", + } + ) + + class SpeechmaticsSTTService(STTService): """Speechmatics STT service implementation. @@ -327,30 +405,56 @@ class SpeechmaticsSTTService(STTService): # Deprecation check self._check_deprecated_args(kwargs, params) - # Voice agent + # Output formatting defaults + speaker_active_format = params.speaker_active_format + if speaker_active_format is None: + speaker_active_format = ( + "@{speaker_id}: {text}" if params.enable_diarization else "{text}" + ) + speaker_passive_format = params.speaker_passive_format or speaker_active_format + + # Typed settings — seeded from InputParams + self._settings = SpeechmaticsSTTSettings( + language=params.language, + domain=params.domain, + turn_detection_mode=params.turn_detection_mode, + speaker_active_format=speaker_active_format, + speaker_passive_format=speaker_passive_format, + focus_speakers=params.focus_speakers, + ignore_speakers=params.ignore_speakers, + focus_mode=params.focus_mode, + known_speakers=params.known_speakers, + additional_vocab=params.additional_vocab, + audio_encoding=params.audio_encoding, + operating_point=params.operating_point, + max_delay=params.max_delay, + end_of_utterance_silence_trigger=params.end_of_utterance_silence_trigger, + end_of_utterance_max_delay=params.end_of_utterance_max_delay, + punctuation_overrides=params.punctuation_overrides, + include_partials=params.include_partials, + split_sentences=params.split_sentences, + enable_diarization=params.enable_diarization, + speaker_sensitivity=params.speaker_sensitivity, + max_speakers=params.max_speakers, + prefer_current_speaker=params.prefer_current_speaker, + extra_params=params.extra_params, + ) + + # Build SDK config from settings self._client: VoiceAgentClient | None = None - self._config: VoiceAgentConfig = self._prepare_config(params) + self._config: VoiceAgentConfig = self._build_config() # Outbound frame queue self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue() - # Output formatting - if params.speaker_active_format is None: - params.speaker_active_format = ( - "@{speaker_id}: {text}" if params.enable_diarization else "{text}" - ) - # Framework options self._enable_vad: bool = self._config.end_of_utterance_mode not in [ EndOfUtteranceMode.FIXED, EndOfUtteranceMode.EXTERNAL, ] - self._speaker_active_format: str = params.speaker_active_format - self._speaker_passive_format: str = ( - params.speaker_passive_format or params.speaker_active_format - ) - # Model + metrics + # Model + metrics (operating_point comes from the SDK config/preset) + self._settings.model = self._config.operating_point.value self.set_model_name(self._config.operating_point.value) # Message queue @@ -374,6 +478,56 @@ class SpeechmaticsSTTService(STTService): await super().start(frame) await self._connect() + async def _update_settings_from_typed(self, update: SpeechmaticsSTTSettings) -> set[str]: + """Apply typed settings update, reconnecting only when necessary. + + Fields are classified into three categories (see + ``SpeechmaticsSTTSettings``): + + * **HOT_FIELDS** – diarization speaker settings that can be pushed + to a live Speechmatics connection without reconnecting. + * **LOCAL_FIELDS** – formatting templates evaluated locally; no + reconnect or API call needed. + * Everything else – baked into ``VoiceAgentConfig`` at connection + time and therefore require a full disconnect / reconnect. + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + no_reconnect = SpeechmaticsSTTSettings.HOT_FIELDS | SpeechmaticsSTTSettings.LOCAL_FIELDS + needs_reconnect = bool(changed - no_reconnect) + + if needs_reconnect: + # Connection-level fields changed — rebuild the SDK config + # from the now-updated self._settings, then reconnect. + self._config = self._build_config() + await self._disconnect() + await self._connect() + elif changed & SpeechmaticsSTTSettings.HOT_FIELDS: + if self._config.enable_diarization: + # Only hot-updatable fields changed — push to the live session. + self._config.speaker_config.focus_speakers = self._settings.focus_speakers + self._config.speaker_config.ignore_speakers = self._settings.ignore_speakers + self._config.speaker_config.focus_mode = self._settings.focus_mode + if self._client: + self._client.update_diarization_config(self._config.speaker_config) + else: + # Diarization not enabled — need a full reconnect to apply. + self._config = self._build_config() + await self._disconnect() + await self._connect() + # LOCAL_FIELDS: already applied by super(); nothing else to do. + + return changed + async def stop(self, frame: EndFrame): """Called when the session ends.""" await super().stop(frame) @@ -484,28 +638,35 @@ class SpeechmaticsSTTService(STTService): # CONFIGURATION # ============================================================================ - def _prepare_config(self, params: InputParams) -> VoiceAgentConfig: - """Parse the InputParams into VoiceAgentConfig.""" - # Preset - config = VoiceAgentConfigPreset.load(params.turn_detection_mode.value) + def _build_config(self) -> VoiceAgentConfig: + """Build a ``VoiceAgentConfig`` from the current ``self._settings``. + + Used both at init time and before reconnecting so the connection + always reflects the latest settings. + """ + s = self._settings + + # Preset from turn detection mode + config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value) # Language + domain - config.language = self._language_to_speechmatics_language(params.language) - config.domain = params.domain - config.output_locale = self._locale_to_speechmatics_locale(config.language, params.language) + language = s.language + config.language = self._language_to_speechmatics_language(language) + config.domain = s.domain if is_given(s.domain) else None + config.output_locale = self._locale_to_speechmatics_locale(config.language, language) # Speaker config config.speaker_config = SpeakerFocusConfig( - focus_speakers=params.focus_speakers, - ignore_speakers=params.ignore_speakers, - focus_mode=params.focus_mode, + focus_speakers=s.focus_speakers if is_given(s.focus_speakers) else [], + ignore_speakers=s.ignore_speakers if is_given(s.ignore_speakers) else [], + focus_mode=s.focus_mode if is_given(s.focus_mode) else SpeakerFocusMode.RETAIN, ) - config.known_speakers = params.known_speakers + config.known_speakers = s.known_speakers if is_given(s.known_speakers) else [] # Custom dictionary - config.additional_vocab = params.additional_vocab + config.additional_vocab = s.additional_vocab if is_given(s.additional_vocab) else [] - # Advanced parameters + # Advanced parameters — only set if given (not NOT_GIVEN or None) for param in [ "operating_point", "max_delay", @@ -519,21 +680,20 @@ class SpeechmaticsSTTService(STTService): "max_speakers", "prefer_current_speaker", ]: - if getattr(params, param) is not None: - setattr(config, param, getattr(params, param)) + val = getattr(s, param) + if is_given(val) and val is not None: + setattr(config, param, val) # Extra parameters - if isinstance(params.extra_params, dict): - for key, value in params.extra_params.items(): + if is_given(s.extra_params) and isinstance(s.extra_params, dict): + for key, value in s.extra_params.items(): if hasattr(config, key): setattr(config, key, value) # Enable sentences - config.speech_segment_config = SpeechSegmentConfig( - emit_sentences=params.split_sentences or False - ) + split = s.split_sentences if is_given(s.split_sentences) else False + config.speech_segment_config = SpeechSegmentConfig(emit_sentences=split or False) - # Return the complete config return config def update_params( @@ -542,12 +702,23 @@ class SpeechmaticsSTTService(STTService): ) -> None: """Updates the speaker configuration. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with + ``SpeechmaticsSTTSettings(...)`` instead. + This can update the speakers to listen to or ignore during an in-flight transcription. Only available if diarization is enabled. Args: params: Update parameters for the service. """ + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "update_params() is deprecated. Use STTUpdateSettingsFrame with " + "SpeechmaticsSTTSettings(...) instead.", + DeprecationWarning, + ) # Check possible if not self._config.enable_diarization: raise ValueError("Diarization is not enabled") @@ -717,9 +888,9 @@ class SpeechmaticsSTTService(STTService): def attr_from_segment(segment: dict[str, Any]) -> dict[str, Any]: # Formats the output text based on the speaker and defined formats from the config. text = ( - self._speaker_active_format + self._settings.speaker_active_format if segment.get("is_active", True) - else self._speaker_passive_format + else self._settings.speaker_passive_format ).format( **{ "speaker_id": segment.get("speaker_id", "UU"), diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 0f3ff0cb6..0907b4e26 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -95,7 +95,7 @@ class SpeechmaticsTTSService(TTSService): self._params = params or SpeechmaticsTTSService.InputParams() # Set voice from constructor parameter - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index b556bb23a..d4e5f4cb5 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -34,6 +34,7 @@ from pipecat.frames.frames import ( from pipecat.metrics.metrics import TTFBMetricsData from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import ServiceSettings, STTSettings from pipecat.services.stt_latency import DEFAULT_TTFS_P99 from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language @@ -101,7 +102,6 @@ class STTService(AIService): self._audio_passthrough = audio_passthrough self._init_sample_rate = sample_rate self._sample_rate = 0 - self._settings: Dict[str, Any] = {} self._tracing_enabled: bool = False self._muted: bool = False self._user_id: str = "" @@ -166,18 +166,36 @@ class STTService(AIService): async def set_model(self, model: str): """Set the speech recognition model. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + Args: model: The name of the model to use for speech recognition. """ - self.set_model_name(model) + logger.info(f"Switching STT model to: [{model}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(model=model)) + else: + self.set_model_name(model) async def set_language(self, language: Language): """Set the language for speech recognition. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + Args: language: The language to use for speech recognition. """ - pass + logger.info(f"Switching STT language to: [{language}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(language=language)) + else: + pass @abstractmethod async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: @@ -224,6 +242,23 @@ class STTService(AIService): else: logger.warning(f"Unknown setting for STT service: {key}") + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed STT settings update. + + Handles ``model`` (via parent). Does **not** call ``set_language`` + — concrete services should override this method and handle language + changes (including any reconnect logic) based on the returned + changed-field set. + + Args: + update: A typed STT settings delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + return changed + async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): """Process an audio frame for speech recognition. @@ -285,7 +320,16 @@ class STTService(AIService): await self._handle_vad_user_stopped_speaking(frame) await self.push_frame(frame, direction) elif isinstance(frame, STTUpdateSettingsFrame): - await self._update_settings(frame.settings) + # New path: typed settings update object. + if frame.update is not None: + await self._update_settings_from_typed(frame.update) + # Legacy path: plain dict, but service uses typed settings — convert. + elif isinstance(self._settings, ServiceSettings): + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings_from_typed(update) + # Legacy path: plain dict, service still uses dict-based settings. + else: + await self._update_settings(frame.settings) elif isinstance(frame, STTMuteFrame): self._muted = frame.mute logger.debug(f"STT service {'muted' if frame.mute else 'unmuted'}") diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 239d2398b..4196e7872 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -52,6 +52,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import ServiceSettings, TTSSettings, is_given from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -189,7 +190,6 @@ class TTSService(AIService): self._init_sample_rate = sample_rate self._sample_rate = 0 self._voice_id: str = "" - self._settings: Dict[str, Any] = {} self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() if text_aggregator: import warnings @@ -263,18 +263,40 @@ class TTSService(AIService): async def set_model(self, model: str): """Set the TTS model to use. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + Args: model: The name of the TTS model. """ - self.set_model_name(model) + logger.info(f"Switching TTS model to: [{model}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(model=model)) + else: + self.set_model_name(model) - def set_voice(self, voice: str): + async def set_voice(self, voice: str): """Set the voice for speech synthesis. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + + .. versionchanged:: 0.0.103 + Now ``async``. In ``__init__`` methods, set + ``self._voice_id`` directly instead of calling this method. + Args: voice: The voice identifier or name. """ - self._voice_id = voice + logger.info(f"Switching TTS voice to: [{voice}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(voice=voice)) + else: + self._voice_id = voice def create_context_id(self) -> str: """Generate a unique context ID for a TTS request. @@ -416,13 +438,42 @@ class TTSService(AIService): elif key == "model": self.set_model_name(value) elif key == "voice" or key == "voice_id": - self.set_voice(value) + self._voice_id = value elif key == "text_filter": for filter in self._text_filters: await filter.update_settings(value) else: logger.warning(f"Unknown setting for TTS service: {key}") + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed TTS settings update. + + Handles ``model`` (via parent) and syncs ``_voice_id`` when voice + changes. Translates language values before applying. Does **not** + call ``set_voice`` or ``set_model`` directly — concrete services + should override this method and handle reconnect logic based on the + returned changed-field set. + + Args: + update: A typed TTS settings delta. + + Returns: + Set of field names whose values actually changed. + """ + # Translate language *before* applying so the stored value is canonical + if is_given(update.language) and update.language is not None: + converted = self.language_to_service_language(update.language) + if converted is not None: + update.language = converted + + changed = await super()._update_settings_from_typed(update) + + # Keep _voice_id in sync for code that reads it directly + if "voice" in changed and isinstance(self._settings, TTSSettings): + self._voice_id = self._settings.voice + + return changed + async def say(self, text: str): """Immediately speak the provided text. @@ -504,7 +555,16 @@ class TTSService(AIService): await self.flush_audio() self._processing_text = processing_text elif isinstance(frame, TTSUpdateSettingsFrame): - await self._update_settings(frame.settings) + # New path: typed settings update object. + if frame.update is not None: + await self._update_settings_from_typed(frame.update) + # Legacy path: plain dict, but service uses typed settings — convert. + elif isinstance(self._settings, ServiceSettings): + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings_from_typed(update) + # Legacy path: plain dict, service still uses dict-based settings. + else: + await self._update_settings(frame.settings) elif isinstance(frame, BotStoppedSpeakingFrame): await self._maybe_resume_frame_processing() await self.push_frame(frame, direction) diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index d549b11e5..9f0658486 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -15,6 +15,7 @@ import asyncio import datetime import json import uuid +from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Union import aiohttp @@ -34,7 +35,6 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, - LLMUpdateSettingsFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, @@ -56,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.time import time_now_iso8601 try: @@ -66,6 +67,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class UltravoxRealtimeLLMSettings(LLMSettings): + """Settings for UltravoxRealtimeLLMService. + + Parameters: + output_medium: The output medium for the model ("voice" or "text"). + """ + + output_medium: str = field(default=NOT_GIVEN) + + class AgentInputParams(BaseModel): """Input parameters for Ultravox Realtime generation using a pre-defined Agent. @@ -163,6 +175,7 @@ class UltravoxRealtimeLLMService(LLMService): **kwargs: Additional arguments passed to parent LLMService. """ super().__init__(**kwargs) + self._settings = UltravoxRealtimeLLMSettings() self._params = params if one_shot_selected_tools: if not isinstance(self._params, OneShotInputParams): @@ -310,6 +323,12 @@ class UltravoxRealtimeLLMService(LLMService): await self.cancel_task(self._receive_task, timeout=1.0) self._receive_task = None + async def _update_settings_from_typed(self, update: UltravoxRealtimeLLMSettings): + changed = await super()._update_settings_from_typed(update) + if "output_medium" in changed: + await self._update_output_medium(self._settings.output_medium) + return changed + # # frame processing # StartFrame, StopFrame, CancelFrame implemented in base class @@ -331,9 +350,6 @@ class UltravoxRealtimeLLMService(LLMService): else LLMContext.from_openai_context(frame.context) ) await self._handle_context(context) - elif isinstance(frame, LLMUpdateSettingsFrame): - if "output_medium" in frame.settings: - await self._update_output_medium(frame.settings.get("output_medium")) elif isinstance(frame, InputTextRawFrame): await self._send_user_text(frame.text) await self.push_frame(frame, direction) diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index bc999dba4..2a02c6ce7 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -10,6 +10,7 @@ This module provides common functionality for services implementing the Whisper interface, including language mapping, metrics generation, and error handling. """ +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -17,6 +18,7 @@ from openai import AsyncOpenAI from openai.types.audio import Transcription from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import WHISPER_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -24,6 +26,22 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt +@dataclass +class BaseWhisperSTTSettings(STTSettings): + """Typed settings for Whisper API-based STT services. + + Parameters: + base_url: API base URL. + prompt: Optional text to guide the model's style or continue + a previous segment. + temperature: Sampling temperature between 0 and 1. + """ + + base_url: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + prompt: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + temperature: Optional[float] = field(default_factory=lambda: NOT_GIVEN) + + def language_to_whisper_language(language: Language) -> Optional[str]: """Maps pipecat Language enum to Whisper API language codes. @@ -143,26 +161,36 @@ class BaseWhisperSTTService(SegmentedSTTService): self._temperature = temperature self._include_prob_metrics = include_prob_metrics - self._settings = { - "base_url": base_url, - "language": self._language, - "prompt": self._prompt, - "temperature": self._temperature, - } + self._settings: BaseWhisperSTTSettings = BaseWhisperSTTSettings( + model=model, + language=self._language, + base_url=base_url, + prompt=self._prompt, + temperature=self._temperature, + ) def _create_client(self, api_key: Optional[str], base_url: Optional[str]): return AsyncOpenAI(api_key=api_key, base_url=base_url) - async def set_model(self, model: str): - """Set the model name for transcription. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, syncing instance variables. - Args: - model: The name of the model to use. + Keeps ``_language``, ``_prompt``, and ``_temperature`` in sync with + the typed settings fields. """ - self.set_model_name(model) + changed = await super()._update_settings_from_typed(update) + + if "language" in changed: + self._language = self.language_to_service_language(Language(self._settings.language)) + if "prompt" in changed: + self._prompt = self._settings.prompt + if "temperature" in changed: + self._temperature = self._settings.temperature + + return changed def can_generate_metrics(self) -> bool: - """Indicates whether this service can generate metrics. + """Whether this service can generate processing metrics. Returns: bool: True, as this service supports metric generation. @@ -180,15 +208,6 @@ class BaseWhisperSTTService(SegmentedSTTService): """ return language_to_whisper_language(language) - async def set_language(self, language: Language): - """Set the language for transcription. - - Args: - language: The Language enum value to use for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._language = self.language_to_service_language(language) - @traced_stt async def _handle_transcription( self, transcript: str, is_final: bool, language: Optional[Language] = None diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index f11978cc2..30451e6d0 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -11,6 +11,7 @@ supporting both Faster Whisper and MLX Whisper backends for efficient inference. """ import asyncio +from dataclasses import dataclass, field from enum import Enum from typing import AsyncGenerator, Optional @@ -19,6 +20,7 @@ from loguru import logger from typing_extensions import TYPE_CHECKING, override from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.time import time_now_iso8601 @@ -172,6 +174,36 @@ def language_to_whisper_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class WhisperSTTSettings(STTSettings): + """Typed settings for the local Whisper (Faster Whisper) STT service. + + Parameters: + device: Inference device ('cpu', 'cuda', or 'auto'). + compute_type: Compute type for inference ('default', 'int8', etc.). + no_speech_prob: Probability threshold for filtering non-speech segments. + """ + + device: str = field(default_factory=lambda: NOT_GIVEN) + compute_type: str = field(default_factory=lambda: NOT_GIVEN) + no_speech_prob: float = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class WhisperMLXSTTSettings(STTSettings): + """Typed settings for the MLX Whisper STT service. + + Parameters: + no_speech_prob: Probability threshold for filtering non-speech segments. + temperature: Sampling temperature (0.0-1.0). + engine: Whisper engine identifier. + """ + + no_speech_prob: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + engine: str = field(default_factory=lambda: NOT_GIVEN) + + class WhisperSTTService(SegmentedSTTService): """Class to transcribe audio with a locally-downloaded Whisper model. @@ -206,12 +238,13 @@ class WhisperSTTService(SegmentedSTTService): self._no_speech_prob = no_speech_prob self._model: Optional[WhisperModel] = None - self._settings = { - "language": language, - "device": self._device, - "compute_type": self._compute_type, - "no_speech_prob": self._no_speech_prob, - } + self._settings: WhisperSTTSettings = WhisperSTTSettings( + model=model if isinstance(model, str) else model.value, + language=language, + device=self._device, + compute_type=self._compute_type, + no_speech_prob=self._no_speech_prob, + ) self._load() @@ -234,15 +267,6 @@ class WhisperSTTService(SegmentedSTTService): """ return language_to_whisper_language(language) - async def set_language(self, language: Language): - """Set the language for transcription. - - Args: - language: The Language enum value to use for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - def _load(self): """Loads the Whisper model. @@ -293,7 +317,7 @@ class WhisperSTTService(SegmentedSTTService): # Divide by 32768 because we have signed 16-bit data. audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0 - whisper_lang = self.language_to_service_language(self._settings["language"]) + whisper_lang = self.language_to_service_language(self._settings.language) segments, _ = await asyncio.to_thread( self._model.transcribe, audio_float, language=whisper_lang ) @@ -305,13 +329,13 @@ class WhisperSTTService(SegmentedSTTService): await self.stop_processing_metrics() if text: - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, ) @@ -347,12 +371,13 @@ class WhisperSTTServiceMLX(WhisperSTTService): self._no_speech_prob = no_speech_prob self._temperature = temperature - self._settings = { - "language": language, - "no_speech_prob": self._no_speech_prob, - "temperature": self._temperature, - "engine": "mlx", - } + self._settings: WhisperMLXSTTSettings = WhisperMLXSTTSettings( + model=model if isinstance(model, str) else model.value, + language=language, + no_speech_prob=self._no_speech_prob, + temperature=self._temperature, + engine="mlx", + ) # No need to call _load() as MLX Whisper loads models on demand @@ -390,7 +415,7 @@ class WhisperSTTServiceMLX(WhisperSTTService): # Divide by 32768 because we have signed 16-bit data. audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0 - whisper_lang = self.language_to_service_language(self._settings["language"]) + whisper_lang = self.language_to_service_language(self._settings.language) chunk = await asyncio.to_thread( mlx_whisper.transcribe, audio_float, @@ -413,13 +438,13 @@ class WhisperSTTServiceMLX(WhisperSTTService): await self.stop_processing_metrics() if text: - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, ) except Exception as e: diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index bf4eb4f03..664d3d4be 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -10,7 +10,8 @@ This module provides integration with Coqui XTTS streaming server for text-to-speech synthesis using local Docker deployment. """ -from typing import Any, AsyncGenerator, Dict, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Dict, Optional import aiohttp from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -68,6 +70,17 @@ def language_to_xtts_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class XTTSTTSSettings(TTSSettings): + """Typed settings for XTTS TTS service. + + Parameters: + base_url: Base URL of the XTTS streaming server. + """ + + base_url: str = field(default_factory=lambda: NOT_GIVEN) + + class XTTSService(TTSService): """Coqui XTTS text-to-speech service. @@ -98,11 +111,12 @@ class XTTSService(TTSService): """ super().__init__(sample_rate=sample_rate, **kwargs) - self._settings = { - "language": self.language_to_service_language(language), - "base_url": base_url, - } - self.set_voice(voice_id) + self._settings: XTTSTTSSettings = XTTSTTSSettings( + voice=voice_id, + language=self.language_to_service_language(language), + base_url=base_url, + ) + self._voice_id = voice_id self._studio_speakers: Optional[Dict[str, Any]] = None self._aiohttp_session = aiohttp_session @@ -138,7 +152,7 @@ class XTTSService(TTSService): if self._studio_speakers: return - async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r: + async with self._aiohttp_session.get(self._settings.base_url + "/studio_speakers") as r: if r.status != 200: text = await r.text() await self.push_error( @@ -166,11 +180,11 @@ class XTTSService(TTSService): embeddings = self._studio_speakers[self._voice_id] - url = self._settings["base_url"] + "/tts_stream" + url = self._settings.base_url + "/tts_stream" payload = { "text": text.replace(".", "").replace("*", ""), - "language": self._settings["language"], + "language": self._settings.language, "speaker_embedding": embeddings["speaker_embedding"], "gpt_cond_latent": embeddings["gpt_cond_latent"], "add_wav_header": False, diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 000000000..62583b00b --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,308 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Tests for the typed settings infrastructure in pipecat.services.settings.""" + +import pytest + +from pipecat.services.settings import ( + NOT_GIVEN, + LLMSettings, + ServiceSettings, + STTSettings, + TTSSettings, + _NotGiven, + is_given, +) + +# --------------------------------------------------------------------------- +# NOT_GIVEN sentinel +# --------------------------------------------------------------------------- + + +class TestNotGiven: + def test_singleton(self): + """NOT_GIVEN is a singleton — every reference is the same object.""" + assert _NotGiven() is _NotGiven() + assert NOT_GIVEN is _NotGiven() + + def test_repr(self): + assert repr(NOT_GIVEN) == "NOT_GIVEN" + + def test_bool_is_false(self): + assert not NOT_GIVEN + assert bool(NOT_GIVEN) is False + + def test_is_given_with_not_given(self): + assert is_given(NOT_GIVEN) is False + + def test_is_given_with_none(self): + assert is_given(None) is True + + def test_is_given_with_values(self): + assert is_given(0) is True + assert is_given("") is True + assert is_given(False) is True + assert is_given(42) is True + assert is_given("hello") is True + + +# --------------------------------------------------------------------------- +# ServiceSettings base +# --------------------------------------------------------------------------- + + +class TestServiceSettings: + def test_default_fields_are_not_given(self): + s = ServiceSettings() + assert not is_given(s.model) + assert s.extra == {} + + def test_given_fields_empty_by_default(self): + s = ServiceSettings() + assert s.given_fields() == {} + + def test_given_fields_includes_set_values(self): + s = ServiceSettings(model="gpt-4o") + assert s.given_fields() == {"model": "gpt-4o"} + + def test_given_fields_includes_extra(self): + s = ServiceSettings(model="gpt-4o") + s.extra = {"custom_key": 42} + result = s.given_fields() + assert result == {"model": "gpt-4o", "custom_key": 42} + + def test_to_dict(self): + s = ServiceSettings(model="gpt-4o") + assert s.to_dict() == {"model": "gpt-4o"} + + def test_copy_is_deep(self): + s = ServiceSettings(model="gpt-4o") + s.extra = {"nested": {"a": 1}} + c = s.copy() + assert c.model == "gpt-4o" + assert c.extra == {"nested": {"a": 1}} + # Mutating the copy shouldn't affect the original + c.extra["nested"]["a"] = 999 + assert s.extra["nested"]["a"] == 1 + + +# --------------------------------------------------------------------------- +# apply_update +# --------------------------------------------------------------------------- + + +class TestApplyUpdate: + def test_apply_update_basic(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="bob") + changed = current.apply_update(delta) + assert changed == {"voice"} + assert current.voice == "bob" + assert current.language == "en" + + def test_apply_update_no_change(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="alice") + changed = current.apply_update(delta) + assert changed == set() + assert current.voice == "alice" + + def test_apply_update_not_given_skipped(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings() # all NOT_GIVEN + changed = current.apply_update(delta) + assert changed == set() + assert current.voice == "alice" + assert current.language == "en" + + def test_apply_update_multiple_fields(self): + current = LLMSettings(temperature=0.7, max_tokens=100) + delta = LLMSettings(temperature=0.9, max_tokens=200, top_p=0.95) + changed = current.apply_update(delta) + assert changed == {"temperature", "max_tokens", "top_p"} + assert current.temperature == 0.9 + assert current.max_tokens == 200 + assert current.top_p == 0.95 + + def test_apply_update_extra_merged(self): + current = TTSSettings(voice="alice") + current.extra = {"speed": 1.0, "stability": 0.5} + delta = TTSSettings() + delta.extra = {"speed": 1.2} + changed = current.apply_update(delta) + assert "speed" in changed + assert current.extra == {"speed": 1.2, "stability": 0.5} + + def test_apply_update_extra_no_change(self): + current = TTSSettings(voice="alice") + current.extra = {"speed": 1.0} + delta = TTSSettings() + delta.extra = {"speed": 1.0} + changed = current.apply_update(delta) + assert changed == set() + + def test_apply_update_model_field(self): + current = ServiceSettings(model="old-model") + delta = ServiceSettings(model="new-model") + changed = current.apply_update(delta) + assert changed == {"model"} + assert current.model == "new-model" + + def test_apply_update_none_is_a_valid_value(self): + """Setting a field to None should be treated as a change from NOT_GIVEN.""" + current = TTSSettings() + delta = TTSSettings(language=None) + changed = current.apply_update(delta) + assert "language" in changed + assert current.language is None + + def test_apply_update_none_to_value(self): + current = TTSSettings(language=None) + delta = TTSSettings(language="en") + changed = current.apply_update(delta) + assert "language" in changed + assert current.language == "en" + + +# --------------------------------------------------------------------------- +# from_mapping +# --------------------------------------------------------------------------- + + +class TestFromMapping: + def test_basic_mapping(self): + s = TTSSettings.from_mapping({"voice": "alice", "language": "en"}) + assert s.voice == "alice" + assert s.language == "en" + assert not is_given(s.model) + + def test_alias_resolution(self): + """'voice_id' is an alias for 'voice' in TTSSettings.""" + s = TTSSettings.from_mapping({"voice_id": "alice"}) + assert s.voice == "alice" + + def test_unknown_keys_go_to_extra(self): + s = TTSSettings.from_mapping({"voice": "alice", "speed": 1.2, "stability": 0.5}) + assert s.voice == "alice" + assert s.extra == {"speed": 1.2, "stability": 0.5} + + def test_model_field(self): + s = LLMSettings.from_mapping({"model": "gpt-4o", "temperature": 0.7}) + assert s.model == "gpt-4o" + assert s.temperature == 0.7 + + def test_empty_mapping(self): + s = ServiceSettings.from_mapping({}) + assert s.given_fields() == {} + + def test_all_unknown_keys(self): + s = ServiceSettings.from_mapping({"foo": 1, "bar": 2}) + assert not is_given(s.model) + assert s.extra == {"foo": 1, "bar": 2} + + def test_llm_settings_from_mapping(self): + s = LLMSettings.from_mapping({"temperature": 0.5, "max_tokens": 1000, "custom_param": True}) + assert s.temperature == 0.5 + assert s.max_tokens == 1000 + assert s.extra == {"custom_param": True} + + def test_stt_settings_from_mapping(self): + s = STTSettings.from_mapping({"language": "fr", "model": "whisper-large"}) + assert s.language == "fr" + assert s.model == "whisper-large" + + +# --------------------------------------------------------------------------- +# LLMSettings specifics +# --------------------------------------------------------------------------- + + +class TestLLMSettings: + def test_all_fields_not_given_by_default(self): + s = LLMSettings() + for name in ( + "model", + "temperature", + "max_tokens", + "top_p", + "top_k", + "frequency_penalty", + "presence_penalty", + "seed", + ): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_given_fields(self): + s = LLMSettings(temperature=0.7, seed=42) + assert s.given_fields() == {"temperature": 0.7, "seed": 42} + + +# --------------------------------------------------------------------------- +# TTSSettings specifics +# --------------------------------------------------------------------------- + + +class TestTTSSettings: + def test_all_fields_not_given_by_default(self): + s = TTSSettings() + for name in ("model", "voice", "language"): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_aliases_class_var(self): + assert TTSSettings._aliases == {"voice_id": "voice"} + + def test_given_fields(self): + s = TTSSettings(voice="alice") + assert s.given_fields() == {"voice": "alice"} + + +# --------------------------------------------------------------------------- +# STTSettings specifics +# --------------------------------------------------------------------------- + + +class TestSTTSettings: + def test_all_fields_not_given_by_default(self): + s = STTSettings() + for name in ("model", "language"): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_given_fields(self): + s = STTSettings(language="en", model="whisper-large") + assert s.given_fields() == {"language": "en", "model": "whisper-large"} + + +# --------------------------------------------------------------------------- +# Integration: roundtrip from_mapping → apply_update +# --------------------------------------------------------------------------- + + +class TestRoundtrip: + def test_from_mapping_then_apply_update(self): + """Simulate the real flow: dict arrives via frame, gets converted, applied.""" + # Simulating current service state + current = TTSSettings(model="eleven_turbo_v2_5", voice="alice", language="en") + current.extra = {"stability": 0.5, "speed": 1.0} + + # Incoming dict-based update + raw = {"voice_id": "bob", "speed": 1.2} + delta = TTSSettings.from_mapping(raw) + + changed = current.apply_update(delta) + assert changed == {"voice", "speed"} + assert current.voice == "bob" + assert current.language == "en" + assert current.extra["speed"] == 1.2 + assert current.extra["stability"] == 0.5 + + def test_from_mapping_preserves_model(self): + current = LLMSettings(model="gpt-4o", temperature=0.7) + delta = LLMSettings.from_mapping({"model": "gpt-4o-mini", "temperature": 0.9}) + changed = current.apply_update(delta) + assert changed == {"model", "temperature"} + assert current.model == "gpt-4o-mini" + assert current.temperature == 0.9