diff --git a/.claude/skills/cleanup/SKILL.md b/.claude/skills/cleanup/SKILL.md index f7dd6ea98..c0f4945b7 100644 --- a/.claude/skills/cleanup/SKILL.md +++ b/.claude/skills/cleanup/SKILL.md @@ -293,7 +293,7 @@ class NewTTSService(TTSService): """ super().__init__(**kwargs) self._api_key = api_key - self.set_voice(voice) + self._voice_id = voice ``` --- diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 4b269ac3e..cacc04459 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # First flush any existing audio to finish the current context await tts.flush_audio() # Then set the new voice - tts.set_voice(VOICE_IDS[voice_name]) + await tts.set_voice(VOICE_IDS[voice_name]) logger.info(f"Switched to {voice_name} voice") else: logger.warning(f"Unknown voice: {voice_name}") diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 5634d79ee..dd12929b9 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -42,6 +42,7 @@ from pipecat.utils.utils import obj_count, obj_id if TYPE_CHECKING: from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven from pipecat.processors.frame_processor import FrameProcessor + from pipecat.services.settings import ServiceSettings class DeprecatedKeypadEntry: @@ -2112,13 +2113,17 @@ class TTSStoppedFrame(ControlFrame): class ServiceUpdateSettingsFrame(ControlFrame): """Base frame for updating service settings. - A control frame containing a request to update service settings. + Supports both the legacy ``settings`` dict and the new typed ``update`` + object. When both are provided, ``update`` takes precedence. Parameters: - settings: Dictionary of setting name to value mappings. + settings: Dictionary of setting name to value mappings (legacy). + update: Typed :class:`~pipecat.services.settings.ServiceSettings` + object describing the delta to apply. """ - settings: Mapping[str, Any] + settings: Mapping[str, Any] = field(default_factory=dict) + update: Optional["ServiceSettings"] = None @dataclass diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index c03ab9d0e..97b7b6443 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -10,7 +10,7 @@ Provides the foundation for all AI services in the Pipecat framework, including model management, settings handling, and frame processing lifecycle methods. """ -from typing import Any, AsyncGenerator, Dict, Mapping +from typing import Any, AsyncGenerator, Dict, Mapping, Set from loguru import logger @@ -23,6 +23,7 @@ from pipecat.frames.frames import ( ) from pipecat.metrics.metrics import MetricsData from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.settings import ServiceSettings class AIService(FrameProcessor): @@ -42,7 +43,7 @@ class AIService(FrameProcessor): """ super().__init__(**kwargs) self._model_name: str = "" - self._settings: Dict[str, Any] = {} + self._settings: Dict[str, Any] | ServiceSettings = {} self._session_properties: Dict[str, Any] = {} @property @@ -135,6 +136,42 @@ class AIService(FrameProcessor): else: logger.warning(f"Unknown setting for {self.name} service: {key}") + async def _update_settings_from_typed(self, update: ServiceSettings) -> Set[str]: + """Apply a typed settings update and return the set of changed field names. + + If ``_settings`` is a :class:`ServiceSettings` object, the update is + applied to it and the changed-field set is returned. The ``model`` + field is handled specially: when it changes, ``set_model_name`` is + called. + + Services that have been migrated to typed settings should override + this method (calling ``super()``) to react to specific changed fields + (e.g. reconnect on voice change). + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + if not isinstance(self._settings, ServiceSettings): + logger.warning( + f"{self.name}: received typed settings update but _settings " + f"is not a ServiceSettings — falling back to dict-based update" + ) + await self._update_settings(update.to_dict()) + return set() + + changed = self._settings.apply_update(update) + + if "model" in changed: + self.set_model_name(self._settings.model) + + if changed: + logger.info(f"{self.name}: updated settings fields: {changed}") + + return changed + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle service lifecycle. diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index a21296fe3..36ee104f5 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -16,8 +16,8 @@ import copy import io import json import re -from dataclasses import dataclass -from typing import Any, Dict, List, Literal, Optional, Union +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Literal, Optional, Union import httpx from loguru import logger @@ -42,7 +42,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -59,6 +58,8 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN +from pipecat.services.settings import LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm try: @@ -69,6 +70,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AnthropicLLMSettings(LLMSettings): + """Typed settings for Anthropic LLM services. + + Parameters: + enable_prompt_caching: Whether to enable prompt caching. + thinking: Extended thinking configuration. + """ + + enable_prompt_caching: Any = field(default_factory=lambda: _NOT_GIVEN) + thinking: Any = field(default_factory=lambda: _NOT_GIVEN) + + @dataclass class AnthropicContextAggregatorPair: """Pair of context aggregators for Anthropic conversations. @@ -210,9 +224,10 @@ class AnthropicLLMService(LLMService): self.set_model_name(model) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout - self._settings = { - "max_tokens": params.max_tokens, - "enable_prompt_caching": ( + self._settings = AnthropicLLMSettings( + model=model, + max_tokens=params.max_tokens, + enable_prompt_caching=( params.enable_prompt_caching if params.enable_prompt_caching is not None else ( @@ -221,12 +236,12 @@ class AnthropicLLMService(LLMService): else False ) ), - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "thinking": params.thinking, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + thinking=params.thinking, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate usage metrics. @@ -280,7 +295,7 @@ class AnthropicLLMService(LLMService): if isinstance(context, LLMContext): adapter: AnthropicLLMAdapter = self.get_llm_adapter() invocation_params = adapter.get_llm_invocation_params( - context, enable_prompt_caching=self._settings["enable_prompt_caching"] + context, enable_prompt_caching=self._settings.enable_prompt_caching ) messages = invocation_params["messages"] system = invocation_params["system"] @@ -294,20 +309,20 @@ class AnthropicLLMService(LLMService): # Build params using the same method as streaming completions params = { "model": self.model_name, - "max_tokens": max_tokens if max_tokens is not None else self._settings["max_tokens"], + "max_tokens": max_tokens if max_tokens is not None else self._settings.max_tokens, "stream": False, - "temperature": self._settings["temperature"], - "top_k": self._settings["top_k"], - "top_p": self._settings["top_p"], + "temperature": self._settings.temperature, + "top_k": self._settings.top_k, + "top_p": self._settings.top_p, "messages": messages, "system": system, "tools": tools, "betas": ["interleaved-thinking-2025-05-14"], } - if self._settings["thinking"]: - params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True) + if self._settings.thinking: + params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True) - params.update(self._settings["extra"]) + params.update(self._settings.extra) # LLM completion response = await self._client.beta.messages.create(**params) @@ -358,14 +373,14 @@ class AnthropicLLMService(LLMService): if isinstance(context, LLMContext): adapter: AnthropicLLMAdapter = self.get_llm_adapter() params = adapter.get_llm_invocation_params( - context, enable_prompt_caching=self._settings["enable_prompt_caching"] + context, enable_prompt_caching=self._settings.enable_prompt_caching ) return params # Anthropic-specific context messages = ( context.get_messages_with_cache_control_markers() - if self._settings["enable_prompt_caching"] + if self._settings.enable_prompt_caching else context.messages ) return AnthropicLLMInvocationParams( @@ -408,21 +423,21 @@ class AnthropicLLMService(LLMService): params = { "model": self.model_name, - "max_tokens": self._settings["max_tokens"], + "max_tokens": self._settings.max_tokens, "stream": True, - "temperature": self._settings["temperature"], - "top_k": self._settings["top_k"], - "top_p": self._settings["top_p"], + "temperature": self._settings.temperature, + "top_k": self._settings.top_k, + "top_p": self._settings.top_p, } # Add thinking parameter if set - if self._settings["thinking"]: - params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True) + if self._settings.thinking: + params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True) # Messages, system, tools params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) # "Interleaved thinking" needed to allow thinking between sequences # of function calls, when extended thinking is enabled. @@ -576,11 +591,9 @@ class AnthropicLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = AnthropicLLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) elif isinstance(frame, LLMEnablePromptCachingFrame): logger.debug(f"Setting enable prompt caching to: [{frame.enable}]") - self._settings["enable_prompt_caching"] = frame.enable + self._settings.enable_prompt_caching = frame.enable else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 41a0ae2a0..278873fdf 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription. import asyncio import json +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, Optional from urllib.parse import urlencode @@ -29,6 +30,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import ASSEMBLYAI_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -52,6 +54,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AssemblyAISTTSettings(STTSettings): + """Typed settings for the AssemblyAI STT service. + + See :class:`AssemblyAIConnectionParams` for detailed parameter descriptions. + + Parameters: + connection_params: Connection configuration parameters. + """ + + connection_params: AssemblyAIConnectionParams = field(default_factory=lambda: NOT_GIVEN) + + class AssemblyAISTTService(WebsocketSTTService): """AssemblyAI real-time speech-to-text service. @@ -96,9 +111,11 @@ class AssemblyAISTTService(WebsocketSTTService): ) self._api_key = api_key - self._language = language + self._settings: AssemblyAISTTSettings = AssemblyAISTTSettings( + language=language, + connection_params=connection_params, + ) self._api_endpoint_base_url = api_endpoint_base_url - self._connection_params = connection_params self._vad_force_turn_endpoint = vad_force_turn_endpoint self._termination_event = asyncio.Event() @@ -165,6 +182,35 @@ class AssemblyAISTTService(WebsocketSTTService): """ return True + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and reconnect if anything changed. + + Any change triggers a WebSocket reconnect since all connection + parameters are encoded in the WebSocket URL. + + Args: + update: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # Re-apply manual turn mode config if vad_force_turn_endpoint is active + # and connection_params were updated. + if self._vad_force_turn_endpoint and "connection_params" in changed: + self._settings.connection_params = self._configure_manual_turn_mode( + self._settings.connection_params + ) + + await self._disconnect() + await self._connect() + + return changed + async def start(self, frame: StartFrame): """Start the speech-to-text service. @@ -239,7 +285,7 @@ class AssemblyAISTTService(WebsocketSTTService): def _build_ws_url(self) -> str: """Build WebSocket URL with query parameters using urllib.parse.urlencode.""" params = {} - for k, v in self._connection_params.model_dump().items(): + for k, v in self._settings.connection_params.model_dump().items(): if v is not None: if k == "keyterms_prompt": params[k] = json.dumps(v) @@ -415,18 +461,18 @@ class AssemblyAISTTService(WebsocketSTTService): if not message.transcript: return if message.end_of_turn and ( - not self._connection_params.formatted_finals or message.turn_is_formatted + not self._settings.connection_params.formatted_finals or message.turn_is_formatted ): await self.push_frame( TranscriptionFrame( message.transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, message, ) ) - await self._trace_transcription(message.transcript, True, self._language) + await self._trace_transcription(message.transcript, True, self._settings.language) await self.stop_processing_metrics() else: await self.push_frame( @@ -434,7 +480,7 @@ class AssemblyAISTTService(WebsocketSTTService): message.transcript, self._user_id, time_now_iso8601(), - self._language, + self._settings.language, message, ) ) diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index 4ff6c928d..aecf69a26 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -9,6 +9,7 @@ import asyncio import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import AudioContextTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -72,6 +74,21 @@ def language_to_async_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class AsyncAITTSSettings(TTSSettings): + """Typed settings for Async AI TTS services. + + Parameters: + output_container: Audio container format (e.g. "raw"). + output_encoding: Audio encoding format (e.g. "pcm_s16le"). + output_sample_rate: Audio sample rate in Hz. + """ + + output_container: str = field(default_factory=lambda: NOT_GIVEN) + output_encoding: str = field(default_factory=lambda: NOT_GIVEN) + output_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class AsyncAITTSService(AudioContextTTSService): """Async TTS service with WebSocket streaming. @@ -131,19 +148,21 @@ class AsyncAITTSService(AudioContextTTSService): self._api_key = api_key self._api_version = version self._url = url - self._settings = { - "output_format": { + self._settings: AsyncAITTSSettings = AsyncAITTSSettings( + model=model, + voice=voice_id, + output_format={ "container": container, "encoding": encoding, "sample_rate": 0, }, - "language": self.language_to_service_language(params.language) + language=self.language_to_service_language(params.language) if params.language else None, - } + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id self._receive_task = None self._keepalive_task = None @@ -179,7 +198,7 @@ class AsyncAITTSService(AudioContextTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -235,8 +254,12 @@ class AsyncAITTSService(AudioContextTTSService): init_msg = { "model_id": self._model_name, "voice": {"mode": "id", "id": self._voice_id}, - "output_format": self._settings["output_format"], - "language": self._settings["language"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, + "language": self._settings.language, } await self._get_websocket().send(json.dumps(init_msg)) @@ -454,17 +477,17 @@ class AsyncAIHttpTTSService(TTSService): self._api_key = api_key self._base_url = url self._api_version = version - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings: AsyncAITTSSettings = AsyncAITTSSettings( + model=model, + voice=voice_id, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - } - self.set_voice(voice_id) + ) + self._voice_id = voice_id self.set_model_name(model) self._session = aiohttp_session @@ -495,7 +518,7 @@ class AsyncAIHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -517,8 +540,12 @@ class AsyncAIHttpTTSService(TTSService): "model_id": self._model_name, "transcript": text, "voice": voice_config, - "output_format": self._settings["output_format"], - "language": self._settings["language"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, + "language": self._settings.language, } yield TTSStartedFrame(context_id=context_id) headers = { diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 1778ae74e..032cee060 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -18,8 +18,8 @@ import io import json import os import re -from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Optional from loguru import logger from PIL import Image @@ -40,7 +40,6 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, - LLMUpdateSettingsFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -57,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm try: @@ -71,6 +71,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AWSBedrockLLMSettings(LLMSettings): + """Typed settings for AWS Bedrock LLM services. + + Parameters: + latency: Performance mode - "standard" or "optimized". + additional_model_request_fields: Additional model-specific parameters. + """ + + latency: Any = field(default_factory=lambda: NOT_GIVEN) + additional_model_request_fields: Any = field(default_factory=lambda: NOT_GIVEN) + + @dataclass class AWSBedrockContextAggregatorPair: """Container for AWS Bedrock context aggregators. @@ -806,15 +819,16 @@ class AWSBedrockLLMService(LLMService): self.set_model_name(model) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout - self._settings = { - "max_tokens": params.max_tokens, - "temperature": params.temperature, - "top_p": params.top_p, - "latency": params.latency, - "additional_model_request_fields": params.additional_model_request_fields + self._settings = AWSBedrockLLMSettings( + model=model, + max_tokens=params.max_tokens, + temperature=params.temperature, + top_p=params.top_p, + latency=params.latency, + additional_model_request_fields=params.additional_model_request_fields if isinstance(params.additional_model_request_fields, dict) else {}, - } + ) logger.info(f"Using AWS Bedrock model: {model}") @@ -836,12 +850,12 @@ class AWSBedrockLLMService(LLMService): Dictionary containing only the inference parameters that are not None. """ inference_config = {} - if self._settings["max_tokens"] is not None: - inference_config["maxTokens"] = self._settings["max_tokens"] - if self._settings["temperature"] is not None: - inference_config["temperature"] = self._settings["temperature"] - if self._settings["top_p"] is not None: - inference_config["topP"] = self._settings["top_p"] + if self._settings.max_tokens is not None: + inference_config["maxTokens"] = self._settings.max_tokens + if self._settings.temperature is not None: + inference_config["temperature"] = self._settings.temperature + if self._settings.top_p is not None: + inference_config["topP"] = self._settings.top_p return inference_config async def run_inference( @@ -879,7 +893,7 @@ class AWSBedrockLLMService(LLMService): request_params = { "modelId": self.model_name, "messages": messages, - "additionalModelRequestFields": self._settings["additional_model_request_fields"], + "additionalModelRequestFields": self._settings.additional_model_request_fields, } if inference_config: @@ -1036,7 +1050,7 @@ class AWSBedrockLLMService(LLMService): request_params = { "modelId": self.model_name, "messages": messages, - "additionalModelRequestFields": self._settings["additional_model_request_fields"], + "additionalModelRequestFields": self._settings.additional_model_request_fields, } # Only add inference config if it has parameters @@ -1081,8 +1095,8 @@ class AWSBedrockLLMService(LLMService): request_params["toolConfig"] = tool_config # Add performance config if latency is specified - if self._settings["latency"] in ["standard", "optimized"]: - request_params["performanceConfig"] = {"latency": self._settings["latency"]} + if self._settings.latency in ["standard", "optimized"]: + request_params["performanceConfig"] = {"latency": self._settings.latency} # Log request params with messages redacted for logging if isinstance(context, LLMContext): @@ -1207,8 +1221,6 @@ class AWSBedrockLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = AWSBedrockLLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index f78bc4d4b..cb52da12a 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -14,6 +14,7 @@ import json import os import random import string +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -28,6 +29,7 @@ from pipecat.frames.frames import ( TranscriptionFrame, ) from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import AWS_TRANSCRIBE_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -43,6 +45,25 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AWSTranscribeSTTSettings(STTSettings): + """Typed settings for the AWS Transcribe STT service. + + Parameters: + sample_rate: Audio sample rate in Hz (8000 or 16000). + media_encoding: Audio encoding format (e.g. "linear16"). + number_of_channels: Number of audio channels. + show_speaker_label: Whether to show speaker labels. + enable_channel_identification: Whether to enable channel identification. + """ + + sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + media_encoding: str = field(default_factory=lambda: NOT_GIVEN) + number_of_channels: int = field(default_factory=lambda: NOT_GIVEN) + show_speaker_label: bool = field(default_factory=lambda: NOT_GIVEN) + enable_channel_identification: bool = field(default_factory=lambda: NOT_GIVEN) + + class AWSTranscribeSTTService(WebsocketSTTService): """AWS Transcribe Speech-to-Text service using WebSocket streaming. @@ -78,21 +99,21 @@ class AWSTranscribeSTTService(WebsocketSTTService): """ super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs) - self._settings = { - "sample_rate": sample_rate, - "language": language, - "media_encoding": "linear16", # AWS expects raw PCM - "number_of_channels": 1, - "show_speaker_label": False, - "enable_channel_identification": False, - } + self._settings: AWSTranscribeSTTSettings = AWSTranscribeSTTSettings( + language=language, + sample_rate=sample_rate, + media_encoding="linear16", + number_of_channels=1, + show_speaker_label=False, + enable_channel_identification=False, + ) # Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz if sample_rate not in [8000, 16000]: logger.warning( f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz." ) - self._settings["sample_rate"] = 16000 + self._settings.sample_rate = 16000 self._credentials = { "aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"), @@ -117,6 +138,20 @@ class AWSTranscribeSTTService(WebsocketSTTService): } return encoding_map.get(encoding, encoding) + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, reconnecting if needed. + + Any change to connection-relevant settings (model, language, etc.) + triggers a WebSocket reconnect so the new configuration takes effect. + """ + changed = await super()._update_settings_from_typed(update) + + if changed and self._websocket: + await self._disconnect() + await self._connect() + + return changed + async def start(self, frame: StartFrame): """Initialize the connection when the service starts. @@ -208,9 +243,9 @@ class AWSTranscribeSTTService(WebsocketSTTService): logger.debug("Connecting to AWS Transcribe WebSocket") - language_code = self.language_to_service_language(Language(self._settings["language"])) + language_code = self.language_to_service_language(Language(self._settings.language)) if not language_code: - raise ValueError(f"Unsupported language: {self._settings['language']}") + raise ValueError(f"Unsupported language: {self._settings.language}") # Generate random websocket key websocket_key = "".join( @@ -237,14 +272,14 @@ class AWSTranscribeSTTService(WebsocketSTTService): }, language_code=language_code, media_encoding=self.get_service_encoding( - self._settings["media_encoding"] + self._settings.media_encoding ), # Convert to AWS format - sample_rate=self._settings["sample_rate"], - number_of_channels=self._settings["number_of_channels"], + sample_rate=self._settings.sample_rate, + number_of_channels=self._settings.number_of_channels, enable_partial_results_stabilization=True, partial_results_stability="high", - show_speaker_label=self._settings["show_speaker_label"], - enable_channel_identification=self._settings["enable_channel_identification"], + show_speaker_label=self._settings.show_speaker_label, + enable_channel_identification=self._settings.enable_channel_identification, ) logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...") @@ -479,14 +514,14 @@ class AWSTranscribeSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, result=result, ) ) await self._handle_transcription( transcript, is_final, - self._settings["language"], + self._settings.language, ) await self.stop_processing_metrics() else: @@ -495,7 +530,7 @@ class AWSTranscribeSTTService(WebsocketSTTService): transcript, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, result=result, ) ) diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index b902564d2..5086b1469 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -11,6 +11,7 @@ supporting multiple languages, voices, and SSML features. """ import os +from dataclasses import dataclass, field from typing import AsyncGenerator, List, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -121,6 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class AWSPollyTTSSettings(TTSSettings): + """Typed settings for AWS Polly TTS service. + + Parameters: + engine: TTS engine to use ('standard', 'neural', etc.). + pitch: Voice pitch adjustment (for standard engine only). + rate: Speech rate adjustment. + volume: Voice volume adjustment. + lexicon_names: List of pronunciation lexicons to apply. + """ + + engine: str = field(default_factory=lambda: NOT_GIVEN) + pitch: str = field(default_factory=lambda: NOT_GIVEN) + rate: str = field(default_factory=lambda: NOT_GIVEN) + volume: str = field(default_factory=lambda: NOT_GIVEN) + lexicon_names: List[str] = field(default_factory=lambda: NOT_GIVEN) + + class AWSPollyTTSService(TTSService): """AWS Polly text-to-speech service. @@ -185,20 +206,21 @@ class AWSPollyTTSService(TTSService): } self._aws_session = aioboto3.Session() - self._settings = { - "engine": params.engine, - "language": self.language_to_service_language(params.language) + self._settings: AWSPollyTTSSettings = AWSPollyTTSSettings( + voice=voice_id, + engine=params.engine, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "pitch": params.pitch, - "rate": params.rate, - "volume": params.volume, - "lexicon_names": params.lexicon_names, - } + pitch=params.pitch, + rate=params.rate, + volume=params.volume, + lexicon_names=params.lexicon_names, + ) self._resampler = create_stream_resampler() - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -222,19 +244,19 @@ class AWSPollyTTSService(TTSService): def _construct_ssml(self, text: str) -> str: ssml = "" - language = self._settings["language"] + language = self._settings.language ssml += f"" prosody_attrs = [] # Prosody tags are only supported for standard and neural engines - if self._settings["engine"] == "standard": - if self._settings["pitch"]: - prosody_attrs.append(f"pitch='{self._settings['pitch']}'") + if self._settings.engine == "standard": + if self._settings.pitch: + prosody_attrs.append(f"pitch='{self._settings.pitch}'") - if self._settings["rate"]: - prosody_attrs.append(f"rate='{self._settings['rate']}'") - if self._settings["volume"]: - prosody_attrs.append(f"volume='{self._settings['volume']}'") + if self._settings.rate: + prosody_attrs.append(f"rate='{self._settings.rate}'") + if self._settings.volume: + prosody_attrs.append(f"volume='{self._settings.volume}'") if prosody_attrs: ssml += f"" @@ -276,10 +298,10 @@ class AWSPollyTTSService(TTSService): "TextType": "ssml", "OutputFormat": "pcm", "VoiceId": self._voice_id, - "Engine": self._settings["engine"], + "Engine": self._settings.engine, # AWS only supports 8000 and 16000 for PCM. We select 16000. "SampleRate": "16000", - "LexiconNames": self._settings["lexicon_names"], + "LexiconNames": self._settings.lexicon_names, } # Filter out None values diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index 1bc7ec70a..bf3f70653 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -11,6 +11,7 @@ Speech SDK for real-time audio transcription. """ import asyncio +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( TranscriptionFrame, ) from pipecat.services.azure.common import language_to_azure_language +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import AZURE_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -48,6 +50,19 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class AzureSTTSettings(STTSettings): + """Typed settings for the Azure STT service. + + Parameters: + region: Azure region for the Speech service. + sample_rate: Audio sample rate in Hz. + """ + + region: str = field(default_factory=lambda: NOT_GIVEN) + sample_rate: Optional[int] = field(default_factory=lambda: NOT_GIVEN) + + class AzureSTTService(STTService): """Azure Speech-to-Text service for real-time audio transcription. @@ -92,11 +107,11 @@ class AzureSTTService(STTService): self._audio_stream = None self._speech_recognizer = None - self._settings = { - "region": region, - "language": language_to_azure_language(language), - "sample_rate": sample_rate, - } + self._settings: AzureSTTSettings = AzureSTTSettings( + region=region, + language=language_to_azure_language(language), + sample_rate=sample_rate, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate performance metrics. @@ -106,6 +121,29 @@ class AzureSTTService(STTService): """ return True + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, reconfiguring the recognizer if needed. + + When ``language`` changes the ``SpeechConfig`` is updated and the + speech recognizer is restarted so that the new language takes effect. + """ + changed = await super()._update_settings_from_typed(update) + + if "language" in changed: + # Convert Language enum to Azure language code if needed. + lang = self._settings.language + if isinstance(lang, Language): + lang = language_to_azure_language(lang) + self._settings.language = lang + self._speech_config.speech_recognition_language = lang + + # Restart the recognizer with the new config. + if self._speech_recognizer: + self._speech_recognizer.stop_continuous_recognition_async() + self._speech_recognizer.start_continuous_recognition_async() + + return changed + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process audio data for speech-to-text conversion. @@ -198,7 +236,7 @@ class AzureSTTService(STTService): def _on_handle_recognized(self, event): if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or self._settings.get("language") + language = getattr(event.result, "language", None) or self._settings.language frame = TranscriptionFrame( event.result.text, self._user_id, @@ -213,7 +251,7 @@ class AzureSTTService(STTService): def _on_handle_recognizing(self, event): if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or self._settings.get("language") + language = getattr(event.result, "language", None) or self._settings.language frame = InterimTranscriptionFrame( event.result.text, self._user_id, diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 7d4aa0253..04b51d10b 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -7,6 +7,7 @@ """Azure Cognitive Services Text-to-Speech service implementations.""" import asyncio +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.azure.common import language_to_azure_language +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService, WordTTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -65,6 +67,31 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma return sample_rate_map.get(sample_rate, SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm) +@dataclass +class AzureTTSSettings(TTSSettings): + """Typed settings for Azure TTS services. + + Parameters: + emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). + language: Language for synthesis. Defaults to English (US). + pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high"). + rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast"). + role: Voice role for expression (e.g., "YoungAdultFemale"). + style: Speaking style (e.g., "cheerful", "sad", "excited"). + style_degree: Intensity of the speaking style (0.01 to 2.0). + volume: Volume level (e.g., "+20%", "loud", "x-soft"). + """ + + emphasis: str = field(default_factory=lambda: NOT_GIVEN) + language: str = field(default_factory=lambda: NOT_GIVEN) + pitch: str = field(default_factory=lambda: NOT_GIVEN) + rate: str = field(default_factory=lambda: NOT_GIVEN) + role: str = field(default_factory=lambda: NOT_GIVEN) + style: str = field(default_factory=lambda: NOT_GIVEN) + style_degree: str = field(default_factory=lambda: NOT_GIVEN) + volume: str = field(default_factory=lambda: NOT_GIVEN) + + class AzureBaseTTSService: """Base mixin class for Azure Cognitive Services text-to-speech implementations. @@ -126,18 +153,18 @@ class AzureBaseTTSService: """ params = params or AzureBaseTTSService.InputParams() - self._settings = { - "emphasis": params.emphasis, - "language": self.language_to_service_language(params.language) + self._settings: AzureTTSSettings = AzureTTSSettings( + emphasis=params.emphasis, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "pitch": params.pitch, - "rate": params.rate, - "role": params.role, - "style": params.style, - "style_degree": params.style_degree, - "volume": params.volume, - } + pitch=params.pitch, + rate=params.rate, + role=params.role, + style=params.style, + style_degree=params.style_degree, + volume=params.volume, + ) self._api_key = api_key self._region = region @@ -156,7 +183,7 @@ class AzureBaseTTSService: return language_to_azure_language(language) def _construct_ssml(self, text: str) -> str: - language = self._settings["language"] + language = self._settings.language # Escape special characters escaped_text = self._escape_text(text) @@ -169,38 +196,38 @@ class AzureBaseTTSService: "" ) - if self._settings["style"]: - ssml += f"" - if self._settings["emphasis"]: - ssml += f"" + if self._settings.emphasis: + ssml += f"" ssml += escaped_text - if self._settings["emphasis"]: + if self._settings.emphasis: ssml += "" if prosody_attrs: ssml += "" - if self._settings["style"]: + if self._settings.style: ssml += "" ssml += "" @@ -314,7 +341,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): subscription=self._api_key, region=self._region, ) - self._speech_config.speech_synthesis_language = self._settings["language"] + self._speech_config.speech_synthesis_language = self._settings.language self._speech_config.set_speech_synthesis_output_format( sample_rate_to_output_format(self.sample_rate) ) @@ -364,7 +391,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): Returns: True if the language is CJK, False otherwise. """ - language = self._settings.get("language", "").lower() + language = (self._settings.language if self._settings.language else "").lower() # Check if language starts with CJK language codes return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu")) @@ -735,7 +762,7 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService): subscription=self._api_key, region=self._region, ) - self._speech_config.speech_synthesis_language = self._settings["language"] + self._speech_config.speech_synthesis_language = self._settings.language self._speech_config.set_speech_synthesis_output_format( sample_rate_to_output_format(self.sample_rate) ) diff --git a/src/pipecat/services/camb/tts.py b/src/pipecat/services/camb/tts.py index def57d3a0..8a6f67231 100644 --- a/src/pipecat/services/camb/tts.py +++ b/src/pipecat/services/camb/tts.py @@ -16,7 +16,8 @@ Features: - Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz) """ -from typing import Any, AsyncGenerator, Dict, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Dict, Optional from camb import StreamTtsOutputConfiguration from camb.client import AsyncCambAI @@ -31,6 +32,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -133,6 +135,18 @@ def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]: return buffer[:aligned_size], buffer[aligned_size:] +@dataclass +class CambTTSSettings(TTSSettings): + """Typed settings for Camb.ai TTS service. + + Parameters: + user_instructions: Custom instructions for mars-instruct model only. + Ignored for other models. Max 1000 characters. + """ + + user_instructions: str = field(default_factory=lambda: NOT_GIVEN) + + class CambTTSService(TTSService): """Camb.ai MARS text-to-speech service using the official SDK. @@ -212,15 +226,16 @@ class CambTTSService(TTSService): ) # Build settings - self._settings = { - "language": ( + self._settings: CambTTSSettings = CambTTSSettings( + model=model, + voice=voice_id, + language=( self.language_to_service_language(params.language) if params.language else "en-us" ), - "user_instructions": params.user_instructions, - } + user_instructions=params.user_instructions, + ) self.set_model_name(model) - self.set_voice(str(voice_id)) self._voice_id = voice_id self._client = None @@ -283,14 +298,14 @@ class CambTTSService(TTSService): tts_kwargs: Dict[str, Any] = { "text": text, "voice_id": self._voice_id, - "language": self._settings["language"], + "language": self._settings.language, "speech_model": self.model_name, "output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"), } # Add user instructions if using mars-instruct model - if self._model_name == "mars-instruct" and self._settings.get("user_instructions"): - tts_kwargs["user_instructions"] = self._settings["user_instructions"] + if self._model_name == "mars-instruct" and self._settings.user_instructions: + tts_kwargs["user_instructions"] = self._settings.user_instructions await self.start_tts_usage_metrics(text) yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index c4429226f..624801bfb 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -12,6 +12,7 @@ the Cartesia Live transcription API for real-time speech recognition. import json import urllib.parse +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import CARTESIA_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -42,6 +44,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class CartesiaSTTSettings(STTSettings): + """Typed settings for the Cartesia STT service. + + Parameters: + encoding: Audio encoding format (e.g. ``"pcm_s16le"``). + """ + + encoding: str = field(default_factory=lambda: NOT_GIVEN) + + class CartesiaLiveOptions: """Configuration options for Cartesia Live STT service. @@ -181,7 +194,11 @@ class CartesiaSTTService(WebsocketSTTService): k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None" } - self._settings = merged_options + self._settings: CartesiaSTTSettings = CartesiaSTTSettings( + model=merged_options["model"], + language=merged_options.get("language"), + encoding=merged_options.get("encoding", "pcm_s16le"), + ) self.set_model_name(merged_options["model"]) self._api_key = api_key self._base_url = base_url or "api.cartesia.ai" @@ -275,13 +292,33 @@ class CartesiaSTTService(WebsocketSTTService): await self._disconnect_websocket() + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and reconnect if anything changed. + + Args: + update: A :class:`STTSettings` (or ``CartesiaSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + if changed: + await self._disconnect() + await self._connect() + return changed + async def _connect_websocket(self): try: if self._websocket and self._websocket.state is State.OPEN: return logger.debug("Connecting to Cartesia STT") - params = self._settings + params = { + "model": self._settings.model, + "language": self._settings.language, + "encoding": self._settings.encoding, + "sample_rate": str(self.sample_rate), + } ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 791c60a18..531aafdf7 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -9,8 +9,9 @@ import base64 import json import warnings +from dataclasses import dataclass, field from enum import Enum -from typing import AsyncGenerator, List, Literal, Optional +from typing import Any, AsyncGenerator, List, Literal, Optional from loguru import logger from pydantic import BaseModel, Field @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -191,6 +193,31 @@ class CartesiaEmotion(str, Enum): DETERMINED = "determined" +@dataclass +class CartesiaTTSSettings(TTSSettings): + """Typed settings for Cartesia TTS services. + + Parameters: + output_container: Audio container format (e.g. "raw"). + output_encoding: Audio encoding format (e.g. "pcm_s16le"). + output_sample_rate: Audio sample rate in Hz. + speed: Voice speed control for non-Sonic-3 models (literal values). + emotion: List of emotion controls for non-Sonic-3 models. + generation_config: Generation configuration for Sonic-3 models. Includes volume, + speed (numeric), and emotion (string) parameters. + pronunciation_dict_id: The ID of the pronunciation dictionary to use for + custom pronunciations. + """ + + output_container: str = field(default_factory=lambda: NOT_GIVEN) + output_encoding: str = field(default_factory=lambda: NOT_GIVEN) + output_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + speed: str = field(default_factory=lambda: NOT_GIVEN) + emotion: List[str] = field(default_factory=lambda: NOT_GIVEN) + generation_config: GenerationConfig = field(default_factory=lambda: NOT_GIVEN) + pronunciation_dict_id: str = field(default_factory=lambda: NOT_GIVEN) + + class CartesiaTTSService(AudioContextWordTTSService): """Cartesia TTS service with WebSocket streaming and word timestamps. @@ -289,22 +316,20 @@ class CartesiaTTSService(AudioContextWordTTSService): self._api_key = api_key self._cartesia_version = cartesia_version self._url = url - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings: CartesiaTTSSettings = CartesiaTTSSettings( + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - "speed": params.speed, - "emotion": params.emotion, - "generation_config": params.generation_config, - "pronunciation_dict_id": params.pronunciation_dict_id, - } + speed=params.speed, + emotion=params.emotion, + generation_config=params.generation_config, + pronunciation_dict_id=params.pronunciation_dict_id, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id self._context_id = None self._receive_task = None @@ -317,16 +342,6 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model. - - Args: - model: The model name to use for synthesis. - """ - self._model_id = model - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - def language_to_service_language(self, language: Language) -> Optional[str]: """Convert a Language enum to Cartesia language format. @@ -391,7 +406,7 @@ class CartesiaTTSService(AudioContextWordTTSService): Returns: List of (word, start_time) tuples processed for the language. """ - current_language = self._settings.get("language") + current_language = self._settings.language # Check if this is a CJK language (if language is None, treat as non-CJK) if current_language and self._is_cjk_language(current_language): @@ -414,7 +429,7 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_config["mode"] = "id" voice_config["id"] = self._voice_id - if self._settings["emotion"]: + if is_given(self._settings.emotion) and self._settings.emotion: with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( @@ -423,8 +438,7 @@ class CartesiaTTSService(AudioContextWordTTSService): stacklevel=2, ) voice_config["__experimental_controls"] = {} - if self._settings["emotion"]: - voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"] + voice_config["__experimental_controls"]["emotion"] = self._settings.emotion msg = { "transcript": text, @@ -432,24 +446,28 @@ class CartesiaTTSService(AudioContextWordTTSService): "context_id": self._context_id, "model_id": self.model_name, "voice": voice_config, - "output_format": self._settings["output_format"], + "output_format": { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + }, "add_timestamps": add_timestamps, "use_original_timestamps": False if self.model_name == "sonic" else True, } - if self._settings["language"]: - msg["language"] = self._settings["language"] + if is_given(self._settings.language) and self._settings.language: + msg["language"] = self._settings.language - if self._settings["speed"]: - msg["speed"] = self._settings["speed"] + if is_given(self._settings.speed) and self._settings.speed: + msg["speed"] = self._settings.speed - if self._settings["generation_config"]: - msg["generation_config"] = self._settings["generation_config"].model_dump( + if is_given(self._settings.generation_config) and self._settings.generation_config: + msg["generation_config"] = self._settings.generation_config.model_dump( exclude_none=True ) - if self._settings["pronunciation_dict_id"]: - msg["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"] + if is_given(self._settings.pronunciation_dict_id) and self._settings.pronunciation_dict_id: + msg["pronunciation_dict_id"] = self._settings.pronunciation_dict_id return json.dumps(msg) @@ -460,7 +478,7 @@ class CartesiaTTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -694,21 +712,21 @@ class CartesiaHttpTTSService(TTSService): self._api_key = api_key self._base_url = base_url self._cartesia_version = cartesia_version - self._settings = { - "output_format": { - "container": container, - "encoding": encoding, - "sample_rate": 0, - }, - "language": self.language_to_service_language(params.language) + self._settings: CartesiaTTSSettings = CartesiaTTSSettings( + model=model, + voice=voice_id, + output_container=container, + output_encoding=encoding, + output_sample_rate=0, + language=self.language_to_service_language(params.language) if params.language else None, - "speed": params.speed, - "emotion": params.emotion, - "generation_config": params.generation_config, - "pronunciation_dict_id": params.pronunciation_dict_id, - } - self.set_voice(voice_id) + speed=params.speed, + emotion=params.emotion, + generation_config=params.generation_config, + pronunciation_dict_id=params.pronunciation_dict_id, + ) + self._voice_id = voice_id self.set_model_name(model) self._client = AsyncCartesia( @@ -742,7 +760,7 @@ class CartesiaHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["output_format"]["sample_rate"] = self.sample_rate + self._settings.output_sample_rate = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Cartesia HTTP TTS service. @@ -778,7 +796,7 @@ class CartesiaHttpTTSService(TTSService): try: voice_config = {"mode": "id", "id": self._voice_id} - if self._settings["emotion"]: + if is_given(self._settings.emotion) and self._settings.emotion: with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( @@ -786,30 +804,39 @@ class CartesiaHttpTTSService(TTSService): DeprecationWarning, stacklevel=2, ) - voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]} + voice_config["__experimental_controls"] = {"emotion": self._settings.emotion} await self.start_ttfb_metrics() + output_format = { + "container": self._settings.output_container, + "encoding": self._settings.output_encoding, + "sample_rate": self._settings.output_sample_rate, + } + payload = { "model_id": self._model_name, "transcript": text, "voice": voice_config, - "output_format": self._settings["output_format"], + "output_format": output_format, } - if self._settings["language"]: - payload["language"] = self._settings["language"] + if is_given(self._settings.language) and self._settings.language: + payload["language"] = self._settings.language - if self._settings["speed"]: - payload["speed"] = self._settings["speed"] + if is_given(self._settings.speed) and self._settings.speed: + payload["speed"] = self._settings.speed - if self._settings["generation_config"]: - payload["generation_config"] = self._settings["generation_config"].model_dump( + if is_given(self._settings.generation_config) and self._settings.generation_config: + payload["generation_config"] = self._settings.generation_config.model_dump( exclude_none=True ) - if self._settings["pronunciation_dict_id"]: - payload["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"] + if ( + is_given(self._settings.pronunciation_dict_id) + and self._settings.pronunciation_dict_id + ): + payload["pronunciation_dict_id"] = self._settings.pronunciation_dict_id yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/cerebras/llm.py b/src/pipecat/services/cerebras/llm.py index 54ea45ddb..01a8165f8 100644 --- a/src/pipecat/services/cerebras/llm.py +++ b/src/pipecat/services/cerebras/llm.py @@ -68,14 +68,14 @@ class CerebrasLLMService(OpenAILLMService): params = { "model": self.model_name, "stream": True, - "seed": self._settings["seed"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_completion_tokens": self._settings["max_completion_tokens"], + "seed": self._settings.seed, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_completion_tokens": self._settings.max_completion_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index 0f79499ba..91d4308cb 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -6,6 +6,7 @@ """Deepgram speech-to-text service implementation.""" +from dataclasses import dataclass, field from typing import AsyncGenerator, Dict, Optional from loguru import logger @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import DEEPGRAM_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -45,6 +47,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramSTTSettings(STTSettings): + """Typed settings for the Deepgram STT service. + + Parameters: + live_options: Deepgram ``LiveOptions`` for detailed configuration. + """ + + live_options: LiveOptions = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramSTTService(STTService): """Deepgram speech-to-text service. @@ -129,11 +142,17 @@ class DeepgramSTTService(STTService): merged_options["language"] = merged_options["language"].value self.set_model_name(merged_options["model"]) - self._settings = merged_options + merged_live_options = LiveOptions(**merged_options) + self._settings: DeepgramSTTSettings = DeepgramSTTSettings( + model=merged_options.get("model"), + language=merged_options.get("language"), + live_options=merged_live_options, + ) + self._addons = addons self._should_interrupt = should_interrupt - if merged_options.get("vad_events"): + if merged_live_options.vad_events: import warnings with warnings.catch_warnings(): @@ -164,7 +183,7 @@ class DeepgramSTTService(STTService): Returns: True if VAD events are enabled in the current settings. """ - return self._settings["vad_events"] + return self._settings.live_options.vad_events def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -174,28 +193,48 @@ class DeepgramSTTService(STTService): """ return True - async def set_model(self, model: str): - """Set the Deepgram model and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, keeping ``live_options`` in sync. - Args: - model: The Deepgram model name to use. + Top-level ``model`` and ``language`` are the source of truth. When + they are given in *update* their values are propagated into + ``live_options``. When only ``live_options`` is given, its ``model`` + and ``language`` are propagated *up* to the top-level fields. + + Any change triggers a WebSocket reconnect. """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._settings["model"] = model + # Determine which top-level fields are explicitly provided. + model_given = isinstance(update, DeepgramSTTSettings) and is_given( + getattr(update, "model", NOT_GIVEN) + ) + language_given = isinstance(update, DeepgramSTTSettings) and is_given( + getattr(update, "language", NOT_GIVEN) + ) + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # --- Sync model -------------------------------------------------- + if model_given: + # Top-level model wins → push into live_options. + self._settings.live_options.model = self._settings.model + elif "live_options" in changed and self._settings.live_options.model is not None: + # Only live_options was given → pull model up. + self._settings.model = self._settings.live_options.model + self.set_model_name(self._settings.model) + + # --- Sync language ----------------------------------------------- + if language_given: + self._settings.live_options.language = self._settings.language + elif "live_options" in changed and self._settings.live_options.language is not None: + self._settings.language = self._settings.live_options.language + await self._disconnect() await self._connect() - async def set_language(self, language: Language): - """Set the recognition language and reconnect. - - Args: - language: The language to use for speech recognition. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - await self._disconnect() - await self._connect() + return changed async def start(self, frame: StartFrame): """Start the Deepgram STT service. @@ -204,7 +243,7 @@ class DeepgramSTTService(STTService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.live_options.sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -257,7 +296,9 @@ class DeepgramSTTService(STTService): self._on_utterance_end, ) - if not await self._connection.start(options=self._settings, addons=self._addons): + if not await self._connection.start( + options=self._settings.live_options, addons=self._addons + ): await self.push_error(error_msg=f"Unable to connect to Deepgram") else: headers = { diff --git a/src/pipecat/services/deepgram/stt_sagemaker.py b/src/pipecat/services/deepgram/stt_sagemaker.py index 99f6cf487..95242ade6 100644 --- a/src/pipecat/services/deepgram/stt_sagemaker.py +++ b/src/pipecat/services/deepgram/stt_sagemaker.py @@ -14,6 +14,7 @@ languages, and various Deepgram features. import asyncio import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -31,6 +32,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import DEEPGRAM_SAGEMAKER_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language @@ -47,6 +49,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramSageMakerSTTSettings(STTSettings): + """Typed settings for the Deepgram SageMaker STT service. + + Parameters: + live_options: Deepgram ``LiveOptions`` for detailed configuration. + """ + + live_options: LiveOptions = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramSageMakerSTTService(STTService): """Deepgram speech-to-text service for AWS SageMaker. @@ -129,7 +142,12 @@ class DeepgramSageMakerSTTService(STTService): merged_options["language"] = merged_options["language"].value self.set_model_name(merged_options["model"]) - self._settings = merged_options + merged_live_options = LiveOptions(**merged_options) + self._settings: DeepgramSageMakerSTTSettings = DeepgramSageMakerSTTSettings( + model=merged_options.get("model"), + language=merged_options.get("language"), + live_options=merged_live_options, + ) self._client: Optional[SageMakerBidiClient] = None self._response_task: Optional[asyncio.Task] = None @@ -143,35 +161,40 @@ class DeepgramSageMakerSTTService(STTService): """ return True - async def set_model(self, model: str): - """Set the Deepgram model and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, keeping ``live_options`` in sync. - Disconnects from the current session, updates the model setting, and - establishes a new connection with the updated model. + Top-level ``model`` and ``language`` are the source of truth. When + they change their values are propagated into ``live_options``. - Args: - model: The Deepgram model name to use (e.g., "nova-3"). + Any change triggers a reconnect. """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._settings["model"] = model - await self._disconnect() - await self._connect() - - async def set_language(self, language: Language): - """Set the recognition language and reconnect. - - Disconnects from the current session, updates the language setting, and - establishes a new connection with the updated language. - - Args: - language: The language to use for speech recognition (e.g., Language.EN, - Language.ES). - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language + model_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given( + getattr(update, "model", NOT_GIVEN) + ) + language_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given( + getattr(update, "language", NOT_GIVEN) + ) + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # Sync model into live_options + if model_given and "model" in changed: + self._settings.live_options.model = self._settings.model + + # Sync language into live_options + if language_given and "language" in changed: + lang = self._settings.language + if isinstance(lang, Language): + lang = lang.value + self._settings.live_options.language = lang + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the Deepgram SageMaker STT service. @@ -180,7 +203,7 @@ class DeepgramSageMakerSTTService(STTService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.live_options.sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -226,12 +249,12 @@ class DeepgramSageMakerSTTService(STTService): """ logger.debug("Connecting to Deepgram on SageMaker...") - # Update sample rate in settings - self._settings["sample_rate"] = self.sample_rate + # Update sample rate in live_options + self._settings.live_options.sample_rate = self.sample_rate - # Build query string from settings, converting booleans to strings + # Build query string from live_options, converting booleans to strings query_params = {} - for key, value in self._settings.items(): + for key, value in self._settings.live_options.to_dict().items(): if value is not None: # Convert boolean values to lowercase strings for Deepgram API if isinstance(value, bool): diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index 12aba4905..4c698dcea 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -11,6 +11,7 @@ for generating speech from text using various voice models. """ import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -29,6 +30,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService, WebsocketTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -43,6 +45,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class DeepgramTTSSettings(TTSSettings): + """Typed settings for Deepgram TTS service. + + Parameters: + encoding: Audio encoding format (linear16, mulaw, alaw). + """ + + encoding: str = field(default_factory=lambda: NOT_GIVEN) + + class DeepgramTTSService(WebsocketTTSService): """Deepgram WebSocket-based text-to-speech service. @@ -91,10 +104,12 @@ class DeepgramTTSService(WebsocketTTSService): self._api_key = api_key self._base_url = base_url - self._settings = { - "encoding": encoding, - } - self.set_voice(voice) + self._settings: DeepgramTTSSettings = DeepgramTTSSettings( + model=voice, + voice=voice, + encoding=encoding, + ) + self._voice_id = voice self._receive_task = None self._context_id: Optional[str] = None @@ -177,7 +192,7 @@ class DeepgramTTSService(WebsocketTTSService): # Build WebSocket URL with query parameters params = [] params.append(f"model={self._voice_id}") - params.append(f"encoding={self._settings['encoding']}") + params.append(f"encoding={self._settings.encoding}") params.append(f"sample_rate={self.sample_rate}") url = f"{self._base_url}/v1/speak?{'&'.join(params)}" @@ -357,10 +372,12 @@ class DeepgramHttpTTSService(TTSService): self._api_key = api_key self._session = aiohttp_session self._base_url = base_url - self._settings = { - "encoding": encoding, - } - self.set_voice(voice) + self._settings: DeepgramTTSSettings = DeepgramTTSSettings( + model=voice, + voice=voice, + encoding=encoding, + ) + self._voice_id = voice def can_generate_metrics(self) -> bool: """Check if the service can generate metrics. @@ -390,7 +407,7 @@ class DeepgramHttpTTSService(TTSService): params = { "model": self._voice_id, - "encoding": self._settings["encoding"], + "encoding": self._settings.encoding, "sample_rate": self.sample_rate, "container": "none", } diff --git a/src/pipecat/services/deepseek/llm.py b/src/pipecat/services/deepseek/llm.py index 56f1ddd18..806dce13d 100644 --- a/src/pipecat/services/deepseek/llm.py +++ b/src/pipecat/services/deepseek/llm.py @@ -68,15 +68,15 @@ class DeepSeekLLMService(OpenAILLMService): "model": self.model_name, "stream": True, "stream_options": {"include_usage": True}, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 388f7146b..950dc5de9 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -14,6 +14,7 @@ transcription results directly. import base64 import io import json +from dataclasses import dataclass, field from enum import Enum from typing import AsyncGenerator, Optional @@ -33,6 +34,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import ELEVENLABS_REALTIME_TTFS_P99, ELEVENLABS_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -167,6 +169,44 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class ElevenLabsSTTSettings(STTSettings): + """Typed settings for the ElevenLabs file-based STT service. + + Parameters: + tag_audio_events: Whether to include audio event tags in transcription. + """ + + tag_audio_events: bool = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class ElevenLabsRealtimeSTTSettings(STTSettings): + """Typed settings for the ElevenLabs Realtime STT service. + + See ``ElevenLabsRealtimeSTTService.InputParams`` for detailed descriptions. + + Parameters: + commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD). + vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0). + vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive). + min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms). + min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms). + include_timestamps: Whether to include word-level timestamps in transcripts. + enable_logging: Whether to enable logging on ElevenLabs' side. + include_language_detection: Whether to include language detection in transcripts. + """ + + commit_strategy: CommitStrategy = field(default_factory=lambda: NOT_GIVEN) + vad_silence_threshold_secs: float = field(default_factory=lambda: NOT_GIVEN) + vad_threshold: float = field(default_factory=lambda: NOT_GIVEN) + min_speech_duration_ms: int = field(default_factory=lambda: NOT_GIVEN) + min_silence_duration_ms: int = field(default_factory=lambda: NOT_GIVEN) + include_timestamps: bool = field(default_factory=lambda: NOT_GIVEN) + enable_logging: bool = field(default_factory=lambda: NOT_GIVEN) + include_language_detection: bool = field(default_factory=lambda: NOT_GIVEN) + + class ElevenLabsSTTService(SegmentedSTTService): """Speech-to-text service using ElevenLabs' file-based API. @@ -223,13 +263,15 @@ class ElevenLabsSTTService(SegmentedSTTService): self._base_url = base_url self._session = aiohttp_session self._model_id = model - self._tag_audio_events = params.tag_audio_events - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: ElevenLabsSTTSettings = ElevenLabsSTTSettings( + model=model, + language=self.language_to_service_language(params.language) if params.language else "eng", - } + tag_audio_events=params.tag_audio_events, + ) + self.set_model_name(model) def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -250,27 +292,30 @@ class ElevenLabsSTTService(SegmentedSTTService): """ return language_to_elevenlabs_language(language) - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update. + + Converts language to ElevenLabs format before applying and keeps + ``_model_id`` in sync with the model setting. Args: - language: The language to use for speech-to-text transcription. + update: A :class:`STTSettings` (or ``ElevenLabsSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = self.language_to_service_language(language) + # Convert language to ElevenLabs format before applying + if is_given(update.language) and isinstance(update.language, Language): + converted = self.language_to_service_language(update.language) + if converted is not None: + update.language = converted - async def set_model(self, model: str): - """Set the STT model. + changed = await super()._update_settings_from_typed(update) - Args: - model: The model name to use for transcription. + if "model" in changed: + self._model_id = self._settings.model - Note: - ElevenLabs STT API does not currently support model selection. - This method is provided for interface compatibility. - """ - await super().set_model(model) - logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model") + return changed async def _transcribe_audio(self, audio_data: bytes) -> dict: """Upload audio data to ElevenLabs and get transcription result. @@ -298,8 +343,8 @@ class ElevenLabsSTTService(SegmentedSTTService): # Add required model_id, language_code, and tag_audio_events data.add_field("model_id", self._model_id) - data.add_field("language_code", self._settings["language"]) - data.add_field("tag_audio_events", str(self._tag_audio_events).lower()) + data.add_field("language_code", self._settings.language) + data.add_field("tag_audio_events", str(self._settings.tag_audio_events).lower()) async with self._session.post(url, data=data, headers=headers) as response: if response.status != 200: @@ -469,11 +514,22 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): self._api_key = api_key self._base_url = base_url self._model_id = model - self._params = params self._audio_format = "" # initialized in start() self._receive_task = None - self._settings = {"language": params.language_code} + self._settings: ElevenLabsRealtimeSTTSettings = ElevenLabsRealtimeSTTSettings( + model=model, + language=params.language_code, + commit_strategy=params.commit_strategy, + vad_silence_threshold_secs=params.vad_silence_threshold_secs, + vad_threshold=params.vad_threshold, + min_speech_duration_ms=params.min_speech_duration_ms, + min_silence_duration_ms=params.min_silence_duration_ms, + include_timestamps=params.include_timestamps, + enable_logging=params.enable_logging, + include_language_detection=params.include_language_detection, + ) + self.set_model_name(model) def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -483,42 +539,35 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and reconnect if anything changed. + + Converts language to ElevenLabs format before applying and keeps + ``_model_id`` in sync. Args: - language: The language to use for speech-to-text transcription. + update: A :class:`STTSettings` (or ``ElevenLabsRealtimeSTTSettings``) delta. - Note: - Changing language requires reconnecting to the WebSocket. + Returns: + Set of field names whose values actually changed. """ - logger.info(f"Switching STT language to: [{language}]") - new_language = ( - language_to_elevenlabs_language(language) - if isinstance(language, Language) - else language - ) - self._params.language_code = new_language - self._settings["language"] = new_language - # Reconnect with new settings - await self._disconnect() - await self._connect() - - async def set_model(self, model: str): - """Set the STT model. - - Args: - model: The model name to use for transcription. - - Note: - Changing model requires reconnecting to the WebSocket. - """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") - self._model_id = model - # Reconnect with new settings + # Convert language to ElevenLabs format before applying + if is_given(update.language) and isinstance(update.language, Language): + converted = language_to_elevenlabs_language(update.language) + if converted is not None: + update.language = converted + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + if "model" in changed: + self._model_id = self._settings.model + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the STT service and establish WebSocket connection. @@ -566,7 +615,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): # Send commit when user stops speaking (manual commit mode) - if self._params.commit_strategy == CommitStrategy.MANUAL: + if self._settings.commit_strategy == CommitStrategy.MANUAL: if self._websocket and self._websocket.state is State.OPEN: try: commit_message = { @@ -656,36 +705,40 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): # Build query parameters params = [f"model_id={self._model_id}"] - if self._params.language_code: - params.append(f"language_code={self._params.language_code}") + if self._settings.language: + params.append(f"language_code={self._settings.language}") params.append(f"audio_format={self._audio_format}") - params.append(f"commit_strategy={self._params.commit_strategy.value}") + params.append(f"commit_strategy={self._settings.commit_strategy.value}") # Add optional parameters - if self._params.include_timestamps: - params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}") - - if self._params.enable_logging: - params.append(f"enable_logging={str(self._params.enable_logging).lower()}") - - if self._params.include_language_detection: + if self._settings.include_timestamps: params.append( - f"include_language_detection={str(self._params.include_language_detection).lower()}" + f"include_timestamps={str(self._settings.include_timestamps).lower()}" + ) + + if self._settings.enable_logging: + params.append(f"enable_logging={str(self._settings.enable_logging).lower()}") + + if self._settings.include_language_detection: + params.append( + f"include_language_detection={str(self._settings.include_language_detection).lower()}" ) # Add VAD parameters if using VAD commit strategy and values are specified - if self._params.commit_strategy == CommitStrategy.VAD: - if self._params.vad_silence_threshold_secs is not None: + if self._settings.commit_strategy == CommitStrategy.VAD: + if self._settings.vad_silence_threshold_secs is not None: params.append( - f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}" + f"vad_silence_threshold_secs={self._settings.vad_silence_threshold_secs}" + ) + if self._settings.vad_threshold is not None: + params.append(f"vad_threshold={self._settings.vad_threshold}") + if self._settings.min_speech_duration_ms is not None: + params.append(f"min_speech_duration_ms={self._settings.min_speech_duration_ms}") + if self._settings.min_silence_duration_ms is not None: + params.append( + f"min_silence_duration_ms={self._settings.min_silence_duration_ms}" ) - if self._params.vad_threshold is not None: - params.append(f"vad_threshold={self._params.vad_threshold}") - if self._params.min_speech_duration_ms is not None: - params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}") - if self._params.min_silence_duration_ms is not None: - params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}") ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}" @@ -817,7 +870,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): """ # If timestamps are enabled, skip this message and wait for the # committed_transcript_with_timestamps message which contains all the data - if self._params.include_timestamps: + if self._settings.include_timestamps: return text = data.get("text", "").strip() diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 4dab0c01a..b061383f3 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -13,7 +13,19 @@ with support for streaming audio, word timestamps, and voice customization. import asyncio import base64 import json -from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union +from dataclasses import dataclass, field +from typing import ( + Any, + AsyncGenerator, + ClassVar, + Dict, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) import aiohttp from loguru import logger @@ -32,6 +44,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import ( AudioContextWordTTSService, WordTTSService, @@ -136,12 +149,12 @@ def output_format_from_sample_rate(sample_rate: int) -> str: def build_elevenlabs_voice_settings( - settings: Dict[str, Any], + settings: Union[Dict[str, Any], "TTSSettings"], ) -> Optional[Dict[str, Union[float, bool]]]: """Build voice settings dictionary for ElevenLabs based on provided settings. Args: - settings: Dictionary containing voice settings parameters. + settings: Dictionary or typed settings containing voice settings parameters. Returns: Dictionary of voice settings or None if no valid settings are provided. @@ -150,8 +163,11 @@ def build_elevenlabs_voice_settings( voice_settings = {} for key in voice_setting_keys: - if key in settings and settings[key] is not None: - voice_settings[key] = settings[key] + val = ( + getattr(settings, key, None) if isinstance(settings, TTSSettings) else settings.get(key) + ) + if val is not None and is_given(val): + voice_settings[key] = val return voice_settings or None @@ -168,6 +184,75 @@ class PronunciationDictionaryLocator(BaseModel): version_id: str +@dataclass +class ElevenLabsTTSSettings(TTSSettings): + """Typed settings for the ElevenLabs WebSocket TTS service. + + Fields that appear in the WebSocket URL (``voice``, ``model``, + ``language``) require a full reconnect when changed. Fields that + affect the voice character (``stability``, ``similarity_boost``, + ``style``, ``use_speaker_boost``, ``speed``) can be applied by closing + the current audio context so a new one is opened with updated settings. + + Parameters: + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.7 to 1.2). + auto_mode: Whether to enable automatic mode optimization. + enable_ssml_parsing: Whether to parse SSML tags in text. + enable_logging: Whether to enable ElevenLabs logging. + apply_text_normalization: Text normalization mode ("auto", "on", "off"). + """ + + stability: float = field(default_factory=lambda: NOT_GIVEN) + similarity_boost: float = field(default_factory=lambda: NOT_GIVEN) + style: float = field(default_factory=lambda: NOT_GIVEN) + use_speaker_boost: bool = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + auto_mode: str = field(default_factory=lambda: NOT_GIVEN) + enable_ssml_parsing: bool = field(default_factory=lambda: NOT_GIVEN) + enable_logging: bool = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: str = field(default_factory=lambda: NOT_GIVEN) + + #: Fields in the WS URL — changing any of these requires a reconnect. + URL_FIELDS: ClassVar[frozenset[str]] = frozenset({"voice", "model", "language"}) + + #: Fields affecting voice character — changing these requires closing the + #: current audio context so the next one picks up new settings. + VOICE_SETTINGS_FIELDS: ClassVar[frozenset[str]] = frozenset( + {"stability", "similarity_boost", "style", "use_speaker_boost", "speed"} + ) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + +@dataclass +class ElevenLabsHttpTTSSettings(TTSSettings): + """Typed settings for the ElevenLabs HTTP TTS service. + + Parameters: + optimize_streaming_latency: Latency optimization level (0-4). + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.25 to 4.0). + apply_text_normalization: Text normalization mode ("auto", "on", "off"). + """ + + optimize_streaming_latency: int = field(default_factory=lambda: NOT_GIVEN) + stability: float = field(default_factory=lambda: NOT_GIVEN) + similarity_boost: float = field(default_factory=lambda: NOT_GIVEN) + style: float = field(default_factory=lambda: NOT_GIVEN) + use_speaker_boost: bool = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: str = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float, @@ -316,22 +401,25 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._api_key = api_key self._url = url - self._settings = { - "language": self.language_to_service_language(params.language) - if params.language - else None, - "stability": params.stability, - "similarity_boost": params.similarity_boost, - "style": params.style, - "use_speaker_boost": params.use_speaker_boost, - "speed": params.speed, - "auto_mode": str(params.auto_mode).lower(), - "enable_ssml_parsing": params.enable_ssml_parsing, - "enable_logging": params.enable_logging, - "apply_text_normalization": params.apply_text_normalization, - } + self._settings: ElevenLabsTTSSettings = ElevenLabsTTSSettings( + model=model, + voice=voice_id, + language=( + self.language_to_service_language(params.language) if params.language else None + ), + stability=params.stability, + similarity_boost=params.similarity_boost, + style=params.style, + use_speaker_boost=params.use_speaker_boost, + speed=params.speed, + auto_mode=str(params.auto_mode).lower(), + enable_ssml_parsing=params.enable_ssml_parsing, + enable_logging=params.enable_logging, + apply_text_normalization=params.apply_text_normalization, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id + self._output_format = "" # initialized in start() self._voice_settings = self._set_voice_settings() self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators @@ -366,54 +454,57 @@ class ElevenLabsTTSService(AudioContextWordTTSService): return language_to_elevenlabs_language(language) def _set_voice_settings(self): - return build_elevenlabs_voice_settings(self._settings) + ts = self._settings + voice_setting_keys = [ + "stability", + "similarity_boost", + "style", + "use_speaker_boost", + "speed", + ] + voice_settings = {} + for key in voice_setting_keys: + val = getattr(ts, key, None) + if val is not None and is_given(val): + voice_settings[key] = val + return voice_settings or None - async def set_model(self, model: str): - """Set the TTS model and reconnect. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update, reconnecting as needed. + + Uses the declarative ``URL_FIELDS`` and ``VOICE_SETTINGS_FIELDS`` + sets on :class:`ElevenLabsTTSSettings` to decide whether to + reconnect the WebSocket or close the current audio context. Args: - model: The model name to use for synthesis. + update: A :class:`TTSSettings` (or ``ElevenLabsTTSSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() + changed = await super()._update_settings_from_typed(update) - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice, model, or language changed.""" - # Track previous values for settings that require reconnection - prev_voice = self._voice_id - prev_model = self.model_name - prev_language = self._settings.get("language") - # Create snapshot of current voice settings to detect changes after update - prev_voice_settings = self._voice_settings.copy() if self._voice_settings else None + if not changed: + return changed - await super()._update_settings(settings) - - # Update voice settings for the next context creation + # Rebuild voice settings for next context self._voice_settings = self._set_voice_settings() - # Check if URL-level settings changed (these require reconnection) - url_changed = ( - prev_voice != self._voice_id - or prev_model != self.model_name - or prev_language != self._settings.get("language") - ) - - # Check if only voice settings changed (speed, stability, etc.) - voice_settings_changed = prev_voice_settings != self._voice_settings + url_changed = bool(changed & ElevenLabsTTSSettings.URL_FIELDS) + voice_settings_changed = bool(changed & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS) if url_changed: - # These settings are in the WebSocket URL, so we need to reconnect logger.debug( - f"URL-level setting changed (voice/model/language), reconnecting WebSocket" + f"URL-level setting changed ({changed & ElevenLabsTTSSettings.URL_FIELDS}), " + f"reconnecting WebSocket" ) await self._disconnect() await self._connect() elif voice_settings_changed and self._context_id: - # Voice settings can be updated by closing current context - # so new one gets created with updated voice settings - logger.debug(f"Voice settings changed, closing current context to apply changes") + logger.debug( + f"Voice settings changed ({changed & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS}), " + f"closing current context to apply changes" + ) try: if self._websocket: await self._websocket.send( @@ -423,6 +514,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._context_id = None + return changed + async def start(self, frame: StartFrame): """Start the ElevenLabs TTS service. @@ -505,19 +598,19 @@ class ElevenLabsTTSService(AudioContextWordTTSService): voice_id = self._voice_id model = self.model_name output_format = self._output_format - url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}" + url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings.auto_mode}" - if self._settings["enable_ssml_parsing"]: - url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}" + if self._settings.enable_ssml_parsing: + url += f"&enable_ssml_parsing={self._settings.enable_ssml_parsing}" - if self._settings["enable_logging"]: - url += f"&enable_logging={self._settings['enable_logging']}" + if self._settings.enable_logging: + url += f"&enable_logging={self._settings.enable_logging}" - if self._settings["apply_text_normalization"] is not None: - url += f"&apply_text_normalization={self._settings['apply_text_normalization']}" + if self._settings.apply_text_normalization is not None: + url += f"&apply_text_normalization={self._settings.apply_text_normalization}" # Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS - language = self._settings["language"] + language = self._settings.language if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None: url += f"&language_code={language}" logger.debug(f"Using language code: {language}") @@ -809,20 +902,22 @@ class ElevenLabsHttpTTSService(WordTTSService): self._params = params self._session = aiohttp_session - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: ElevenLabsHttpTTSSettings = ElevenLabsHttpTTSSettings( + model=model, + voice=voice_id, + language=self.language_to_service_language(params.language) if params.language else None, - "optimize_streaming_latency": params.optimize_streaming_latency, - "stability": params.stability, - "similarity_boost": params.similarity_boost, - "style": params.style, - "use_speaker_boost": params.use_speaker_boost, - "speed": params.speed, - "apply_text_normalization": params.apply_text_normalization, - } + optimize_streaming_latency=params.optimize_streaming_latency, + stability=params.stability, + similarity_boost=params.similarity_boost, + style=params.style, + use_speaker_boost=params.use_speaker_boost, + speed=params.speed, + apply_text_normalization=params.apply_text_normalization, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id self._output_format = "" # initialized in start() self._voice_settings = self._set_voice_settings() self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators @@ -859,10 +954,19 @@ class ElevenLabsHttpTTSService(WordTTSService): def _set_voice_settings(self): return build_elevenlabs_voice_settings(self._settings) - async def _update_settings(self, settings: Mapping[str, Any]): - await super()._update_settings(settings) - # Update voice settings for the next context creation - self._voice_settings = self._set_voice_settings() + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and rebuild voice settings. + + Args: + update: A :class:`TTSSettings` (or ``ElevenLabsHttpTTSSettings``) delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + if changed: + self._voice_settings = self._set_voice_settings() + return changed def _reset_state(self): """Reset internal state variables.""" @@ -999,10 +1103,13 @@ class ElevenLabsHttpTTSService(WordTTSService): locator.model_dump() for locator in self._pronunciation_dictionary_locators ] - if self._settings["apply_text_normalization"] is not None: - payload["apply_text_normalization"] = self._settings["apply_text_normalization"] + if ( + is_given(self._settings.apply_text_normalization) + and self._settings.apply_text_normalization is not None + ): + payload["apply_text_normalization"] = self._settings.apply_text_normalization - language = self._settings["language"] + language = self._settings.language if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language: payload["language_code"] = language logger.debug(f"Using language code: {language}") @@ -1020,8 +1127,11 @@ class ElevenLabsHttpTTSService(WordTTSService): params = { "output_format": self._output_format, } - if self._settings["optimize_streaming_latency"] is not None: - params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"] + if ( + is_given(self._settings.optimize_streaming_latency) + and self._settings.optimize_streaming_latency is not None + ): + params["optimize_streaming_latency"] = self._settings.optimize_streaming_latency try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index 4e8a655ec..eef0e0487 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -11,12 +11,14 @@ transcription using segmented audio processing. """ import os +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import FAL_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -146,6 +148,22 @@ def language_to_fal_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class FalSTTSettings(STTSettings): + """Typed settings for the Fal Wizper STT service. + + Parameters: + task: Task to perform ('transcribe' or 'translate'). Defaults to + 'transcribe'. + chunk_level: Level of chunking ('segment'). Defaults to 'segment'. + version: Version of Wizper model to use. Defaults to '3'. + """ + + task: str = field(default_factory=lambda: NOT_GIVEN) + chunk_level: str = field(default_factory=lambda: NOT_GIVEN) + version: str = field(default_factory=lambda: NOT_GIVEN) + + class FalSTTService(SegmentedSTTService): """Speech-to-text service using Fal's Wizper API. @@ -203,14 +221,14 @@ class FalSTTService(SegmentedSTTService): ) self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY")) - self._settings = { - "task": params.task, - "language": self.language_to_service_language(params.language) + self._settings: FalSTTSettings = FalSTTSettings( + language=self.language_to_service_language(params.language) if params.language else "en", - "chunk_level": params.chunk_level, - "version": params.version, - } + task=params.task, + chunk_level=params.chunk_level, + version=params.version, + ) def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -231,23 +249,17 @@ class FalSTTService(SegmentedSTTService): """ return language_to_fal_language(language) - async def set_language(self, language: Language): - """Set the transcription language. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, converting language if changed.""" + changed = await super()._update_settings_from_typed(update) - Args: - language: The language to use for speech-to-text transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = self.language_to_service_language(language) + if "language" in changed: + # Convert the Language enum to a Fal language code. + lang = self._settings.language + if isinstance(lang, Language): + self._settings.language = self.language_to_service_language(lang) - async def set_model(self, model: str): - """Set the STT model. - - Args: - model: The model name to use for transcription. - """ - await super().set_model(model) - logger.info(f"Switching STT model to: [{model}]") + return changed @traced_stt async def _handle_transcription( @@ -276,19 +288,19 @@ class FalSTTService(SegmentedSTTService): data_uri = fal_client.encode(audio, "audio/x-wav") response = await self._fal_client.run( "fal-ai/wizper", - arguments={"audio_url": data_uri, **self._settings}, + arguments={"audio_url": data_uri, **self._settings.given_fields()}, ) if response and "text" in response: text = response["text"].strip() if text: # Only yield non-empty text - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - Language(self._settings["language"]), + Language(self._settings.language), result=response, ) diff --git a/src/pipecat/services/fireworks/llm.py b/src/pipecat/services/fireworks/llm.py index d7bf57908..9338d8c5a 100644 --- a/src/pipecat/services/fireworks/llm.py +++ b/src/pipecat/services/fireworks/llm.py @@ -68,15 +68,15 @@ class FireworksLLMService(OpenAILLMService): params = { "model": self.model_name, "stream": True, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 93a718429..5517758ad 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -11,6 +11,7 @@ for streaming text-to-speech synthesis with customizable voice parameters. """ import uuid +from dataclasses import dataclass, field from typing import AsyncGenerator, Literal, Optional from loguru import logger @@ -28,6 +29,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleTTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -45,6 +47,29 @@ except ModuleNotFoundError as e: FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] +@dataclass +class FishAudioTTSSettings(TTSSettings): + """Typed settings for Fish Audio TTS service. + + Parameters: + fish_sample_rate: Audio sample rate sent to the API. + latency: Latency mode ("normal" or "balanced"). Defaults to "normal". + format: Audio output format. + normalize: Whether to normalize audio output. Defaults to True. + prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0. + prosody_volume: Volume adjustment in dB. Defaults to 0. + reference_id: Reference ID of the voice model. + """ + + fish_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + latency: str = field(default_factory=lambda: NOT_GIVEN) + format: str = field(default_factory=lambda: NOT_GIVEN) + normalize: bool = field(default_factory=lambda: NOT_GIVEN) + prosody_speed: float = field(default_factory=lambda: NOT_GIVEN) + prosody_volume: int = field(default_factory=lambda: NOT_GIVEN) + reference_id: str = field(default_factory=lambda: NOT_GIVEN) + + class FishAudioTTSService(InterruptibleTTSService): """Fish Audio text-to-speech service with WebSocket streaming. @@ -136,17 +161,16 @@ class FishAudioTTSService(InterruptibleTTSService): self._receive_task = None self._request_id = None - self._settings = { - "sample_rate": 0, - "latency": params.latency, - "format": output_format, - "normalize": params.normalize, - "prosody": { - "speed": params.prosody_speed, - "volume": params.prosody_volume, - }, - "reference_id": reference_id, - } + self._settings: FishAudioTTSSettings = FishAudioTTSSettings( + voice=reference_id, + fish_sample_rate=0, + latency=params.latency, + format=output_format, + normalize=params.normalize, + prosody_speed=params.prosody_speed, + prosody_volume=params.prosody_volume, + reference_id=reference_id, + ) self.set_model_name(model_id) @@ -158,16 +182,22 @@ class FishAudioTTSService(InterruptibleTTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model and reconnect. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if needed. + + Any change to voice or model triggers a WebSocket reconnect. Args: - model: The model name to use for synthesis. + update: A :class:`TTSSettings` (or ``FishAudioTTSSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() + changed = await super()._update_settings_from_typed(update) + if changed: + await self._disconnect() + await self._connect() + return changed async def start(self, frame: StartFrame): """Start the Fish Audio TTS service. @@ -176,7 +206,7 @@ class FishAudioTTSService(InterruptibleTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.fish_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -225,7 +255,18 @@ class FishAudioTTSService(InterruptibleTTSService): self._websocket = await websocket_connect(self._base_url, additional_headers=headers) # Send initial start message with ormsgpack - start_message = {"event": "start", "request": {"text": "", **self._settings}} + request_settings = { + "sample_rate": self._settings.fish_sample_rate, + "latency": self._settings.latency, + "format": self._settings.format, + "normalize": self._settings.normalize, + "prosody": { + "speed": self._settings.prosody_speed, + "volume": self._settings.prosody_volume, + }, + "reference_id": self._settings.reference_id, + } + start_message = {"event": "start", "request": {"text": "", **request_settings}} await self._websocket.send(ormsgpack.packb(start_message)) logger.debug("Sent start message to Fish Audio") diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index 475a7213e..76a1620e1 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -14,6 +14,7 @@ import asyncio import base64 import json import warnings +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, Literal, Optional import aiohttp @@ -32,6 +33,7 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.services.gladia.config import GladiaInputParams +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import GLADIA_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -178,6 +180,17 @@ class _InputParamsDescriptor: return GladiaInputParams +@dataclass +class GladiaSTTSettings(STTSettings): + """Typed settings for Gladia STT service. + + Parameters: + input_params: Gladia ``GladiaInputParams`` for detailed configuration. + """ + + input_params: GladiaInputParams = field(default_factory=lambda: NOT_GIVEN) + + class GladiaSTTService(WebsocketSTTService): """Speech-to-Text service using Gladia's API. @@ -265,9 +278,8 @@ class GladiaSTTService(WebsocketSTTService): self._region = region self._url = url self.set_model_name(model) - self._params = params self._receive_task = None - self._settings = {} + self._settings = GladiaSTTSettings(model=model, input_params=params) # Session management self._session_url = None @@ -307,31 +319,33 @@ class GladiaSTTService(WebsocketSTTService): return language_to_gladia_language(language) def _prepare_settings(self) -> Dict[str, Any]: + params = self._settings.input_params + settings = { - "encoding": self._params.encoding or "wav/pcm", - "bit_depth": self._params.bit_depth or 16, + "encoding": params.encoding or "wav/pcm", + "bit_depth": params.bit_depth or 16, "sample_rate": self.sample_rate, - "channels": self._params.channels or 1, + "channels": params.channels or 1, "model": self._model_name, } # Add custom_metadata if provided - settings["custom_metadata"] = dict(self._params.custom_metadata or {}) + settings["custom_metadata"] = dict(params.custom_metadata or {}) settings["custom_metadata"]["pipecat"] = pipecat_version() # Add endpointing parameters if provided - if self._params.endpointing is not None: - settings["endpointing"] = self._params.endpointing - if self._params.maximum_duration_without_endpointing is not None: + if params.endpointing is not None: + settings["endpointing"] = params.endpointing + if params.maximum_duration_without_endpointing is not None: settings["maximum_duration_without_endpointing"] = ( - self._params.maximum_duration_without_endpointing + params.maximum_duration_without_endpointing ) # Add language configuration (prioritize language_config over deprecated language) - if self._params.language_config: - settings["language_config"] = self._params.language_config.model_dump(exclude_none=True) - elif self._params.language: # Backward compatibility for deprecated parameter - language_code = self.language_to_service_language(self._params.language) + if params.language_config: + settings["language_config"] = params.language_config.model_dump(exclude_none=True) + elif params.language: # Backward compatibility for deprecated parameter + language_code = self.language_to_service_language(params.language) if language_code: settings["language_config"] = { "languages": [language_code], @@ -339,21 +353,18 @@ class GladiaSTTService(WebsocketSTTService): } # Add pre_processing configuration if provided - if self._params.pre_processing: - settings["pre_processing"] = self._params.pre_processing.model_dump(exclude_none=True) + if params.pre_processing: + settings["pre_processing"] = params.pre_processing.model_dump(exclude_none=True) # Add realtime_processing configuration if provided - if self._params.realtime_processing: - settings["realtime_processing"] = self._params.realtime_processing.model_dump( + if params.realtime_processing: + settings["realtime_processing"] = params.realtime_processing.model_dump( exclude_none=True ) # Add messages_config if provided - if self._params.messages_config: - settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True) - - # Store settings for tracing - self._settings = settings + if params.messages_config: + settings["messages_config"] = params.messages_config.model_dump(exclude_none=True) return settings @@ -366,6 +377,31 @@ class GladiaSTTService(WebsocketSTTService): await super().start(frame) await self._connect() + async def _update_settings_from_typed(self, update: GladiaSTTSettings) -> set[str]: + """Apply typed settings update. + + Gladia sessions are fixed at creation time, so any change requires + a full session teardown and reconnect. + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # Gladia sessions are fixed — need to tear down and recreate + self._session_url = None + self._session_id = None + await self._disconnect() + await self._connect() + + return changed + async def stop(self, frame: EndFrame): """Stop the Gladia STT websocket connection. @@ -522,7 +558,7 @@ class GladiaSTTService(WebsocketSTTService): Broadcasts UserStartedSpeakingFrame and optionally triggers interruption when VAD is enabled. """ - if not self._params.enable_vad or self._is_speaking: + if not self._settings.input_params.enable_vad or self._is_speaking: return logger.debug(f"{self} User started speaking") @@ -537,7 +573,7 @@ class GladiaSTTService(WebsocketSTTService): Broadcasts UserStoppedSpeakingFrame when VAD is enabled. """ - if not self._params.enable_vad or not self._is_speaking: + if not self._settings.input_params.enable_vad or not self._is_speaking: return self._is_speaking = False await self.broadcast_frame(UserStoppedSpeakingFrame) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index e209f3d0a..1edab5783 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -17,9 +17,9 @@ import io import time import uuid import warnings -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, ClassVar, Dict, List, Optional, Union from loguru import logger from PIL import Image @@ -47,7 +47,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, @@ -77,6 +76,7 @@ from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.string import match_endofsentence from pipecat.utils.time import time_now_iso8601 @@ -602,6 +602,31 @@ class InputParams(BaseModel): extra: Optional[Dict[str, Any]] = Field(default_factory=dict) +@dataclass +class GeminiLiveLLMSettings(LLMSettings): + """Typed settings for Gemini Live LLM services. + + Parameters: + modalities: Response modalities. + language: Language for generation. + media_resolution: Media resolution setting. + vad: Voice activity detection parameters. + context_window_compression: Context window compression configuration. + thinking: Thinking configuration. + enable_affective_dialog: Whether to enable affective dialog. + proactivity: Proactivity configuration. + """ + + modalities: Any = field(default_factory=lambda: NOT_GIVEN) + language: Any = field(default_factory=lambda: NOT_GIVEN) + media_resolution: Any = field(default_factory=lambda: NOT_GIVEN) + vad: Any = field(default_factory=lambda: NOT_GIVEN) + context_window_compression: Any = field(default_factory=lambda: NOT_GIVEN) + thinking: Any = field(default_factory=lambda: NOT_GIVEN) + enable_affective_dialog: Any = field(default_factory=lambda: NOT_GIVEN) + proactivity: Any = field(default_factory=lambda: NOT_GIVEN) + + class GeminiLiveLLMService(LLMService): """Provides access to Google's Gemini Live API. @@ -714,25 +739,26 @@ class GeminiLiveLLMService(LLMService): self._consecutive_failures = 0 self._connection_start_time = None - self._settings = { - "frequency_penalty": params.frequency_penalty, - "max_tokens": params.max_tokens, - "presence_penalty": params.presence_penalty, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "modalities": params.modalities, - "language": self._language_code, - "media_resolution": params.media_resolution, - "vad": params.vad, - "context_window_compression": params.context_window_compression.model_dump() + self._settings = GeminiLiveLLMSettings( + model=model, + frequency_penalty=params.frequency_penalty, + max_tokens=params.max_tokens, + presence_penalty=params.presence_penalty, + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + modalities=params.modalities, + language=self._language_code, + media_resolution=params.media_resolution, + vad=params.vad, + context_window_compression=params.context_window_compression.model_dump() if params.context_window_compression else {}, - "thinking": params.thinking or {}, - "enable_affective_dialog": params.enable_affective_dialog or False, - "proactivity": params.proactivity or {}, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + thinking=params.thinking or {}, + enable_affective_dialog=params.enable_affective_dialog or False, + proactivity=params.proactivity or {}, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) self._file_api_base_url = file_api_base_url self._file_api: Optional[GeminiFileAPI] = None @@ -798,7 +824,7 @@ class GeminiLiveLLMService(LLMService): Args: modalities: The modalities to use for responses. """ - self._settings["modalities"] = modalities + self._settings.modalities = modalities def set_language(self, language: Language): """Set the language for generation. @@ -808,7 +834,7 @@ class GeminiLiveLLMService(LLMService): """ self._language = language self._language_code = language_to_gemini_language(language) or "en-US" - self._settings["language"] = self._language_code + self._settings.language = self._language_code logger.info(f"Set Gemini language to: {self._language_code}") async def set_context(self, context: OpenAILLMContext): @@ -866,7 +892,7 @@ class GeminiLiveLLMService(LLMService): async def _handle_interruption(self): if self._bot_is_responding: await self._set_bot_is_responding(False) - if self._settings.get("modalities") == GeminiModalities.AUDIO: + if self._settings.modalities == GeminiModalities.AUDIO: await self.push_frame(TTSStoppedFrame()) # Do not send LLMFullResponseEndFrame here - an interruption # already tells the assistant context aggregator that the response @@ -947,8 +973,6 @@ class GeminiLiveLLMService(LLMService): # uses this frame *without* a user context aggregator still works # (we have an example that does just that, actually). await self._create_single_response(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() else: @@ -1074,20 +1098,20 @@ class GeminiLiveLLMService(LLMService): # Assemble basic configuration config = LiveConnectConfig( generation_config=GenerationConfig( - frequency_penalty=self._settings["frequency_penalty"], - max_output_tokens=self._settings["max_tokens"], - presence_penalty=self._settings["presence_penalty"], - temperature=self._settings["temperature"], - top_k=self._settings["top_k"], - top_p=self._settings["top_p"], - response_modalities=[Modality(self._settings["modalities"].value)], + frequency_penalty=self._settings.frequency_penalty, + max_output_tokens=self._settings.max_tokens, + presence_penalty=self._settings.presence_penalty, + temperature=self._settings.temperature, + top_k=self._settings.top_k, + top_p=self._settings.top_p, + response_modalities=[Modality(self._settings.modalities.value)], speech_config=SpeechConfig( voice_config=VoiceConfig( prebuilt_voice_config={"voice_name": self._voice_id} ), - language_code=self._settings["language"], + language_code=self._settings.language, ), - media_resolution=MediaResolution(self._settings["media_resolution"].value), + media_resolution=MediaResolution(self._settings.media_resolution.value), ), input_audio_transcription=AudioTranscriptionConfig(), output_audio_transcription=AudioTranscriptionConfig(), @@ -1095,37 +1119,36 @@ class GeminiLiveLLMService(LLMService): ) # Add context window compression to configuration, if enabled - if self._settings.get("context_window_compression", {}).get("enabled", False): + cwc = self._settings.context_window_compression or {} + if cwc.get("enabled", False): compression_config = ContextWindowCompressionConfig() # Add sliding window (always true if compression is enabled) compression_config.sliding_window = SlidingWindow() # Add trigger_tokens if specified - trigger_tokens = self._settings.get("context_window_compression", {}).get( - "trigger_tokens" - ) + trigger_tokens = cwc.get("trigger_tokens") if trigger_tokens is not None: compression_config.trigger_tokens = trigger_tokens config.context_window_compression = compression_config # Add thinking configuration to configuration, if provided - if self._settings.get("thinking"): - config.thinking_config = self._settings["thinking"] + if self._settings.thinking: + config.thinking_config = self._settings.thinking # Add affective dialog setting, if provided - if self._settings.get("enable_affective_dialog", False): - config.enable_affective_dialog = self._settings["enable_affective_dialog"] + if self._settings.enable_affective_dialog: + config.enable_affective_dialog = self._settings.enable_affective_dialog # Add proactivity configuration to configuration, if provided - if self._settings.get("proactivity"): - config.proactivity = self._settings["proactivity"] + if self._settings.proactivity: + config.proactivity = self._settings.proactivity # Add VAD configuration to configuration, if provided - if self._settings.get("vad"): + if self._settings.vad: vad_config = AutomaticActivityDetection() - vad_params = self._settings["vad"] + vad_params = self._settings.vad has_vad_settings = False # Only add parameters that are explicitly set @@ -1604,7 +1627,7 @@ class GeminiLiveLLMService(LLMService): text: The transcription text to push result: Optional LiveServerMessage that triggered this transcription """ - await self._handle_user_transcription(text, True, self._settings["language"]) + await self._handle_user_transcription(text, True, self._settings.language) await self.push_frame( TranscriptionFrame( text=text, diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 563acadb3..692106241 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -15,8 +15,8 @@ import io import json import os import uuid -from dataclasses import dataclass -from typing import Any, AsyncIterator, Dict, List, Literal, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncIterator, ClassVar, Dict, List, Literal, Optional from loguru import logger from PIL import Image @@ -39,7 +39,6 @@ from pipecat.frames.frames import ( LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, - LLMUpdateSettingsFrame, ) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext @@ -59,6 +58,7 @@ from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm # Suppress gRPC fork warnings @@ -673,6 +673,17 @@ class GoogleLLMContext(OpenAILLMContext): self._messages = [m for m in self._messages if m.parts] +@dataclass +class GoogleLLMSettings(LLMSettings): + """Typed settings for Google LLM services. + + Parameters: + thinking: Thinking configuration. + """ + + thinking: Any = field(default_factory=lambda: NOT_GIVEN) + + class GoogleLLMService(LLMService): """Google AI (Gemini) LLM service implementation. @@ -773,14 +784,15 @@ class GoogleLLMService(LLMService): self._system_instruction = system_instruction self._http_options = update_google_client_http_options(http_options) - self._settings = { - "max_tokens": params.max_tokens, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "thinking": params.thinking, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + self._settings = GoogleLLMSettings( + model=model, + max_tokens=params.max_tokens, + temperature=params.temperature, + top_k=params.top_k, + top_p=params.top_p, + thinking=params.thinking, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) self._tools = tools self._tool_config = tool_config @@ -874,10 +886,10 @@ class GoogleLLMService(LLMService): k: v for k, v in { "system_instruction": system_instruction, - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "top_k": self._settings["top_k"], - "max_output_tokens": self._settings["max_tokens"], + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "top_k": self._settings.top_k, + "max_output_tokens": self._settings.max_tokens, "tools": tools, "tool_config": tool_config, }.items() @@ -885,13 +897,13 @@ class GoogleLLMService(LLMService): } # Add thinking parameters if configured - if self._settings["thinking"]: - generation_params["thinking_config"] = self._settings["thinking"].model_dump( + if self._settings.thinking: + generation_params["thinking_config"] = self._settings.thinking.model_dump( exclude_unset=True ) - if self._settings["extra"]: - generation_params.update(self._settings["extra"]) + if self._settings.extra: + generation_params.update(self._settings.extra) return generation_params @@ -1190,8 +1202,6 @@ class GoogleLLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = GoogleLLMContext(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 23396b0b8..8f762da9d 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -15,13 +15,15 @@ import asyncio import json import os import time +import warnings +from dataclasses import dataclass, field from pipecat.utils.tracing.service_decorators import traced_stt # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" -from typing import AsyncGenerator, List, Optional, Union +from typing import Any, AsyncGenerator, List, Optional, Union from loguru import logger from pydantic import BaseModel, Field, field_validator @@ -34,6 +36,7 @@ from pipecat.frames.frames import ( StartFrame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import GOOGLE_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -355,6 +358,44 @@ def language_to_google_stt_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class GoogleSTTSettings(STTSettings): + """Typed settings for Google Cloud Speech-to-Text V2. + + Parameters: + languages: List of ``Language`` enums for recognition + (e.g. ``[Language.EN_US]``). Preferred over ``language_codes``. + language_codes: List of Google STT language code strings + (e.g. ``["en-US"]``). + + .. deprecated:: 0.0.103 + Use ``languages`` instead. If both are provided, ``languages`` + takes precedence. This field is here just for backward + compatibility with dict-based settings updates. + use_separate_recognition_per_channel: Process each audio channel separately. + enable_automatic_punctuation: Add punctuation to transcripts. + enable_spoken_punctuation: Include spoken punctuation in transcript. + enable_spoken_emojis: Include spoken emojis in transcript. + profanity_filter: Filter profanity from transcript. + enable_word_time_offsets: Include timing information for each word. + enable_word_confidence: Include confidence scores for each word. + enable_interim_results: Stream partial recognition results. + enable_voice_activity_events: Detect voice activity in audio. + """ + + languages: Any = field(default_factory=lambda: NOT_GIVEN) + language_codes: Any = field(default_factory=lambda: NOT_GIVEN) + use_separate_recognition_per_channel: Any = field(default_factory=lambda: NOT_GIVEN) + enable_automatic_punctuation: Any = field(default_factory=lambda: NOT_GIVEN) + enable_spoken_punctuation: Any = field(default_factory=lambda: NOT_GIVEN) + enable_spoken_emojis: Any = field(default_factory=lambda: NOT_GIVEN) + profanity_filter: Any = field(default_factory=lambda: NOT_GIVEN) + enable_word_time_offsets: Any = field(default_factory=lambda: NOT_GIVEN) + enable_word_confidence: Any = field(default_factory=lambda: NOT_GIVEN) + enable_interim_results: Any = field(default_factory=lambda: NOT_GIVEN) + enable_voice_activity_events: Any = field(default_factory=lambda: NOT_GIVEN) + + class GoogleSTTService(STTService): """Google Cloud Speech-to-Text V2 service implementation. @@ -508,21 +549,19 @@ class GoogleSTTService(STTService): self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options) - self._settings = { - "language_codes": [ - self.language_to_service_language(lang) for lang in params.language_list - ], - "model": params.model, - "use_separate_recognition_per_channel": params.use_separate_recognition_per_channel, - "enable_automatic_punctuation": params.enable_automatic_punctuation, - "enable_spoken_punctuation": params.enable_spoken_punctuation, - "enable_spoken_emojis": params.enable_spoken_emojis, - "profanity_filter": params.profanity_filter, - "enable_word_time_offsets": params.enable_word_time_offsets, - "enable_word_confidence": params.enable_word_confidence, - "enable_interim_results": params.enable_interim_results, - "enable_voice_activity_events": params.enable_voice_activity_events, - } + self._settings = GoogleSTTSettings( + languages=list(params.language_list), + model=params.model, + use_separate_recognition_per_channel=params.use_separate_recognition_per_channel, + enable_automatic_punctuation=params.enable_automatic_punctuation, + enable_spoken_punctuation=params.enable_spoken_punctuation, + enable_spoken_emojis=params.enable_spoken_emojis, + profanity_filter=params.profanity_filter, + enable_word_time_offsets=params.enable_word_time_offsets, + enable_word_confidence=params.enable_word_confidence, + enable_interim_results=params.enable_interim_results, + enable_voice_activity_events=params.enable_voice_activity_events, + ) def can_generate_metrics(self) -> bool: """Check if the service can generate metrics. @@ -545,6 +584,23 @@ class GoogleSTTService(STTService): return [language_to_google_stt_language(lang) or "en-US" for lang in language] return language_to_google_stt_language(language) or "en-US" + def _get_language_codes(self) -> List[str]: + """Resolve the current language settings to Google STT language code strings. + + Prefers ``languages`` (``Language`` enums) over the deprecated + ``language_codes`` (raw strings). Falls back to ``["en-US"]``. + + Returns: + List[str]: Google STT language code strings. + """ + from pipecat.services.settings import is_given + + if is_given(self._settings.languages): + return [self.language_to_service_language(lang) for lang in self._settings.languages] + if is_given(self._settings.language_codes): + return list(self._settings.language_codes) + return ["en-US"] + async def _reconnect_if_needed(self): """Reconnect the stream if it's currently active.""" if self._streaming_task: @@ -552,41 +608,65 @@ class GoogleSTTService(STTService): await self._disconnect() await self._connect() - async def set_language(self, language: Language): - """Update the service's recognition language. - - A convenience method for setting a single language. - - Args: - language: New language for recognition. - """ - logger.debug(f"Switching STT language to: {language}") - await self.set_languages([language]) - async def set_languages(self, languages: List[Language]): """Update the service's recognition languages. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(languages=...)`` + instead. + Args: languages: List of languages for recognition. First language is primary. """ + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "set_languages() is deprecated. Use STTUpdateSettingsFrame with " + "GoogleSTTSettings(languages=...) instead.", + DeprecationWarning, + ) logger.debug(f"Switching STT languages to: {languages}") - self._settings["language_codes"] = [ - self.language_to_service_language(lang) for lang in languages - ] - # Recreate stream with new languages - await self._reconnect_if_needed() + await self._update_settings_from_typed(GoogleSTTSettings(languages=list(languages))) - async def set_model(self, model: str): - """Update the service's recognition model. + async def _update_settings_from_typed(self, update: GoogleSTTSettings) -> set[str]: + """Apply typed settings update and reconnect if anything changed. + + Handles ``language`` from base ``set_language`` by converting it to + ``languages``. Emits a deprecation warning if ``language_codes`` is + used. All other fields (model, boolean flags) are applied directly. + Reconnects the stream on any change. Args: - model: The new recognition model to use. + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. """ - logger.debug(f"Switching STT model to: {model}") - await super().set_model(model) - self._settings["model"] = model - # Recreate stream with new model - await self._reconnect_if_needed() + from pipecat.services.settings import is_given + + # If base set_language sent a Language value, convert to languages list + if is_given(update.language): + update.languages = [update.language] + # Clear language so the base class doesn't try to store it + update.language = NOT_GIVEN + + # Warn on deprecated language_codes usage + if is_given(update.language_codes): + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "GoogleSTTSettings.language_codes is deprecated. " + "Use GoogleSTTSettings.languages (List[Language]) instead.", + DeprecationWarning, + stacklevel=2, + ) + + changed = await super()._update_settings_from_typed(update) + + if changed: + await self._reconnect_if_needed() + + return changed async def start(self, frame: StartFrame): """Start the STT service and establish connection. @@ -632,6 +712,10 @@ class GoogleSTTService(STTService): ) -> None: """Update service options dynamically. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(...)`` + instead. + Args: languages: New list of recognition languages. model: New recognition model. @@ -649,55 +733,42 @@ class GoogleSTTService(STTService): Changes that affect the streaming configuration will cause the stream to be reconnected. """ - # Update settings with new values + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "update_options() is deprecated. Use STTUpdateSettingsFrame with " + "GoogleSTTSettings(...) instead.", + DeprecationWarning, + ) + # Build a typed settings delta from the provided options + update = GoogleSTTSettings() + if languages is not None: - logger.debug(f"Updating language to: {languages}") - self._settings["language_codes"] = [ - self.language_to_service_language(lang) for lang in languages - ] - + update.languages = list(languages) if model is not None: - logger.debug(f"Updating model to: {model}") - self._settings["model"] = model - + update.model = model if enable_automatic_punctuation is not None: - logger.debug(f"Updating automatic punctuation to: {enable_automatic_punctuation}") - self._settings["enable_automatic_punctuation"] = enable_automatic_punctuation - + update.enable_automatic_punctuation = enable_automatic_punctuation if enable_spoken_punctuation is not None: - logger.debug(f"Updating spoken punctuation to: {enable_spoken_punctuation}") - self._settings["enable_spoken_punctuation"] = enable_spoken_punctuation - + update.enable_spoken_punctuation = enable_spoken_punctuation if enable_spoken_emojis is not None: - logger.debug(f"Updating spoken emojis to: {enable_spoken_emojis}") - self._settings["enable_spoken_emojis"] = enable_spoken_emojis - + update.enable_spoken_emojis = enable_spoken_emojis if profanity_filter is not None: - logger.debug(f"Updating profanity filter to: {profanity_filter}") - self._settings["profanity_filter"] = profanity_filter - + update.profanity_filter = profanity_filter if enable_word_time_offsets is not None: - logger.debug(f"Updating word time offsets to: {enable_word_time_offsets}") - self._settings["enable_word_time_offsets"] = enable_word_time_offsets - + update.enable_word_time_offsets = enable_word_time_offsets if enable_word_confidence is not None: - logger.debug(f"Updating word confidence to: {enable_word_confidence}") - self._settings["enable_word_confidence"] = enable_word_confidence - + update.enable_word_confidence = enable_word_confidence if enable_interim_results is not None: - logger.debug(f"Updating interim results to: {enable_interim_results}") - self._settings["enable_interim_results"] = enable_interim_results - + update.enable_interim_results = enable_interim_results if enable_voice_activity_events is not None: - logger.debug(f"Updating voice activity events to: {enable_voice_activity_events}") - self._settings["enable_voice_activity_events"] = enable_voice_activity_events + update.enable_voice_activity_events = enable_voice_activity_events if location is not None: logger.debug(f"Updating location to: {location}") self._location = location - # Reconnect the stream for updates - await self._reconnect_if_needed() + await self._update_settings_from_typed(update) async def _connect(self): """Initialize streaming recognition config and stream.""" @@ -714,20 +785,20 @@ class GoogleSTTService(STTService): sample_rate_hertz=self.sample_rate, audio_channel_count=1, ), - language_codes=self._settings["language_codes"], - model=self._settings["model"], + language_codes=self._get_language_codes(), + model=self._settings.model, features=cloud_speech.RecognitionFeatures( - enable_automatic_punctuation=self._settings["enable_automatic_punctuation"], - enable_spoken_punctuation=self._settings["enable_spoken_punctuation"], - enable_spoken_emojis=self._settings["enable_spoken_emojis"], - profanity_filter=self._settings["profanity_filter"], - enable_word_time_offsets=self._settings["enable_word_time_offsets"], - enable_word_confidence=self._settings["enable_word_confidence"], + enable_automatic_punctuation=self._settings.enable_automatic_punctuation, + enable_spoken_punctuation=self._settings.enable_spoken_punctuation, + enable_spoken_emojis=self._settings.enable_spoken_emojis, + profanity_filter=self._settings.profanity_filter, + enable_word_time_offsets=self._settings.enable_word_time_offsets, + enable_word_confidence=self._settings.enable_word_confidence, ), ), streaming_features=cloud_speech.StreamingRecognitionFeatures( - enable_voice_activity_events=self._settings["enable_voice_activity_events"], - interim_results=self._settings["enable_interim_results"], + enable_voice_activity_events=self._settings.enable_voice_activity_events, + interim_results=self._settings.enable_interim_results, ), ) @@ -857,7 +928,7 @@ class GoogleSTTService(STTService): if not transcript: continue - primary_language = self._settings["language_codes"][0] + primary_language = self._get_language_codes()[0] if result.is_final: self._last_transcript_was_final = True diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 4016286df..d015571d0 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -23,7 +23,8 @@ from pipecat.utils.tracing.service_decorators import traced_tts # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" -from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, List, Literal, Optional from loguru import logger from pydantic import BaseModel @@ -36,6 +37,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language @@ -474,6 +476,63 @@ def language_to_gemini_tts_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class GoogleHttpTTSSettings(TTSSettings): + """Typed settings for Google HTTP TTS service. + + Parameters: + pitch: Voice pitch adjustment (e.g., "+2st", "-50%"). + rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). Used for + SSML prosody tags (non-Chirp voices). + speaking_rate: Speaking rate for AudioConfig (Chirp/Journey voices). + Range [0.25, 2.0]. + volume: Volume adjustment (e.g., "loud", "soft", "+6dB"). + emphasis: Emphasis level for the text. + language: Language for synthesis. Defaults to English. + gender: Voice gender preference. + google_style: Google-specific voice style. + """ + + pitch: str = field(default_factory=lambda: NOT_GIVEN) + rate: str = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float = field(default_factory=lambda: NOT_GIVEN) + volume: str = field(default_factory=lambda: NOT_GIVEN) + emphasis: str = field(default_factory=lambda: NOT_GIVEN) + language: str = field(default_factory=lambda: NOT_GIVEN) + gender: str = field(default_factory=lambda: NOT_GIVEN) + google_style: str = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class GoogleStreamTTSSettings(TTSSettings): + """Typed settings for Google streaming TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + speaking_rate: The speaking rate, in the range [0.25, 2.0]. + """ + + language: str = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class GeminiTTSSettings(TTSSettings): + """Typed settings for Gemini TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + prompt: Optional style instructions for how to synthesize the content. + multi_speaker: Whether to enable multi-speaker support. + speaker_configs: List of speaker configurations for multi-speaker mode. + """ + + language: str = field(default_factory=lambda: NOT_GIVEN) + prompt: str = field(default_factory=lambda: NOT_GIVEN) + multi_speaker: bool = field(default_factory=lambda: NOT_GIVEN) + speaker_configs: List[dict] = field(default_factory=lambda: NOT_GIVEN) + + class GoogleHttpTTSService(TTSService): """Google Cloud Text-to-Speech HTTP service with SSML support. @@ -538,19 +597,19 @@ class GoogleHttpTTSService(TTSService): params = params or GoogleHttpTTSService.InputParams() self._location = location - self._settings = { - "pitch": params.pitch, - "rate": params.rate, - "speaking_rate": params.speaking_rate, - "volume": params.volume, - "emphasis": params.emphasis, - "language": self.language_to_service_language(params.language) + self._settings: GoogleHttpTTSSettings = GoogleHttpTTSSettings( + pitch=params.pitch, + rate=params.rate, + speaking_rate=params.speaking_rate, + volume=params.volume, + emphasis=params.emphasis, + language=self.language_to_service_language(params.language) if params.language else "en-US", - "gender": params.gender, - "google_style": params.google_style, - } - self.set_voice(voice_id) + gender=params.gender, + google_style=params.google_style, + ) + self._voice_id = voice_id self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path ) @@ -619,21 +678,20 @@ class GoogleHttpTTSService(TTSService): """ return language_to_google_tts_language(language) - async def _update_settings(self, settings: Mapping[str, Any]): - """Override to handle speaking_rate updates for Chirp/Journey voices. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Override to handle speaking_rate validation. Args: - settings: Dictionary of settings to update. Can include 'speaking_rate' (float) + update: Typed settings delta. Can include 'speaking_rate' (float). """ - if "speaking_rate" in settings: - rate_value = float(settings["speaking_rate"]) - if 0.25 <= rate_value <= 2.0: - self._settings["speaking_rate"] = rate_value - else: + if isinstance(update, GoogleHttpTTSSettings) and is_given(update.speaking_rate): + rate_value = float(update.speaking_rate) + if not (0.25 <= rate_value <= 2.0): logger.warning( f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0" ) - await super()._update_settings(settings) + update.speaking_rate = NOT_GIVEN + return await super()._update_settings_from_typed(update) def _construct_ssml(self, text: str) -> str: ssml = "" @@ -641,39 +699,39 @@ class GoogleHttpTTSService(TTSService): # Voice tag voice_attrs = [f"name='{self._voice_id}'"] - language = self._settings["language"] + language = self._settings.language voice_attrs.append(f"language='{language}'") - if self._settings["gender"]: - voice_attrs.append(f"gender='{self._settings['gender']}'") + if self._settings.gender: + voice_attrs.append(f"gender='{self._settings.gender}'") ssml += f"" # Prosody tag prosody_attrs = [] - if self._settings["pitch"]: - prosody_attrs.append(f"pitch='{self._settings['pitch']}'") - if self._settings["rate"]: - prosody_attrs.append(f"rate='{self._settings['rate']}'") - if self._settings["volume"]: - prosody_attrs.append(f"volume='{self._settings['volume']}'") + if self._settings.pitch: + prosody_attrs.append(f"pitch='{self._settings.pitch}'") + if self._settings.rate: + prosody_attrs.append(f"rate='{self._settings.rate}'") + if self._settings.volume: + prosody_attrs.append(f"volume='{self._settings.volume}'") if prosody_attrs: ssml += f"" # Emphasis tag - if self._settings["emphasis"]: - ssml += f"" + if self._settings.emphasis: + ssml += f"" # Google style tag - if self._settings["google_style"]: - ssml += f"" + if self._settings.google_style: + ssml += f"" ssml += text # Close tags - if self._settings["google_style"]: + if self._settings.google_style: ssml += "" - if self._settings["emphasis"]: + if self._settings.emphasis: ssml += "" if prosody_attrs: ssml += "" @@ -710,7 +768,7 @@ class GoogleHttpTTSService(TTSService): synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], name=self._voice_id + language_code=self._settings.language, name=self._voice_id ) # Build audio config with conditional speaking_rate audio_config_params = { @@ -719,8 +777,8 @@ class GoogleHttpTTSService(TTSService): } # For Chirp and Journey voices, include speaking_rate in AudioConfig - if (is_chirp_voice or is_journey_voice) and self._settings["speaking_rate"] is not None: - audio_config_params["speaking_rate"] = self._settings["speaking_rate"] + if (is_chirp_voice or is_journey_voice) and self._settings.speaking_rate is not None: + audio_config_params["speaking_rate"] = self._settings.speaking_rate audio_config = texttospeech_v1.AudioConfig(**audio_config_params) @@ -950,33 +1008,32 @@ class GoogleTTSService(GoogleBaseTTSService): params = params or GoogleTTSService.InputParams() self._location = location - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: GoogleStreamTTSSettings = GoogleStreamTTSSettings( + language=self.language_to_service_language(params.language) if params.language else "en-US", - "speaking_rate": params.speaking_rate, - } - self.set_voice(voice_id) + speaking_rate=params.speaking_rate, + ) + self._voice_id = voice_id self._voice_cloning_key = voice_cloning_key self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path ) - async def _update_settings(self, settings: Mapping[str, Any]): - """Override to handle speaking_rate updates for streaming API. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Override to handle speaking_rate validation. Args: - settings: Dictionary of settings to update. Can include 'speaking_rate' (float) + update: Typed settings delta. Can include 'speaking_rate' (float). """ - if "speaking_rate" in settings: - rate_value = float(settings["speaking_rate"]) - if 0.25 <= rate_value <= 2.0: - self._settings["speaking_rate"] = rate_value - else: + if isinstance(update, GoogleStreamTTSSettings) and is_given(update.speaking_rate): + rate_value = float(update.speaking_rate) + if not (0.25 <= rate_value <= 2.0): logger.warning( f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0" ) - await super()._update_settings(settings) + update.speaking_rate = NOT_GIVEN + return await super()._update_settings_from_typed(update) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -1000,11 +1057,11 @@ class GoogleTTSService(GoogleBaseTTSService): voice_cloning_key=self._voice_cloning_key ) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], voice_clone=voice_clone_params + language_code=self._settings.language, voice_clone=voice_clone_params ) else: voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], name=self._voice_id + language_code=self._settings.language, name=self._voice_id ) # Create streaming config @@ -1013,7 +1070,7 @@ class GoogleTTSService(GoogleBaseTTSService): streaming_audio_config=texttospeech_v1.StreamingAudioConfig( audio_encoding=texttospeech_v1.AudioEncoding.PCM, sample_rate_hertz=self.sample_rate, - speaking_rate=self._settings["speaking_rate"], + speaking_rate=self._settings.speaking_rate, ), ) @@ -1159,14 +1216,14 @@ class GeminiTTSService(GoogleBaseTTSService): self._location = location self._model = model self._voice_id = voice_id - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: GeminiTTSSettings = GeminiTTSSettings( + language=self.language_to_service_language(params.language) if params.language else "en-US", - "prompt": params.prompt, - "multi_speaker": params.multi_speaker, - "speaker_configs": params.speaker_configs, - } + prompt=params.prompt, + multi_speaker=params.multi_speaker, + speaker_configs=params.speaker_configs, + ) self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( credentials, credentials_path @@ -1183,7 +1240,7 @@ class GeminiTTSService(GoogleBaseTTSService): """ return language_to_gemini_tts_language(language) - def set_voice(self, voice_id: str): + async def set_voice(self, voice_id: str): """Set the voice for TTS generation. Args: @@ -1206,15 +1263,13 @@ class GeminiTTSService(GoogleBaseTTSService): f"Current rate of {self.sample_rate}Hz may cause issues." ) - async def _update_settings(self, settings: Mapping[str, Any]): + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: """Override to handle prompt updates. Args: - settings: Dictionary of settings to update. Can include 'prompt' (str) + update: Typed settings delta. Can include 'prompt' (str). """ - if "prompt" in settings: - self._settings["prompt"] = settings["prompt"] - await super()._update_settings(settings) + return await super()._update_settings_from_typed(update) @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -1234,10 +1289,10 @@ class GeminiTTSService(GoogleBaseTTSService): await self.start_ttfb_metrics() # Build voice selection params - if self._settings["multi_speaker"] and self._settings["speaker_configs"]: + if self._settings.multi_speaker and self._settings.speaker_configs: # Multi-speaker mode speaker_voice_configs = [] - for speaker_config in self._settings["speaker_configs"]: + for speaker_config in self._settings.speaker_configs: speaker_voice_configs.append( texttospeech_v1.MultispeakerPrebuiltVoice( speaker_alias=speaker_config["speaker_alias"], @@ -1250,14 +1305,14 @@ class GeminiTTSService(GoogleBaseTTSService): ) voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], + language_code=self._settings.language, model_name=self._model, multi_speaker_voice_config=multi_speaker_voice_config, ) else: # Single speaker mode voice = texttospeech_v1.VoiceSelectionParams( - language_code=self._settings["language"], + language_code=self._settings.language, name=self._voice_id, model_name=self._model, ) @@ -1273,7 +1328,7 @@ class GeminiTTSService(GoogleBaseTTSService): # Use base class streaming logic with prompt support async for frame in self._stream_tts( - streaming_config, text, context_id, self._settings["prompt"] + streaming_config, text, context_id, self._settings.prompt ): yield frame diff --git a/src/pipecat/services/gradium/stt.py b/src/pipecat/services/gradium/stt.py index 7433c2549..2bad8cf30 100644 --- a/src/pipecat/services/gradium/stt.py +++ b/src/pipecat/services/gradium/stt.py @@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription. import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -27,6 +28,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import GRADIUM_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -64,6 +66,18 @@ def language_to_gradium_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class GradiumSTTSettings(STTSettings): + """Typed settings for the Gradium STT service. + + Parameters: + delay_in_frames: Delay in audio frames (80ms each) before text is + generated. Higher delays allow more context but increase latency. + """ + + delay_in_frames: int = field(default_factory=lambda: NOT_GIVEN) + + class GradiumSTTService(WebsocketSTTService): """Gradium real-time speech-to-text service. @@ -127,9 +141,15 @@ class GradiumSTTService(WebsocketSTTService): self._api_key = api_key self._api_endpoint_base_url = api_endpoint_base_url self._websocket = None - self._params = params or GradiumSTTService.InputParams() self._json_config = json_config + params = params or GradiumSTTService.InputParams() + + self._settings: GradiumSTTSettings = GradiumSTTSettings( + language=params.language, + delay_in_frames=params.delay_in_frames if params.delay_in_frames else NOT_GIVEN, + ) + self._receive_task = None self._audio_buffer = bytearray() @@ -149,16 +169,22 @@ class GradiumSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the recognition language and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, sync params, and reconnect. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``GradiumSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - logger.info(f"Switching STT language to: [{language}]") - self._params.language = language + changed = await super()._update_settings_from_typed(update) + if not changed: + return changed + await self._disconnect() await self._connect() + return changed async def start(self, frame: StartFrame): """Start the speech-to-text service. @@ -298,12 +324,12 @@ class GradiumSTTService(WebsocketSTTService): json_config = {} if self._json_config: json_config = json.loads(self._json_config) - if self._params.language: - gradium_language = language_to_gradium_language(self._params.language) + if is_given(self._settings.language) and self._settings.language: + gradium_language = language_to_gradium_language(self._settings.language) if gradium_language: json_config["language"] = gradium_language - if self._params.delay_in_frames: - json_config["delay_in_frames"] = self._params.delay_in_frames + if is_given(self._settings.delay_in_frames) and self._settings.delay_in_frames: + json_config["delay_in_frames"] = self._settings.delay_in_frames if json_config: setup_msg["json_config"] = json_config await self._websocket.send(json.dumps(setup_msg)) diff --git a/src/pipecat/services/gradium/tts.py b/src/pipecat/services/gradium/tts.py index 0e9865cf0..e129fba68 100644 --- a/src/pipecat/services/gradium/tts.py +++ b/src/pipecat/services/gradium/tts.py @@ -6,7 +6,8 @@ import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleWordTTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -37,6 +39,17 @@ except ModuleNotFoundError as e: SAMPLE_RATE = 48000 +@dataclass +class GradiumTTSSettings(TTSSettings): + """Typed settings for the Gradium TTS service. + + Parameters: + output_format: Audio output format. + """ + + output_format: str = field(default_factory=lambda: NOT_GIVEN) + + class GradiumTTSService(InterruptibleWordTTSService): """Text-to-Speech service using Gradium's websocket API.""" @@ -86,12 +99,11 @@ class GradiumTTSService(InterruptibleWordTTSService): self._url = url self._voice_id = voice_id self._json_config = json_config - self._model = model - self._settings = { - "voice_id": voice_id, - "model_name": model, - "output_format": "pcm", - } + self._settings: GradiumTTSSettings = GradiumTTSSettings( + model=model, + voice=voice_id, + output_format="pcm", + ) # State tracking self._receive_task = None @@ -105,24 +117,21 @@ class GradiumTTSService(InterruptibleWordTTSService): """ return True - async def set_model(self, model: str): - """Update the TTS model. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if voice changed. Args: - model: The model name to use for synthesis. - """ - self._model = model - await super().set_model(model) + update: A :class:`TTSSettings` (or ``GradiumTTSSettings``) delta. - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" + Returns: + Set of field names whose values actually changed. + """ prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - self._settings["voice_id"] = self._voice_id - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + changed = await super()._update_settings_from_typed(update) + if self._voice_id != prev_voice: await self._disconnect() await self._connect() + return changed def _build_msg(self, text: str = "") -> dict: """Build JSON message for Gradium API.""" diff --git a/src/pipecat/services/grok/realtime/llm.py b/src/pipecat/services/grok/realtime/llm.py index e1355ce31..7cb619a7d 100644 --- a/src/pipecat/services/grok/realtime/llm.py +++ b/src/pipecat/services/grok/realtime/llm.py @@ -13,8 +13,8 @@ https://docs.x.ai/docs/guides/voice/agent import base64 import json import time -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger @@ -56,6 +56,7 @@ from pipecat.processors.aggregators.llm_response_universal import ( from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.time import time_now_iso8601 from . import events @@ -85,6 +86,17 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class GrokRealtimeLLMSettings(LLMSettings): + """Typed settings for Grok Realtime LLM services. + + Parameters: + session_properties: Grok Realtime session configuration. + """ + + session_properties: Any = field(default_factory=lambda: NOT_GIVEN) + + class GrokRealtimeLLMService(LLMService): """Grok Realtime Voice Agent LLM service providing real-time audio and text communication. @@ -134,9 +146,8 @@ class GrokRealtimeLLMService(LLMService): self.api_key = api_key self.base_url = base_url - # Initialize session_properties - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = GrokRealtimeLLMSettings( + session_properties=session_properties or events.SessionProperties(), ) self._audio_input_paused = start_audio_paused @@ -186,13 +197,13 @@ class GrokRealtimeLLMService(LLMService): Configured sample rate or None if not manually configured. For PCMU/PCMA formats, returns 8000 Hz (G.711 standard). """ - if not self._session_properties.audio: + if not self._settings.session_properties.audio: return None audio_config = ( - self._session_properties.audio.input + self._settings.session_properties.audio.input if direction == "input" - else self._session_properties.audio.output + else self._settings.session_properties.audio.output ) if audio_config and audio_config.format: @@ -222,8 +233,8 @@ class GrokRealtimeLLMService(LLMService): def _is_turn_detection_enabled(self) -> bool: """Check if server-side VAD is enabled.""" - if self._session_properties.turn_detection: - return self._session_properties.turn_detection.type == "server_vad" + if self._settings.session_properties.turn_detection: + return self._settings.session_properties.turn_detection.type == "server_vad" return False async def _handle_interruption(self): @@ -290,18 +301,18 @@ class GrokRealtimeLLMService(LLMService): await super().start(frame) # Ensure audio configuration exists with both input and output - if not self._session_properties.audio: - self._session_properties.audio = events.AudioConfiguration() + if not self._settings.session_properties.audio: + self._settings.session_properties.audio = events.AudioConfiguration() # Fill in missing input configuration - if not self._session_properties.audio.input: - self._session_properties.audio.input = events.AudioInput( + if not self._settings.session_properties.audio.input: + self._settings.session_properties.audio.input = events.AudioInput( format=events.PCMAudioFormat(rate=frame.audio_in_sample_rate) ) # Fill in missing output configuration - if not self._session_properties.audio.output: - self._session_properties.audio.output = events.AudioOutput( + if not self._settings.session_properties.audio.output: + self._settings.session_properties.audio.output = events.AudioOutput( format=events.PCMAudioFormat(rate=frame.audio_out_sample_rate) ) @@ -336,6 +347,16 @@ class GrokRealtimeLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Legacy dict path: frame.settings contains SessionProperties fields, + # not our Settings fields, so we construct SessionProperties directly. + # The new typed path (frame.update) falls through to super, which calls + # _update_settings_from_typed → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._update_settings() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -355,9 +376,6 @@ class GrokRealtimeLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() @@ -436,9 +454,16 @@ class GrokRealtimeLLMService(LLMService): return await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) + async def _update_settings_from_typed(self, update): + """Apply a typed settings update, sending a session update if needed.""" + changed = await super()._update_settings_from_typed(update) + if "session_properties" in changed: + await self._update_settings() + return changed + async def _update_settings(self): """Update session settings on the server.""" - settings = self._session_properties + settings = self._settings.session_properties adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter() if self._context: diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 331af8eb7..678a2426d 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -8,6 +8,7 @@ import io import wave +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -20,6 +21,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language from pipecat.utils.tracing.service_decorators import traced_tts @@ -32,6 +34,21 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class GroqTTSSettings(TTSSettings): + """Typed settings for the Groq TTS service. + + Parameters: + output_format: Audio output format. + speed: Speech speed multiplier. Defaults to 1.0. + groq_sample_rate: Audio sample rate. + """ + + output_format: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + groq_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class GroqTTSService(TTSService): """Groq text-to-speech service implementation. @@ -92,14 +109,14 @@ class GroqTTSService(TTSService): self._voice_id = voice_id self._params = params - self._settings = { - "model": model_name, - "voice_id": voice_id, - "output_format": output_format, - "language": str(params.language) if params.language else "en", - "speed": params.speed, - "sample_rate": sample_rate, - } + self._settings: GroqTTSSettings = GroqTTSSettings( + model=model_name, + voice=voice_id, + language=str(params.language) if params.language else "en", + output_format=output_format, + speed=params.speed, + groq_sample_rate=sample_rate, + ) self._client = AsyncGroq(api_key=self._api_key) diff --git a/src/pipecat/services/hathora/stt.py b/src/pipecat/services/hathora/stt.py index defdc355d..b0e3beead 100644 --- a/src/pipecat/services/hathora/stt.py +++ b/src/pipecat/services/hathora/stt.py @@ -8,6 +8,7 @@ import base64 import os +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -18,6 +19,7 @@ from pipecat.frames.frames import ( Frame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import HATHORA_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language @@ -27,6 +29,19 @@ from pipecat.utils.tracing.service_decorators import traced_stt from .utils import ConfigOption +@dataclass +class HathoraSTTSettings(STTSettings): + """Typed settings for the Hathora STT service. + + Parameters: + config: Some models support additional config, refer to + `docs `_ for each model to see + what is supported. + """ + + config: Optional[list] = field(default_factory=lambda: NOT_GIVEN) + + class HathoraSTTService(SegmentedSTTService): """This service supports several different speech-to-text models hosted by Hathora. @@ -83,10 +98,11 @@ class HathoraSTTService(SegmentedSTTService): params = params or HathoraSTTService.InputParams() - self._settings = { - "language": params.language, - "config": params.config, - } + self._settings: HathoraSTTSettings = HathoraSTTSettings( + model=model, + language=params.language, + config=params.config, + ) self.set_model_name(model) @@ -123,12 +139,11 @@ class HathoraSTTService(SegmentedSTTService): "model": self._model, } - if self._settings["language"] is not None: - payload["language"] = self._settings["language"] - if self._settings["config"] is not None: + if self._settings.language is not None: + payload["language"] = self._settings.language + if self._settings.config is not None: payload["model_config"] = [ - {"name": option.name, "value": option.value} - for option in self._settings["config"] + {"name": option.name, "value": option.value} for option in self._settings.config ] base64_audio = base64.b64encode(audio).decode("utf-8") @@ -147,7 +162,7 @@ class HathoraSTTService(SegmentedSTTService): if text: # Only yield non-empty text # Hathora's API currently doesn't return language info # so we default to the requested language or "en" - response_language = self._settings["language"] or "en" + response_language = self._settings.language or "en" await self._handle_transcription(text, True, response_language) yield TranscriptionFrame( text, diff --git a/src/pipecat/services/hathora/tts.py b/src/pipecat/services/hathora/tts.py index 80cbd4fe8..b821b1e05 100644 --- a/src/pipecat/services/hathora/tts.py +++ b/src/pipecat/services/hathora/tts.py @@ -9,6 +9,7 @@ import io import os import wave +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional, Tuple import aiohttp @@ -21,6 +22,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -45,6 +47,21 @@ def _decode_audio_payload( return audio_bytes, fallback_sample_rate, fallback_channels +@dataclass +class HathoraTTSSettings(TTSSettings): + """Typed settings for Hathora TTS service. + + Parameters: + speed: Speech speed multiplier (if supported by model). + config: Some models support additional config, refer to + [docs](https://models.hathora.dev) for each model to see + what is supported. + """ + + speed: float = field(default_factory=lambda: NOT_GIVEN) + config: list = field(default_factory=lambda: NOT_GIVEN) + + class HathoraTTSService(TTSService): """This service supports several different text-to-speech models hosted by Hathora. @@ -98,13 +115,15 @@ class HathoraTTSService(TTSService): params = params or HathoraTTSService.InputParams() - self._settings = { - "speed": params.speed, - "config": params.config, - } + self._settings: HathoraTTSSettings = HathoraTTSSettings( + model=model, + voice=voice_id, + speed=params.speed, + config=params.config, + ) self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -135,12 +154,11 @@ class HathoraTTSService(TTSService): if self._voice_id is not None: payload["voice"] = self._voice_id - if self._settings["speed"] is not None: - payload["speed"] = self._settings["speed"] - if self._settings["config"] is not None: + if self._settings.speed is not None: + payload["speed"] = self._settings.speed + if self._settings.config is not None: payload["model_config"] = [ - {"name": option.name, "value": option.value} - for option in self._settings["config"] + {"name": option.name, "value": option.value} for option in self._settings.config ] yield TTSStartedFrame(context_id=context_id) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 2d98e1f8c..3b45cc249 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -117,7 +117,7 @@ class HumeTTSService(WordTTSService): self._params = params or HumeTTSService.InputParams() # Store voice in the base class (mirrors other services) - self.set_voice(voice_id) + self._voice_id = voice_id self._audio_bytes = b"" @@ -196,7 +196,7 @@ class HumeTTSService(WordTTSService): key_l = (key or "").lower() if key_l == "voice_id": - self.set_voice(str(value)) + await self.set_voice(str(value)) logger.debug(f"HumeTTSService voice_id set to: {self.voice}") elif key_l == "description": self._params.description = None if value is None else str(value) diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index 2ea94399b..68c140187 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -16,6 +16,7 @@ Inworld’s text-to-speech (TTS) models offer ultra-realistic, context-aware spe import asyncio import base64 import json +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple import aiohttp @@ -23,6 +24,8 @@ import websockets from loguru import logger from pydantic import BaseModel +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given + try: from websockets.asyncio.client import connect as websocket_connect from websockets.protocol import State @@ -47,6 +50,31 @@ from pipecat.services.tts_service import AudioContextWordTTSService, WordTTSServ from pipecat.utils.tracing.service_decorators import traced_tts +@dataclass +class InworldTTSSettings(TTSSettings): + """Typed settings for Inworld TTS services. + + Parameters: + audio_encoding: Audio encoding format (e.g. LINEAR16). + audio_sample_rate: Audio sample rate in Hz. + speaking_rate: Speaking rate for speech synthesis. + temperature: Temperature for speech synthesis. + auto_mode: Whether to use auto mode. Recommended when texts are sent + in full sentences/phrases. When enabled, the server controls + flushing of buffered text to achieve minimal latency while + maintaining high quality audio output. If None (default), + automatically set based on aggregate_sentences. + apply_text_normalization: Whether to apply text normalization. + """ + + audio_encoding: str = field(default_factory=lambda: NOT_GIVEN) + audio_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + speaking_rate: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + auto_mode: bool = field(default_factory=lambda: NOT_GIVEN) + apply_text_normalization: str = field(default_factory=lambda: NOT_GIVEN) + + class InworldHttpTTSService(WordTTSService): """Inworld AI HTTP-based TTS service. @@ -110,23 +138,21 @@ class InworldHttpTTSService(WordTTSService): else: self._base_url = "https://api.inworld.ai/tts/v1/voice" - self._settings = { - "voiceId": voice_id, - "modelId": model, - "audioConfig": { - "audioEncoding": encoding, - "sampleRateHertz": 0, - }, - } + self._settings: InworldTTSSettings = InworldTTSSettings( + model=model, + voice=voice_id, + audio_encoding=encoding, + audio_sample_rate=0, + ) if params.temperature is not None: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature if params.speaking_rate is not None: - self._settings["audioConfig"]["speakingRate"] = params.speaking_rate + self._settings.speaking_rate = params.speaking_rate self._cumulative_time = 0.0 - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) def can_generate_metrics(self) -> bool: @@ -144,7 +170,7 @@ class InworldHttpTTSService(WordTTSService): frame: The start frame. """ await super().start(frame) - self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate async def stop(self, frame: EndFrame): """Stop the Inworld TTS service. @@ -223,15 +249,22 @@ class InworldHttpTTSService(WordTTSService): """ logger.debug(f"{self}: Generating TTS [{text}] (streaming={self._streaming})") + audio_config = { + "audioEncoding": self._settings.audio_encoding, + "sampleRateHertz": self._settings.audio_sample_rate, + } + if is_given(self._settings.speaking_rate): + audio_config["speakingRate"] = self._settings.speaking_rate + payload = { "text": text, - "voiceId": self._settings["voiceId"], - "modelId": self._settings["modelId"], - "audioConfig": self._settings["audioConfig"], + "voiceId": self._settings.voice, + "modelId": self._settings.model, + "audioConfig": audio_config, } - if "temperature" in self._settings: - payload["temperature"] = self._settings["temperature"] + if is_given(self._settings.temperature): + payload["temperature"] = self._settings.temperature # Use WORD timestamps for simplicity and correct spacing/capitalization payload["timestampType"] = self._timestamp_type @@ -470,27 +503,25 @@ class InworldTTSService(AudioContextWordTTSService): self._api_key = api_key self._url = url - self._settings: Dict[str, Any] = { - "voiceId": voice_id, - "modelId": model, - "audioConfig": { - "audioEncoding": encoding, - "sampleRateHertz": 0, - }, - } + self._settings: InworldTTSSettings = InworldTTSSettings( + model=model, + voice=voice_id, + audio_encoding=encoding, + audio_sample_rate=0, + ) self._timestamp_type = "WORD" if params.temperature is not None: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature if params.speaking_rate is not None: - self._settings["audioConfig"]["speakingRate"] = params.speaking_rate + self._settings.speaking_rate = params.speaking_rate if params.apply_text_normalization is not None: - self._settings["applyTextNormalization"] = params.apply_text_normalization + self._settings.apply_text_normalization = params.apply_text_normalization if params.auto_mode is not None: - self._settings["autoMode"] = params.auto_mode + self._settings.auto_mode = params.auto_mode else: - self._settings["autoMode"] = aggregate_sentences + self._settings.auto_mode = aggregate_sentences self._buffer_settings = { "maxBufferDelayMs": params.max_buffer_delay_ms, @@ -509,7 +540,7 @@ class InworldTTSService(AudioContextWordTTSService): # Track the end time of the last word in the current generation self._generation_end_time = 0.0 - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) def can_generate_metrics(self) -> bool: @@ -527,7 +558,7 @@ class InworldTTSService(AudioContextWordTTSService): frame: The start frame. """ await super().start(frame) - self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -859,18 +890,25 @@ class InworldTTSService(AudioContextWordTTSService): Args: context_id: The context ID. """ + audio_config = { + "audioEncoding": self._settings.audio_encoding, + "sampleRateHertz": self._settings.audio_sample_rate, + } + if is_given(self._settings.speaking_rate): + audio_config["speakingRate"] = self._settings.speaking_rate + create_config: Dict[str, Any] = { - "voiceId": self._settings["voiceId"], - "modelId": self._settings["modelId"], - "audioConfig": self._settings["audioConfig"], + "voiceId": self._settings.voice, + "modelId": self._settings.model, + "audioConfig": audio_config, } - if "temperature" in self._settings: - create_config["temperature"] = self._settings["temperature"] - if "applyTextNormalization" in self._settings: - create_config["applyTextNormalization"] = self._settings["applyTextNormalization"] - if "autoMode" in self._settings: - create_config["autoMode"] = self._settings["autoMode"] + if is_given(self._settings.temperature): + create_config["temperature"] = self._settings.temperature + if is_given(self._settings.apply_text_normalization): + create_config["applyTextNormalization"] = self._settings.apply_text_normalization + if is_given(self._settings.auto_mode): + create_config["autoMode"] = self._settings.auto_mode # Set buffer settings for timely audio generation. # Use provided values or defaults that work well for streaming LLM output. diff --git a/src/pipecat/services/kokoro/tts.py b/src/pipecat/services/kokoro/tts.py index 49ede2409..242446de9 100644 --- a/src/pipecat/services/kokoro/tts.py +++ b/src/pipecat/services/kokoro/tts.py @@ -7,6 +7,7 @@ """Kokoro TTS service implementation using kokoro-onnx.""" import os +from dataclasses import dataclass, field from pathlib import Path from typing import AsyncGenerator, Optional @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -87,6 +89,17 @@ def language_to_kokoro_language(language: Language) -> str: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class KokoroTTSSettings(TTSSettings): + """Typed settings for the Kokoro TTS service. + + Parameters: + lang_code: Kokoro language code for synthesis. + """ + + lang_code: str = field(default_factory=lambda: NOT_GIVEN) + + class KokoroTTSService(TTSService): """Kokoro TTS service implementation. @@ -129,6 +142,12 @@ class KokoroTTSService(TTSService): self._voice_id = voice_id self._lang_code = language_to_kokoro_language(params.language) + self._settings: KokoroTTSSettings = KokoroTTSSettings( + voice=voice_id, + language=language_to_kokoro_language(params.language), + lang_code=language_to_kokoro_language(params.language), + ) + model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx" voices = Path(voices_path) if voices_path else KOKORO_CACHE_DIR / "voices-v1.0.bin" diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index af7e691b0..77af50f15 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -44,6 +44,7 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, + LLMUpdateSettingsFrame, StartFrame, UserImageRequestFrame, ) @@ -58,6 +59,7 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import ServiceSettings from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin from pipecat.utils.context.llm_context_summarization import ( LLMContextSummarizationUtil, @@ -351,6 +353,17 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): await self._handle_interruptions(frame) elif isinstance(frame, LLMConfigureOutputFrame): self._skip_tts = frame.skip_tts + elif isinstance(frame, LLMUpdateSettingsFrame): + # New path: typed settings update object. + if frame.update is not None: + await self._update_settings_from_typed(frame.update) + # Legacy path: plain dict, but service uses typed settings — convert. + elif isinstance(self._settings, ServiceSettings): + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings_from_typed(update) + # Legacy path: plain dict, service still uses dict-based settings. + else: + await self._update_settings(frame.settings) elif isinstance(frame, LLMContextSummaryRequestFrame): await self._handle_summary_request(frame) diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index 4c34e28d5..97569fa1d 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -7,6 +7,7 @@ """LMNT text-to-speech service implementation.""" import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleTTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -71,6 +73,17 @@ def language_to_lmnt_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class LmntTTSSettings(TTSSettings): + """Typed settings for LMNT TTS service. + + Parameters: + format: Audio output format. Defaults to "raw". + """ + + format: str = field(default_factory=lambda: NOT_GIVEN) + + class LmntTTSService(InterruptibleTTSService): """LMNT real-time text-to-speech service. @@ -107,12 +120,14 @@ class LmntTTSService(InterruptibleTTSService): ) self._api_key = api_key - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) - self._settings = { - "language": self.language_to_service_language(language), - "format": "raw", # Use raw format for direct PCM data - } + self._settings: LmntTTSSettings = LmntTTSSettings( + model=model, + voice=voice_id, + language=self.language_to_service_language(language), + format="raw", + ) self._receive_task = None self._context_id: Optional[str] = None @@ -202,9 +217,9 @@ class LmntTTSService(InterruptibleTTSService): init_msg = { "X-API-Key": self._api_key, "voice": self._voice_id, - "format": self._settings["format"], + "format": self._settings.format, "sample_rate": self.sample_rate, - "language": self._settings["language"], + "language": self._settings.language, "model": self.model_name, } diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 7284d9630..6ce3e4b45 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -11,6 +11,7 @@ for streaming text-to-speech synthesis. """ import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -25,6 +26,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -85,6 +87,40 @@ def language_to_minimax_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class MiniMaxTTSSettings(TTSSettings): + """Typed settings for MiniMax TTS service. + + Parameters: + stream: Whether to use streaming mode. + speed: Speech speed (range: 0.5 to 2.0). + volume: Speech volume (range: 0 to 10). + pitch: Pitch adjustment (range: -12 to 12). + emotion: Emotional tone (options: "happy", "sad", "angry", "fearful", + "disgusted", "surprised", "calm", "fluent"). + text_normalization: Enable text normalization (Chinese/English). + latex_read: Enable LaTeX formula reading. + audio_bitrate: Audio bitrate in bps. + audio_format: Audio output format. + audio_channel: Number of audio channels. + audio_sample_rate: Audio sample rate in Hz. + language_boost: Language boost string for multilingual support. + """ + + stream: bool = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + volume: float = field(default_factory=lambda: NOT_GIVEN) + pitch: int = field(default_factory=lambda: NOT_GIVEN) + emotion: str = field(default_factory=lambda: NOT_GIVEN) + text_normalization: bool = field(default_factory=lambda: NOT_GIVEN) + latex_read: bool = field(default_factory=lambda: NOT_GIVEN) + audio_bitrate: int = field(default_factory=lambda: NOT_GIVEN) + audio_format: str = field(default_factory=lambda: NOT_GIVEN) + audio_channel: int = field(default_factory=lambda: NOT_GIVEN) + audio_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + language_boost: str = field(default_factory=lambda: NOT_GIVEN) + + class MiniMaxHttpTTSService(TTSService): """Text-to-speech service using MiniMax's T2A (Text-to-Audio) API. @@ -172,29 +208,27 @@ class MiniMaxHttpTTSService(TTSService): self._voice_id = voice_id # Create voice settings - self._settings = { - "stream": True, - "voice_setting": { - "speed": params.speed, - "vol": params.volume, - "pitch": params.pitch, - }, - "audio_setting": { - "bitrate": 128000, - "format": "pcm", - "channel": 1, - }, - } + self._settings: MiniMaxTTSSettings = MiniMaxTTSSettings( + model=model, + voice=voice_id, + stream=True, + speed=params.speed, + volume=params.volume, + pitch=params.pitch, + audio_bitrate=128000, + audio_format="pcm", + audio_channel=1, + ) # Set voice and model - self.set_voice(voice_id) + self._voice_id = voice_id self.set_model_name(model) # Add language boost if provided if params.language: service_lang = self.language_to_service_language(params.language) if service_lang: - self._settings["language_boost"] = service_lang + self._settings.language_boost = service_lang # Add optional emotion if provided if params.emotion: @@ -210,7 +244,7 @@ class MiniMaxHttpTTSService(TTSService): "fluent", ] if params.emotion in supported_emotions: - self._settings["voice_setting"]["emotion"] = params.emotion + self._settings.emotion = params.emotion else: logger.warning( f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}" @@ -226,15 +260,15 @@ class MiniMaxHttpTTSService(TTSService): "Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.", DeprecationWarning, ) - self._settings["voice_setting"]["text_normalization"] = params.english_normalization + self._settings.text_normalization = params.english_normalization # Add text_normalization if provided (corrected parameter name) if params.text_normalization is not None: - self._settings["voice_setting"]["text_normalization"] = params.text_normalization + self._settings.text_normalization = params.text_normalization # Add latex_read if provided if params.latex_read is not None: - self._settings["voice_setting"]["latex_read"] = params.latex_read + self._settings.latex_read = params.latex_read def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -263,16 +297,6 @@ class MiniMaxHttpTTSService(TTSService): """ self._model_name = model - def set_voice(self, voice: str): - """Set the voice to use. - - Args: - voice: The voice identifier to use for synthesis. - """ - self._voice_id = voice - if "voice_setting" in self._settings: - self._settings["voice_setting"]["voice_id"] = voice - async def start(self, frame: StartFrame): """Start the MiniMax TTS service. @@ -280,7 +304,7 @@ class MiniMaxHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["audio_setting"]["sample_rate"] = self.sample_rate + self._settings.audio_sample_rate = self.sample_rate logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}") @traced_tts @@ -302,10 +326,38 @@ class MiniMaxHttpTTSService(TTSService): "Authorization": f"Bearer {self._api_key}", } + # Build voice_setting dict for API + voice_setting = { + "voice_id": self._voice_id, + "speed": self._settings.speed, + "vol": self._settings.volume, + "pitch": self._settings.pitch, + } + if is_given(self._settings.emotion): + voice_setting["emotion"] = self._settings.emotion + if is_given(self._settings.text_normalization): + voice_setting["text_normalization"] = self._settings.text_normalization + if is_given(self._settings.latex_read): + voice_setting["latex_read"] = self._settings.latex_read + + # Build audio_setting dict for API + audio_setting = { + "bitrate": self._settings.audio_bitrate, + "format": self._settings.audio_format, + "channel": self._settings.audio_channel, + "sample_rate": self._settings.audio_sample_rate, + } + # Create payload from settings - payload = self._settings.copy() - payload["model"] = self._model_name - payload["text"] = text + payload = { + "stream": self._settings.stream, + "voice_setting": voice_setting, + "audio_setting": audio_setting, + "model": self._model_name, + "text": text, + } + if is_given(self._settings.language_boost): + payload["language_boost"] = self._settings.language_boost try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/mistral/llm.py b/src/pipecat/services/mistral/llm.py index 54361ef28..7a8f5b71a 100644 --- a/src/pipecat/services/mistral/llm.py +++ b/src/pipecat/services/mistral/llm.py @@ -185,19 +185,19 @@ class MistralLLMService(OpenAILLMService): "messages": fixed_messages, "tools": params_from_context["tools"], "tool_choice": params_from_context["tool_choice"], - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, } # Handle Mistral-specific parameter mapping # Mistral uses "random_seed" instead of "seed" - if self._settings["seed"]: - params["random_seed"] = self._settings["seed"] + if self._settings.seed: + params["random_seed"] = self._settings.seed # Add any extra parameters - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 24eb05bd3..b7019e6d6 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -13,7 +13,8 @@ text-to-speech API for real-time audio synthesis. import asyncio import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Optional import aiohttp from loguru import logger @@ -34,6 +35,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -72,6 +74,23 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class NeuphonicTTSSettings(TTSSettings): + """Typed settings for Neuphonic TTS service. + + Parameters: + lang_code: Neuphonic language code. + speed: Speech speed multiplier. Defaults to 1.0. + encoding: Audio encoding format. + sampling_rate: Audio sample rate. + """ + + lang_code: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + encoding: str = field(default_factory=lambda: NOT_GIVEN) + sampling_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class NeuphonicTTSService(InterruptibleTTSService): """Neuphonic real-time text-to-speech service using WebSocket streaming. @@ -127,13 +146,13 @@ class NeuphonicTTSService(InterruptibleTTSService): self._api_key = api_key self._url = url - self._settings = { - "lang_code": self.language_to_service_language(params.language), - "speed": params.speed, - "encoding": encoding, - "sampling_rate": sample_rate, - } - self.set_voice(voice_id) + self._settings: NeuphonicTTSSettings = NeuphonicTTSSettings( + lang_code=self.language_to_service_language(params.language), + speed=params.speed, + encoding=encoding, + sampling_rate=sample_rate, + ) + self._voice_id = voice_id self._cumulative_time = 0 @@ -160,15 +179,14 @@ class NeuphonicTTSService(InterruptibleTTSService): """ return language_to_neuphonic_lang_code(language) - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect with new configuration.""" - if "voice_id" in settings: - self.set_voice(settings["voice_id"]) - - await super()._update_settings(settings) - await self._disconnect() - await self._connect() - logger.info(f"Switching TTS to settings: [{self._settings}]") + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect with new configuration.""" + changed = await super()._update_settings_from_typed(update) + if changed: + await self._disconnect() + await self._connect() + logger.info(f"Switching TTS to settings: [{self._settings}]") + return changed async def start(self, frame: StartFrame): """Start the Neuphonic TTS service. @@ -266,7 +284,10 @@ class NeuphonicTTSService(InterruptibleTTSService): logger.debug("Connecting to Neuphonic") tts_config = { - **self._settings, + "lang_code": self._settings.lang_code, + "speed": self._settings.speed, + "encoding": self._settings.encoding, + "sampling_rate": self._settings.sampling_rate, "voice_id": self._voice_id, } @@ -275,7 +296,7 @@ class NeuphonicTTSService(InterruptibleTTSService): if value is not None: query_params.append(f"{key}={value}") - url = f"{self._url}/speak/{self._settings['lang_code']}" + url = f"{self._url}/speak/{self._settings.lang_code}" if query_params: url += f"?{'&'.join(query_params)}" @@ -429,7 +450,7 @@ class NeuphonicHttpTTSService(TTSService): self._lang_code = self.language_to_service_language(params.language) or "en" self._speed = params.speed self._encoding = encoding - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. diff --git a/src/pipecat/services/nvidia/stt.py b/src/pipecat/services/nvidia/stt.py index 8eb6d7bb5..c65d6da62 100644 --- a/src/pipecat/services/nvidia/stt.py +++ b/src/pipecat/services/nvidia/stt.py @@ -8,6 +8,7 @@ import asyncio from concurrent.futures import CancelledError as FuturesCancelledError +from dataclasses import dataclass, field from typing import AsyncGenerator, List, Mapping, Optional from loguru import logger @@ -22,6 +23,7 @@ from pipecat.frames.frames import ( StartFrame, TranscriptionFrame, ) +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import NVIDIA_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService, STTService from pipecat.transcriptions.language import Language, resolve_language @@ -89,6 +91,32 @@ def language_to_nvidia_riva_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class NvidiaSTTSettings(STTSettings): + """Typed settings for the NVIDIA Riva streaming STT service.""" + + pass + + +@dataclass +class NvidiaSegmentedSTTSettings(STTSettings): + """Typed settings for the NVIDIA Riva segmented STT service. + + Parameters: + profanity_filter: Whether to filter profanity from results. + automatic_punctuation: Whether to add automatic punctuation. + verbatim_transcripts: Whether to return verbatim transcripts. + boosted_lm_words: List of words to boost in language model. + boosted_lm_score: Score boost for specified words. + """ + + profanity_filter: bool = field(default_factory=lambda: NOT_GIVEN) + automatic_punctuation: bool = field(default_factory=lambda: NOT_GIVEN) + verbatim_transcripts: bool = field(default_factory=lambda: NOT_GIVEN) + boosted_lm_words: Optional[List[str]] = field(default_factory=lambda: NOT_GIVEN) + boosted_lm_score: float = field(default_factory=lambda: NOT_GIVEN) + + class NvidiaSTTService(STTService): """Real-time speech-to-text service using NVIDIA Riva streaming ASR. @@ -141,12 +169,6 @@ class NvidiaSTTService(STTService): self._server = server self._api_key = api_key self._use_ssl = use_ssl - self._profanity_filter = False - self._automatic_punctuation = True - self._no_verbatim_transcripts = False - self._language_code = params.language - self._boosted_lm_words = None - self._boosted_lm_score = 4.0 self._start_history = -1 self._start_threshold = -1.0 self._stop_history = -1 @@ -156,14 +178,9 @@ class NvidiaSTTService(STTService): self._custom_configuration = "" self._function_id = model_function_map.get("function_id") - self._settings = { - "language": str(params.language), - "profanity_filter": self._profanity_filter, - "automatic_punctuation": self._automatic_punctuation, - "verbatim_transcripts": not self._no_verbatim_transcripts, - "boosted_lm_words": self._boosted_lm_words, - "boosted_lm_score": self._boosted_lm_score, - } + self._settings: NvidiaSTTSettings = NvidiaSTTSettings( + language=params.language, + ) self.set_model_name(model_function_map.get("model_name")) @@ -186,22 +203,18 @@ class NvidiaSTTService(STTService): config = riva.client.StreamingRecognitionConfig( config=riva.client.RecognitionConfig( encoding=riva.client.AudioEncoding.LINEAR_PCM, - language_code=self._language_code, + language_code=self._settings.language, model="", max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=not self._no_verbatim_transcripts, + profanity_filter=False, + enable_automatic_punctuation=True, + verbatim_transcripts=True, sample_rate_hertz=self.sample_rate, audio_channel_count=1, ), interim_results=True, ) - riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score - ) - riva.client.add_endpoint_parameters_to_config( config, self._start_history, @@ -318,14 +331,14 @@ class NvidiaSTTService(STTService): transcript, self._user_id, time_now_iso8601(), - self._language_code, + self._settings.language, result=result, ) ) await self._handle_transcription( transcript=transcript, is_final=result.is_final, - language=self._language_code, + language=self._settings.language, ) else: await self.push_frame( @@ -333,7 +346,7 @@ class NvidiaSTTService(STTService): transcript, self._user_id, time_now_iso8601(), - self._language_code, + self._settings.language, result=result, ) ) @@ -445,18 +458,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): self._server = server self._use_ssl = use_ssl self._function_id = model_function_map.get("function_id") - self._model_name = model_function_map.get("model_name") - - # Store the language as a Language enum and as a string - self._language_enum = params.language or Language.EN_US - self._language = self.language_to_service_language(self._language_enum) or "en-US" - - # Configure transcription parameters - self._profanity_filter = params.profanity_filter - self._automatic_punctuation = params.automatic_punctuation - self._verbatim_transcripts = params.verbatim_transcripts - self._boosted_lm_words = params.boosted_lm_words - self._boosted_lm_score = params.boosted_lm_score # Voice activity detection thresholds (use NVIDIA Riva defaults) self._start_history = -1 @@ -467,10 +468,16 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): self._stop_threshold_eou = -1.0 self._custom_configuration = "" - # Create NVIDIA Riva client self._config = None self._asr_service = None - self._settings = {"language": self._language_enum} + self._settings: NvidiaSegmentedSTTSettings = NvidiaSegmentedSTTSettings( + language=params.language or Language.EN_US, + profanity_filter=params.profanity_filter, + automatic_punctuation=params.automatic_punctuation, + verbatim_transcripts=params.verbatim_transcripts, + boosted_lm_words=params.boosted_lm_words, + boosted_lm_score=params.boosted_lm_score, + ) def language_to_service_language(self, language: Language) -> Optional[str]: """Convert pipecat Language enum to NVIDIA Riva's language code. @@ -498,21 +505,25 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): auth = riva.client.Auth(None, self._use_ssl, self._server, metadata) self._asr_service = riva.client.ASRService(auth) + def _get_language_code(self) -> str: + """Resolve the current language enum to an NVIDIA Riva language code string.""" + return self.language_to_service_language(self._settings.language) or "en-US" + def _create_recognition_config(self): """Create the NVIDIA Riva ASR recognition configuration.""" # Create base configuration config = riva.client.RecognitionConfig( - language_code=self._language, # Now using the string, not a tuple + language_code=self._get_language_code(), max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=self._verbatim_transcripts, + profanity_filter=self._settings.profanity_filter, + enable_automatic_punctuation=self._settings.automatic_punctuation, + verbatim_transcripts=self._settings.verbatim_transcripts, ) # Add word boosting if specified - if self._boosted_lm_words: + if self._settings.boosted_lm_words: riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score + config, self._settings.boosted_lm_words, self._settings.boosted_lm_score ) # Add voice activity detection parameters @@ -567,20 +578,21 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): self._config = self._create_recognition_config() logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}") - async def set_language(self, language: Language): - """Set the language for the STT service. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and sync internal state. Args: - language: Target language for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._language_enum = language - self._language = self.language_to_service_language(language) or "en-US" - self._settings["language"] = language + update: A :class:`STTSettings` (or ``NvidiaSegmentedSTTSettings``) delta. - # Update configuration with new language - if self._config: - self._config.language_code = self._language + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if changed: + self._config = self._create_recognition_config() + + return changed @traced_stt async def _handle_transcription( @@ -633,11 +645,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService): text, self._user_id, time_now_iso8601(), - self._language_enum, + self._settings.language, ) transcription_found = True - await self._handle_transcription(text, True, self._language_enum) + await self._handle_transcription(text, True, self._settings.language) if not transcription_found: logger.debug(f"{self}: No transcription results found in NVIDIA Riva response") diff --git a/src/pipecat/services/nvidia/tts.py b/src/pipecat/services/nvidia/tts.py index 6bac54e3a..8a018d6aa 100644 --- a/src/pipecat/services/nvidia/tts.py +++ b/src/pipecat/services/nvidia/tts.py @@ -100,7 +100,7 @@ class NvidiaTTSService(TTSService): self._function_id = model_function_map.get("function_id") self._use_ssl = use_ssl self.set_model_name(model_function_map.get("model_name")) - self.set_voice(voice_id) + self._voice_id = voice_id self._service = None self._config = None diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 2cdde51ea..2ac53794c 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -10,7 +10,8 @@ import asyncio import base64 import json from contextlib import asynccontextmanager -from typing import Any, Dict, List, Mapping, Optional +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Mapping, Optional import httpx from loguru import logger @@ -32,7 +33,6 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, - LLMUpdateSettingsFrame, ) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext @@ -42,9 +42,24 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN +from pipecat.services.settings import LLMSettings from pipecat.utils.tracing.service_decorators import traced_llm +@dataclass +class OpenAILLMSettings(LLMSettings): + """Typed settings for OpenAI-compatible LLM services. + + Parameters: + max_completion_tokens: Maximum completion tokens to generate. + service_tier: Service tier to use (e.g., "auto", "flex", "priority"). + """ + + max_completion_tokens: Any = field(default_factory=lambda: _NOT_GIVEN) + service_tier: Any = field(default_factory=lambda: _NOT_GIVEN) + + class BaseOpenAILLMService(LLMService): """Base class for all services that use the AsyncOpenAI client. @@ -120,17 +135,18 @@ class BaseOpenAILLMService(LLMService): params = params or BaseOpenAILLMService.InputParams() - self._settings = { - "frequency_penalty": params.frequency_penalty, - "presence_penalty": params.presence_penalty, - "seed": params.seed, - "temperature": params.temperature, - "top_p": params.top_p, - "max_tokens": params.max_tokens, - "max_completion_tokens": params.max_completion_tokens, - "service_tier": params.service_tier, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } + self._settings = OpenAILLMSettings( + model=model, + frequency_penalty=params.frequency_penalty, + presence_penalty=params.presence_penalty, + seed=params.seed, + temperature=params.temperature, + top_p=params.top_p, + max_tokens=params.max_tokens, + max_completion_tokens=params.max_completion_tokens, + service_tier=params.service_tier, + extra=params.extra if isinstance(params.extra, dict) else {}, + ) self._retry_timeout_secs = retry_timeout_secs self._retry_on_timeout = retry_on_timeout self.set_model_name(model) @@ -250,20 +266,20 @@ class BaseOpenAILLMService(LLMService): "model": self.model_name, "stream": True, "stream_options": {"include_usage": True}, - "frequency_penalty": self._settings["frequency_penalty"], - "presence_penalty": self._settings["presence_penalty"], - "seed": self._settings["seed"], - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], - "max_completion_tokens": self._settings["max_completion_tokens"], - "service_tier": self._settings["service_tier"], + "frequency_penalty": self._settings.frequency_penalty, + "presence_penalty": self._settings.presence_penalty, + "seed": self._settings.seed, + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, + "max_completion_tokens": self._settings.max_completion_tokens, + "service_tier": self._settings.service_tier, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params async def run_inference( @@ -508,8 +524,6 @@ class BaseOpenAILLMService(LLMService): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it context = OpenAILLMContext.from_messages(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index cf249408c..abd66963b 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -10,8 +10,8 @@ import base64 import io import json import time -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger from PIL import Image @@ -59,6 +59,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt @@ -90,6 +91,17 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class OpenAIRealtimeLLMSettings(LLMSettings): + """Typed settings for OpenAI Realtime LLM services. + + Parameters: + session_properties: OpenAI Realtime session configuration. + """ + + session_properties: Any = field(default_factory=lambda: NOT_GIVEN) + + class OpenAIRealtimeLLMService(LLMService): """OpenAI Realtime LLM service providing real-time audio and text communication. @@ -161,9 +173,9 @@ class OpenAIRealtimeLLMService(LLMService): self.base_url = full_url self.set_model_name(model) - # Initialize session_properties - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = OpenAIRealtimeLLMSettings( + model=model, + session_properties=session_properties or events.SessionProperties(), ) self._audio_input_paused = start_audio_paused self._video_input_paused = start_video_paused @@ -227,12 +239,12 @@ class OpenAIRealtimeLLMService(LLMService): def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" - modalities = self._session_properties.output_modalities or ["audio", "text"] + modalities = self._settings.session_properties.output_modalities or ["audio", "text"] return modality in modalities def _get_enabled_modalities(self) -> list[str]: """Get the list of enabled modalities.""" - modalities = self._session_properties.output_modalities or ["audio", "text"] + modalities = self._settings.session_properties.output_modalities or ["audio", "text"] # API only supports single modality responses: either ["text"] or ["audio"] if "audio" in modalities: return ["audio"] @@ -305,9 +317,9 @@ class OpenAIRealtimeLLMService(LLMService): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. turn_detection_disabled = ( - self._session_properties.audio - and self._session_properties.audio.input - and self._session_properties.audio.input.turn_detection is False + self._settings.session_properties.audio + and self._settings.session_properties.audio.input + and self._settings.session_properties.audio.input.turn_detection is False ) if turn_detection_disabled: await self.send_client_event(events.InputAudioBufferClearEvent()) @@ -327,9 +339,9 @@ class OpenAIRealtimeLLMService(LLMService): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. turn_detection_disabled = ( - self._session_properties.audio - and self._session_properties.audio.input - and self._session_properties.audio.input.turn_detection is False + self._settings.session_properties.audio + and self._settings.session_properties.audio.input + and self._settings.session_properties.audio.input.turn_detection is False ) if turn_detection_disabled: await self.send_client_event(events.InputAudioBufferCommitEvent()) @@ -397,6 +409,16 @@ class OpenAIRealtimeLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Legacy dict path: frame.settings contains SessionProperties fields, + # not our Settings fields, so we construct SessionProperties directly. + # The new typed path (frame.update) falls through to super, which calls + # _update_settings_from_typed → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._update_settings() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -424,9 +446,6 @@ class OpenAIRealtimeLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() @@ -513,8 +532,15 @@ class OpenAIRealtimeLLMService(LLMService): # treat a send-side error as fatal. await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) + async def _update_settings_from_typed(self, update): + """Apply a typed settings update, sending a session update if needed.""" + changed = await super()._update_settings_from_typed(update) + if "session_properties" in changed: + await self._update_settings() + return changed + async def _update_settings(self): - settings = self._session_properties + settings = self._settings.session_properties adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() if self._context: diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index 4dd16be6e..12eada24e 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -16,6 +16,7 @@ Provides two STT services: import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Literal, Optional, Union from loguru import logger @@ -34,6 +35,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -123,6 +125,17 @@ class OpenAISTTService(BaseWhisperSTTService): _OPENAI_SAMPLE_RATE = 24000 +@dataclass +class OpenAIRealtimeSTTSettings(STTSettings): + """Typed settings for the OpenAI Realtime STT service. + + Parameters: + prompt: Optional prompt text to guide transcription style. + """ + + prompt: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + + class OpenAIRealtimeSTTService(WebsocketSTTService): """OpenAI Realtime Speech-to-Text service using WebSocket transcription sessions. @@ -213,12 +226,17 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): self._base_url = base_url self.set_model_name(model) - self._language_code = self._language_to_code(language) if language else None self._prompt = prompt self._turn_detection = turn_detection self._noise_reduction = noise_reduction self._should_interrupt = should_interrupt + self._settings: OpenAIRealtimeSTTSettings = OpenAIRealtimeSTTSettings( + model=model, + language=language, + prompt=prompt, + ) + self._receive_task = None self._session_ready = False self._resampler = create_stream_resampler() @@ -248,19 +266,31 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): """ return True - async def set_language(self, language: Language): - """Set the language for speech recognition. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update and send session update if needed. - If the session is already active, sends an updated configuration - to the server. + Keeps ``_language_code`` and ``_prompt`` in sync with typed settings + and sends a ``session.update`` to the server when the session is active. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``OpenAIRealtimeSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. """ - self._language_code = self._language_to_code(language) + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + if "prompt" in changed and isinstance(self._settings, OpenAIRealtimeSTTSettings): + self._prompt = self._settings.prompt + if self._session_ready: await self._send_session_update() + return changed + async def start(self, frame: StartFrame): """Start the service and establish WebSocket connection. @@ -407,8 +437,11 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): """Send ``session.update`` to configure the transcription session.""" transcription: dict = {"model": self.model_name} - if self._language_code: - transcription["language"] = self._language_code + language_code = ( + self._language_to_code(self._settings.language) if self._settings.language else None + ) + if language_code: + transcription["language"] = language_code if self._prompt: transcription["prompt"] = self._prompt diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index f59f0b31b..ee1e34316 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -10,6 +10,7 @@ This module provides integration with OpenAI's text-to-speech API for generating high-quality synthetic speech from text input. """ +from dataclasses import dataclass, field from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts @@ -60,6 +62,19 @@ VALID_VOICES: Dict[str, ValidVoice] = { } +@dataclass +class OpenAITTSSettings(TTSSettings): + """Typed settings for OpenAI TTS service. + + Parameters: + instructions: Instructions to guide voice synthesis behavior. + speed: Voice speed control (0.25 to 4.0, default 1.0). + """ + + instructions: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + + class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. @@ -118,7 +133,7 @@ class OpenAITTSService(TTSService): super().__init__(sample_rate=sample_rate, **kwargs) self.set_model_name(model) - self.set_voice(voice) + self._voice_id = voice self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) if instructions or speed: @@ -132,10 +147,12 @@ class OpenAITTSService(TTSService): stacklevel=2, ) - self._settings = { - "instructions": params.instructions if params else instructions, - "speed": params.speed if params else speed, - } + self._settings: OpenAITTSSettings = OpenAITTSSettings( + model=model, + voice=voice, + instructions=params.instructions if params else instructions, + speed=params.speed if params else speed, + ) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -145,15 +162,6 @@ class OpenAITTSService(TTSService): """ return True - async def set_model(self, model: str): - """Set the TTS model to use. - - Args: - model: The model name to use for text-to-speech synthesis. - """ - logger.info(f"Switching TTS model to: [{model}]") - self.set_model_name(model) - async def start(self, frame: StartFrame): """Start the OpenAI TTS service. @@ -190,11 +198,11 @@ class OpenAITTSService(TTSService): "response_format": "pcm", } - if self._settings["instructions"]: - create_params["instructions"] = self._settings["instructions"] + if self._settings.instructions: + create_params["instructions"] = self._settings.instructions - if self._settings["speed"]: - create_params["speed"] = self._settings["speed"] + if self._settings.speed: + create_params["speed"] = self._settings.speed async with self._client.audio.speech.with_streaming_response.create( **create_params diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 1199d8556..d37b1434e 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -10,8 +10,8 @@ import base64 import json import time import warnings -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Any, Optional from loguru import logger @@ -54,6 +54,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.openai.llm import OpenAIContextAggregatorPair +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt @@ -91,6 +92,17 @@ class CurrentAudioResponse: total_size: int = 0 +@dataclass +class OpenAIRealtimeBetaLLMSettings(LLMSettings): + """Typed settings for OpenAI Realtime Beta LLM services. + + Parameters: + session_properties: OpenAI Realtime session configuration. + """ + + session_properties: Any = field(default_factory=lambda: NOT_GIVEN) + + class OpenAIRealtimeBetaLLMService(LLMService): """OpenAI Realtime Beta LLM service providing real-time audio and text communication. @@ -146,8 +158,9 @@ class OpenAIRealtimeBetaLLMService(LLMService): self.base_url = full_url self.set_model_name(model) - self._session_properties: events.SessionProperties = ( - session_properties or events.SessionProperties() + self._settings = OpenAIRealtimeBetaLLMSettings( + model=model, + session_properties=session_properties or events.SessionProperties(), ) self._audio_input_paused = start_audio_paused self._send_transcription_frames = send_transcription_frames @@ -187,12 +200,12 @@ class OpenAIRealtimeBetaLLMService(LLMService): def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" - modalities = self._session_properties.modalities or ["audio", "text"] + modalities = self._settings.session_properties.modalities or ["audio", "text"] return modality in modalities def _get_enabled_modalities(self) -> list[str]: """Get the list of enabled modalities.""" - return self._session_properties.modalities or ["audio", "text"] + return self._settings.session_properties.modalities or ["audio", "text"] async def retrieve_conversation_item(self, item_id: str): """Retrieve a conversation item by ID from the server. @@ -259,7 +272,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_interruption(self): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. - if self._session_properties.turn_detection is False: + if self._settings.session_properties.turn_detection is False: await self.send_client_event(events.InputAudioBufferClearEvent()) await self.send_client_event(events.ResponseCancelEvent()) await self._truncate_current_audio_response() @@ -276,7 +289,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_user_stopped_speaking(self, frame): # None and False are different. Check for False. None means we're using OpenAI's # built-in turn detection defaults. - if self._session_properties.turn_detection is False: + if self._settings.session_properties.turn_detection is False: await self.send_client_event(events.InputAudioBufferCommitEvent()) await self.send_client_event(events.ResponseCreateEvent()) @@ -342,6 +355,16 @@ class OpenAIRealtimeBetaLLMService(LLMService): frame: The frame to process. direction: The direction of frame flow in the pipeline. """ + # Legacy dict path: frame.settings contains SessionProperties fields, + # not our Settings fields, so we construct SessionProperties directly. + # The new typed path (frame.update) falls through to super, which calls + # _update_settings_from_typed → our override handles the rest. + if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None: + self._settings.session_properties = events.SessionProperties(**frame.settings) + await self._update_settings() + await self.push_frame(frame, direction) + return + await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -377,9 +400,6 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_messages_append(frame) elif isinstance(frame, RealtimeMessagesUpdateFrame): self._context = frame.context - elif isinstance(frame, LLMUpdateSettingsFrame): - self._session_properties = events.SessionProperties(**frame.settings) - await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() elif isinstance(frame, RealtimeFunctionCallResultFrame): @@ -456,8 +476,15 @@ class OpenAIRealtimeBetaLLMService(LLMService): # treat a send-side error as fatal. await self.push_error(error_msg=f"Error sending client event: {e}", exception=e) + async def _update_settings_from_typed(self, update): + """Apply a typed settings update, sending a session update if needed.""" + changed = await super()._update_settings_from_typed(update) + if "session_properties" in changed: + await self._update_settings() + return changed + async def _update_settings(self): - settings = self._session_properties + settings = self._settings.session_properties # tools given in the context override the tools in the session properties if self._context and self._context.tools: settings.tools = self._context.tools diff --git a/src/pipecat/services/perplexity/llm.py b/src/pipecat/services/perplexity/llm.py index 4ea23aa82..d2dd40a57 100644 --- a/src/pipecat/services/perplexity/llm.py +++ b/src/pipecat/services/perplexity/llm.py @@ -72,16 +72,16 @@ class PerplexityLLMService(OpenAILLMService): } # Add OpenAI-compatible parameters if they're set - if self._settings["frequency_penalty"] is not NOT_GIVEN: - params["frequency_penalty"] = self._settings["frequency_penalty"] - if self._settings["presence_penalty"] is not NOT_GIVEN: - params["presence_penalty"] = self._settings["presence_penalty"] - if self._settings["temperature"] is not NOT_GIVEN: - params["temperature"] = self._settings["temperature"] - if self._settings["top_p"] is not NOT_GIVEN: - params["top_p"] = self._settings["top_p"] - if self._settings["max_tokens"] is not NOT_GIVEN: - params["max_tokens"] = self._settings["max_tokens"] + if self._settings.frequency_penalty is not NOT_GIVEN: + params["frequency_penalty"] = self._settings.frequency_penalty + if self._settings.presence_penalty is not NOT_GIVEN: + params["presence_penalty"] = self._settings.presence_penalty + if self._settings.temperature is not NOT_GIVEN: + params["temperature"] = self._settings.temperature + if self._settings.top_p is not NOT_GIVEN: + params["top_p"] = self._settings.top_p + if self._settings.max_tokens is not NOT_GIVEN: + params["max_tokens"] = self._settings.max_tokens return params diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 2d4cd0427..f79f54560 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -14,6 +14,7 @@ import io import json import struct import warnings +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional import aiohttp @@ -32,6 +33,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -97,6 +99,25 @@ def language_to_playht_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class PlayHTTTSSettings(TTSSettings): + """Typed settings for PlayHT TTS services. + + Parameters: + output_format: Audio output format. + voice_engine: Voice engine to use. + speed: Speech speed multiplier. Defaults to 1.0. + seed: Random seed for voice consistency. + playht_sample_rate: Audio sample rate sent to the API. + """ + + output_format: str = field(default_factory=lambda: NOT_GIVEN) + voice_engine: str = field(default_factory=lambda: NOT_GIVEN) + speed: float = field(default_factory=lambda: NOT_GIVEN) + seed: int = field(default_factory=lambda: NOT_GIVEN) + playht_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class PlayHTTTSService(InterruptibleTTSService): """PlayHT WebSocket-based text-to-speech service. @@ -170,17 +191,19 @@ class PlayHTTTSService(InterruptibleTTSService): self._receive_task = None self._context_id = None - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: PlayHTTTSSettings = PlayHTTTSSettings( + model=voice_engine, + voice=voice_url, + language=self.language_to_service_language(params.language) if params.language else "english", - "output_format": output_format, - "voice_engine": voice_engine, - "speed": params.speed, - "seed": params.seed, - } + output_format=output_format, + voice_engine=voice_engine, + speed=params.speed, + seed=params.seed, + ) self.set_model_name(voice_engine) - self.set_voice(voice_url) + self._voice_id = voice_url def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -304,13 +327,13 @@ class PlayHTTTSService(InterruptibleTTSService): # Handle the new response format with multiple URLs if "websocket_urls" in data: # Select URL based on voice_engine - if self._settings["voice_engine"] in data["websocket_urls"]: + if self._settings.voice_engine in data["websocket_urls"]: self._websocket_url = data["websocket_urls"][ - self._settings["voice_engine"] + self._settings.voice_engine ] else: raise ValueError( - f"Unsupported voice engine: {self._settings['voice_engine']}" + f"Unsupported voice engine: {self._settings.voice_engine}" ) else: raise ValueError("Invalid response: missing websocket_urls") @@ -382,12 +405,12 @@ class PlayHTTTSService(InterruptibleTTSService): tts_command = { "text": text, "voice": self._voice_id, - "voice_engine": self._settings["voice_engine"], - "output_format": self._settings["output_format"], + "voice_engine": self._settings.voice_engine, + "output_format": self._settings.output_format, "sample_rate": self.sample_rate, - "language": self._settings["language"], - "speed": self._settings["speed"], - "seed": self._settings["seed"], + "language": self._settings.language, + "speed": self._settings.speed, + "seed": self._settings.seed, "request_id": self._context_id, } @@ -499,17 +522,18 @@ class PlayHTHttpTTSService(TTSService): # Extract the base engine name voice_engine = voice_engine.replace("-ws", "") - self._settings = { - "language": self.language_to_service_language(params.language) + self._settings: PlayHTTTSSettings = PlayHTTTSSettings( + voice=voice_url, + language=self.language_to_service_language(params.language) if params.language else "english", - "output_format": output_format, - "voice_engine": voice_engine, - "speed": params.speed, - "seed": params.seed, - } + output_format=output_format, + voice_engine=voice_engine, + speed=params.speed, + seed=params.seed, + ) self.set_model_name(voice_engine) - self.set_voice(voice_url) + self._voice_id = voice_url async def start(self, frame: StartFrame): """Start the PlayHT HTTP TTS service. @@ -518,7 +542,7 @@ class PlayHTHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.playht_sample_rate = self.sample_rate def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -559,17 +583,17 @@ class PlayHTHttpTTSService(TTSService): payload = { "text": text, "voice": self._voice_id, - "voice_engine": self._settings["voice_engine"], - "output_format": self._settings["output_format"], + "voice_engine": self._settings.voice_engine, + "output_format": self._settings.output_format, "sample_rate": self.sample_rate, - "language": self._settings["language"], + "language": self._settings.language, } # Add optional parameters if they exist - if self._settings["speed"] is not None: - payload["speed"] = self._settings["speed"] - if self._settings["seed"] is not None: - payload["seed"] = self._settings["seed"] + if self._settings.speed is not None: + payload["speed"] = self._settings.speed + if self._settings.seed is not None: + payload["seed"] = self._settings.seed headers = { "Authorization": f"Bearer {self._api_key}", diff --git a/src/pipecat/services/resembleai/tts.py b/src/pipecat/services/resembleai/tts.py index 964b9fa18..08f9b81bd 100644 --- a/src/pipecat/services/resembleai/tts.py +++ b/src/pipecat/services/resembleai/tts.py @@ -8,6 +8,7 @@ import base64 import json +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import AudioContextWordTTSService from pipecat.transcriptions.language import Language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -38,6 +40,21 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class ResembleAITTSSettings(TTSSettings): + """Typed settings for Resemble AI TTS service. + + Parameters: + precision: PCM bit depth (PCM_32, PCM_24, PCM_16, or MULAW). + output_format: Audio format (wav or mp3). + resemble_sample_rate: Audio sample rate sent to the API. + """ + + precision: str = field(default_factory=lambda: NOT_GIVEN) + output_format: str = field(default_factory=lambda: NOT_GIVEN) + resemble_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + class ResembleAITTSService(AudioContextWordTTSService): """Resemble AI TTS service with WebSocket streaming and word timestamps. @@ -76,11 +93,12 @@ class ResembleAITTSService(AudioContextWordTTSService): self._api_key = api_key self._voice_id = voice_id self._url = url - self._settings = { - "precision": precision, - "output_format": output_format, - "sample_rate": sample_rate, - } + self._settings: ResembleAITTSSettings = ResembleAITTSSettings( + voice=voice_id, + precision=precision, + output_format=output_format, + resemble_sample_rate=sample_rate, + ) self._websocket = None self._request_id_counter = 0 @@ -101,7 +119,7 @@ class ResembleAITTSService(AudioContextWordTTSService): self._jitter_buffer_bytes = 44100 # ~1000ms at 22050Hz to handle 400ms+ network gaps self._playback_started: dict[str, bool] = {} # Track if we've started playback per request - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -125,9 +143,9 @@ class ResembleAITTSService(AudioContextWordTTSService): "data": text, "binary_response": False, # Use JSON frames to get timestamps "request_id": self._request_id_counter, # ResembleAI only accepts number - "output_format": self._settings["output_format"], - "sample_rate": self._settings["sample_rate"], - "precision": self._settings["precision"], + "output_format": self._settings.output_format, + "sample_rate": self._settings.resemble_sample_rate, + "precision": self._settings.precision, "no_audio_header": True, } @@ -141,7 +159,7 @@ class ResembleAITTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.resemble_sample_rate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index e38e840e6..5a3ed67a2 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -12,7 +12,8 @@ using Rime's API for streaming and batch audio synthesis. import base64 import json -from typing import Any, AsyncGenerator, Mapping, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Optional import aiohttp from loguru import logger @@ -30,6 +31,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import ( AudioContextWordTTSService, InterruptibleTTSService, @@ -68,6 +70,62 @@ def language_to_rime_language(language: Language) -> str: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class RimeTTSSettings(TTSSettings): + """Typed settings for Rime WS JSON and HTTP TTS services. + + Parameters: + speaker: Voice speaker ID. + modelId: Rime model identifier. + audioFormat: Audio output format. + samplingRate: Audio sample rate. + lang: Rime language code. + speedAlpha: Speech speed multiplier. Defaults to 1.0. + reduceLatency: Whether to reduce latency at potential quality cost. + pauseBetweenBrackets: Whether to add pauses between bracketed content. + phonemizeBetweenBrackets: Whether to phonemize bracketed content. + inlineSpeedAlpha: Inline speed control markup. + """ + + speaker: str = field(default_factory=lambda: NOT_GIVEN) + modelId: str = field(default_factory=lambda: NOT_GIVEN) + audioFormat: str = field(default_factory=lambda: NOT_GIVEN) + samplingRate: int = field(default_factory=lambda: NOT_GIVEN) + lang: str = field(default_factory=lambda: NOT_GIVEN) + speedAlpha: float = field(default_factory=lambda: NOT_GIVEN) + reduceLatency: bool = field(default_factory=lambda: NOT_GIVEN) + pauseBetweenBrackets: bool = field(default_factory=lambda: NOT_GIVEN) + phonemizeBetweenBrackets: bool = field(default_factory=lambda: NOT_GIVEN) + inlineSpeedAlpha: str = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class RimeNonJsonTTSSettings(TTSSettings): + """Typed settings for Rime non-JSON WS TTS service. + + Parameters: + speaker: Voice speaker ID. + modelId: Rime model identifier. + audioFormat: Audio output format. + samplingRate: Audio sample rate. + lang: Rime language code. + segment: Text segmentation mode ("immediate", "bySentence", "never"). + repetition_penalty: Token repetition penalty (1.0-2.0). + temperature: Sampling temperature (0.0-1.0). + top_p: Cumulative probability threshold (0.0-1.0). + """ + + speaker: str = field(default_factory=lambda: NOT_GIVEN) + modelId: str = field(default_factory=lambda: NOT_GIVEN) + audioFormat: str = field(default_factory=lambda: NOT_GIVEN) + samplingRate: int = field(default_factory=lambda: NOT_GIVEN) + lang: str = field(default_factory=lambda: NOT_GIVEN) + segment: str = field(default_factory=lambda: NOT_GIVEN) + repetition_penalty: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + top_p: float = field(default_factory=lambda: NOT_GIVEN) + + class RimeTTSService(AudioContextWordTTSService): """Text-to-Speech service using Rime's websocket API. @@ -149,19 +207,17 @@ class RimeTTSService(AudioContextWordTTSService): self._url = url self._voice_id = voice_id self._model = model - self._settings = { - "speaker": voice_id, - "modelId": model, - "audioFormat": "pcm", - "samplingRate": 0, - "lang": self.language_to_service_language(params.language) - if params.language - else "eng", - "speedAlpha": params.speed_alpha, - "reduceLatency": params.reduce_latency, - "pauseBetweenBrackets": json.dumps(params.pause_between_brackets), - "phonemizeBetweenBrackets": json.dumps(params.phonemize_between_brackets), - } + self._settings: RimeTTSSettings = RimeTTSSettings( + speaker=voice_id, + modelId=model, + audioFormat="pcm", + samplingRate=0, + lang=self.language_to_service_language(params.language) if params.language else "eng", + speedAlpha=params.speed_alpha, + reduceLatency=params.reduce_latency, + pauseBetweenBrackets=json.dumps(params.pause_between_brackets), + phonemizeBetweenBrackets=json.dumps(params.phonemize_between_brackets), + ) # State tracking self._context_id = None # Tracks current turn @@ -188,15 +244,6 @@ class RimeTTSService(AudioContextWordTTSService): """ return language_to_rime_language(language) - async def set_model(self, model: str): - """Update the TTS model. - - Args: - model: The model name to use for synthesis. - """ - self._model = model - await super().set_model(model) - # A set of Rime-specific helpers for text transformations def SPELL(text: str) -> str: """Wrap text in Rime spell function.""" @@ -222,15 +269,15 @@ class RimeTTSService(AudioContextWordTTSService): self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)]) return f"[{text}]" - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if voice changed.""" prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - self._settings["speaker"] = self._voice_id - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + changed = await super()._update_settings_from_typed(update) + if "voice" in changed: + self._settings.speaker = self._voice_id await self._disconnect() await self._connect() + return changed def _build_msg(self, text: str = "") -> dict: """Build JSON message for Rime API.""" @@ -255,7 +302,7 @@ class RimeTTSService(AudioContextWordTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["samplingRate"] = self.sample_rate + self._settings.samplingRate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -301,7 +348,20 @@ class RimeTTSService(AudioContextWordTTSService): if self._websocket and self._websocket.state is State.OPEN: return - params = "&".join(f"{k}={v}" for k, v in self._settings.items()) + params = "&".join( + f"{k}={v}" + for k, v in { + "speaker": self._settings.speaker, + "modelId": self._settings.modelId, + "audioFormat": self._settings.audioFormat, + "samplingRate": self._settings.samplingRate, + "lang": self._settings.lang, + "speedAlpha": self._settings.speedAlpha, + "reduceLatency": self._settings.reduceLatency, + "pauseBetweenBrackets": self._settings.pauseBetweenBrackets, + "phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets, + }.items() + ) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect(url, additional_headers=headers) @@ -525,21 +585,17 @@ class RimeHttpTTSService(TTSService): self._api_key = api_key self._session = aiohttp_session self._base_url = "https://users.rime.ai/v1/rime-tts" - self._settings = { - "lang": self.language_to_service_language(params.language) - if params.language - else "eng", - "speedAlpha": params.speed_alpha, - "reduceLatency": params.reduce_latency, - "pauseBetweenBrackets": params.pause_between_brackets, - "phonemizeBetweenBrackets": params.phonemize_between_brackets, - } - self.set_voice(voice_id) + self._settings: RimeTTSSettings = RimeTTSSettings( + lang=self.language_to_service_language(params.language) if params.language else "eng", + speedAlpha=params.speed_alpha, + reduceLatency=params.reduce_latency, + pauseBetweenBrackets=params.pause_between_brackets, + phonemizeBetweenBrackets=params.phonemize_between_brackets, + inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else NOT_GIVEN, + ) + self._voice_id = voice_id self.set_model_name(model) - if params.inline_speed_alpha: - self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha - def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -578,7 +634,15 @@ class RimeHttpTTSService(TTSService): "Content-Type": "application/json", } - payload = self._settings.copy() + payload = { + "lang": self._settings.lang, + "speedAlpha": self._settings.speedAlpha, + "reduceLatency": self._settings.reduceLatency, + "pauseBetweenBrackets": self._settings.pauseBetweenBrackets, + "phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets, + } + if is_given(self._settings.inlineSpeedAlpha): + payload["inlineSpeedAlpha"] = self._settings.inlineSpeedAlpha payload["text"] = text payload["speaker"] = self._voice_id payload["modelId"] = self._model_name @@ -699,26 +763,24 @@ class RimeNonJsonTTSService(InterruptibleTTSService): self._url = url self._voice_id = voice_id self._model = model - self._settings = { - "speaker": voice_id, - "modelId": model, - "audioFormat": audio_format, - "samplingRate": sample_rate, - } - - if params.language: - self._settings["lang"] = self.language_to_service_language(params.language) - if params.segment is not None: - self._settings["segment"] = params.segment - if params.repetition_penalty is not None: - self._settings["repetition_penalty"] = params.repetition_penalty - if params.temperature is not None: - self._settings["temperature"] = params.temperature - if params.top_p is not None: - self._settings["top_p"] = params.top_p + self._settings: RimeNonJsonTTSSettings = RimeNonJsonTTSSettings( + speaker=voice_id, + modelId=model, + audioFormat=audio_format, + samplingRate=sample_rate, + lang=self.language_to_service_language(params.language) + if params.language + else NOT_GIVEN, + segment=params.segment if params.segment is not None else NOT_GIVEN, + repetition_penalty=params.repetition_penalty + if params.repetition_penalty is not None + else NOT_GIVEN, + temperature=params.temperature if params.temperature is not None else NOT_GIVEN, + top_p=params.top_p if params.top_p is not None else NOT_GIVEN, + ) # Add any extra parameters for future compatibility if params.extra: - self._settings.update(params.extra) + self._settings.extra.update(params.extra) self._receive_task = None self._context_id: Optional[str] = None @@ -750,7 +812,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["samplingRate"] = self.sample_rate + self._settings.samplingRate = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): @@ -794,8 +856,26 @@ class RimeNonJsonTTSService(InterruptibleTTSService): try: if self._websocket and self._websocket.state is State.OPEN: return - # Build URL with query parameters (only non-None values) - params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None) + # Build URL with query parameters (only given, non-None values) + settings_dict = { + "speaker": self._settings.speaker, + "modelId": self._settings.modelId, + "audioFormat": self._settings.audioFormat, + "samplingRate": self._settings.samplingRate, + } + if is_given(self._settings.lang): + settings_dict["lang"] = self._settings.lang + if is_given(self._settings.segment): + settings_dict["segment"] = self._settings.segment + if is_given(self._settings.repetition_penalty): + settings_dict["repetition_penalty"] = self._settings.repetition_penalty + if is_given(self._settings.temperature): + settings_dict["temperature"] = self._settings.temperature + if is_given(self._settings.top_p): + settings_dict["top_p"] = self._settings.top_p + # Include extras + settings_dict.update(self._settings.extra) + params = "&".join(f"{k}={v}" for k, v in settings_dict.items() if v is not None) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect( @@ -889,68 +969,23 @@ class RimeNonJsonTTSService(InterruptibleTTSService): except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if necessary. + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and reconnect if necessary. Since all settings are WebSocket URL query parameters, any setting change requires reconnecting to apply the new values. """ - needs_reconnect = False + changed = await super()._update_settings_from_typed(update) - # Track previous values from self._settings only - prev_settings = self._settings.copy() + # Sync voice and model to settings dict fields + if "voice" in changed: + self._settings.speaker = self._voice_id + if "model" in changed: + self._settings.modelId = self._model_name - # Let parent class handle standard settings (voice, model, language) - await super()._update_settings(settings) - - # Check if voice changed and update settings dict - if "voice" in settings or "voice_id" in settings: - self._settings["speaker"] = self._voice_id - if prev_settings.get("speaker") != self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") - needs_reconnect = True - - # Check if model changed and update settings dict - if "model" in settings: - self._settings["modelId"] = self._model - if prev_settings.get("modelId") != self._model: - logger.info(f"Switching TTS model to: [{self._model}]") - needs_reconnect = True - - # Handle language explicitly - if "language" in settings: - new_lang = self.language_to_service_language(settings["language"]) - if new_lang and new_lang != prev_settings.get("lang"): - logger.info(f"Updating language to: [{new_lang}]") - self._settings["lang"] = new_lang - needs_reconnect = True - - # Check other parameters - for key in ["segment", "repetition_penalty", "temperature", "top_p"]: - if key in settings and settings[key] != prev_settings.get(key): - logger.info(f"Updating {key} to: [{settings[key]}]") - self._settings[key] = settings[key] - needs_reconnect = True - - # Handle extra parameters - for key, value in settings.items(): - if key not in [ - "voice", - "voice_id", - "model", - "language", - "segment", - "repetition_penalty", - "temperature", - "top_p", - ]: - if value != prev_settings.get(key): - logger.info(f"Updating extra parameter {key} to: [{value}]") - self._settings[key] = value - needs_reconnect = True - - # Reconnect if any setting changed - if needs_reconnect: + if changed: logger.debug("Settings changed, reconnecting WebSocket with new parameters") await self._disconnect() await self._connect() + + return changed diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index 047ce0e6c..99c7bca2c 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -87,16 +87,16 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore "model": self.model_name, "stream": True, "stream_options": {"include_usage": True}, - "temperature": self._settings["temperature"], - "top_p": self._settings["top_p"], - "max_tokens": self._settings["max_tokens"], - "max_completion_tokens": self._settings["max_completion_tokens"], + "temperature": self._settings.temperature, + "top_p": self._settings.top_p, + "max_tokens": self._settings.max_tokens, + "max_completion_tokens": self._settings.max_completion_tokens, } # Messages, tools, tool_choice params.update(params_from_context) - params.update(self._settings["extra"]) + params.update(self._settings.extra) return params @traced_llm # type: ignore diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index 998597956..e2bc6a08f 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -12,7 +12,7 @@ can handle multiple audio formats for Indian language speech recognition. """ import base64 -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger @@ -32,6 +32,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.sarvam._sdk import sdk_headers +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import SARVAM_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -130,6 +131,23 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = { } +@dataclass +class SarvamSTTSettings(STTSettings): + """Typed settings for the Sarvam STT service. + + Parameters: + prompt: Optional prompt to guide transcription/translation style. + mode: Mode of operation (transcribe, translate, verbatim, etc.). + vad_signals: Enable VAD signals in response. + high_vad_sensitivity: Enable high VAD sensitivity. + """ + + prompt: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + mode: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + vad_signals: Optional[bool] = field(default_factory=lambda: NOT_GIVEN) + high_vad_sensitivity: Optional[bool] = field(default_factory=lambda: NOT_GIVEN) + + class SarvamSTTService(STTService): """Sarvam speech-to-text service. @@ -207,22 +225,8 @@ class SarvamSTTService(STTService): self.set_model_name(model) self._api_key = api_key - self._language_code: Optional[Language] = params.language - - # Set language string: use provided language or model's default - if params.language: - self._language_string = language_to_sarvam_language(params.language) - else: - self._language_string = self._config.default_language - - self._prompt = params.prompt - - # Set mode: use provided mode or model's default - self._mode = params.mode if params.mode is not None else self._config.default_mode # Store connection parameters - self._vad_signals = params.vad_signals - self._high_vad_sensitivity = params.high_vad_sensitivity self._input_audio_codec = input_audio_codec # Initialize Sarvam SDK client @@ -240,7 +244,19 @@ class SarvamSTTService(STTService): self._socket_client = None self._receive_task = None - if self._vad_signals: + # Resolve mode default from model config + mode = params.mode if params.mode is not None else self._config.default_mode + + self._settings: SarvamSTTSettings = SarvamSTTSettings( + model=model, + language=params.language, + prompt=params.prompt if params.prompt is not None else NOT_GIVEN, + mode=mode if mode is not None else NOT_GIVEN, + vad_signals=params.vad_signals, + high_vad_sensitivity=params.high_vad_sensitivity, + ) + + if params.vad_signals: self._register_event_handler("on_speech_started") self._register_event_handler("on_speech_stopped") self._register_event_handler("on_utterance_end") @@ -258,6 +274,12 @@ class SarvamSTTService(STTService): """ return language_to_sarvam_language(language) + def _get_language_string(self) -> Optional[str]: + """Resolve the current language setting to a Sarvam language code string.""" + if self._settings.language: + return language_to_sarvam_language(self._settings.language) + return self._config.default_language + def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -275,42 +297,74 @@ class SarvamSTTService(STTService): await super().process_frame(frame, direction) # Only handle VAD frames when not using Sarvam's VAD signals - if not self._vad_signals: + if not self._settings.vad_signals: if isinstance(frame, VADUserStartedSpeakingFrame): await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): if self._socket_client: await self._socket_client.flush() - async def set_language(self, language: Language): - """Set the recognition language and reconnect. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, validate, sync state, and reconnect. Args: - language: The language to use for speech recognition. + update: A :class:`STTSettings` (or ``SarvamSTTSettings``) delta. + + Returns: + Set of field names whose values actually changed. Raises: - ValueError: If called on a model that auto-detects language. + ValueError: If a setting is not supported by the current model. """ - if not self._config.supports_language: - raise ValueError( - f"Model '{self.model_name}' does not support language parameter " - "(auto-detects language)." - ) + # Validate against model capabilities before applying + if is_given(update.language) and update.language is not None: + if not self._config.supports_language: + raise ValueError( + f"Model '{self.model_name}' does not support language parameter " + "(auto-detects language)." + ) + + if isinstance(update, SarvamSTTSettings): + if is_given(update.prompt) and update.prompt is not None: + if not self._config.supports_prompt: + raise ValueError( + f"Model '{self.model_name}' does not support prompt parameter." + ) + if is_given(update.mode) and update.mode is not None: + if not self._config.supports_mode: + raise ValueError(f"Model '{self.model_name}' does not support mode parameter.") + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed - logger.info(f"Switching STT language to: [{language}]") - self._language_code = language - self._language_string = language_to_sarvam_language(language) await self._disconnect() await self._connect() + return changed async def set_prompt(self, prompt: Optional[str]): """Set the transcription/translation prompt and reconnect. + .. deprecated:: + Use ``STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...))`` instead. + Args: prompt: Prompt text to guide transcription/translation style/context. Pass None to clear/disable prompt. Only applicable to models that support prompts. """ + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + f"{self.__class__.__name__}.set_prompt() is deprecated. " + "Use STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...)) instead.", + DeprecationWarning, + stacklevel=2, + ) + if not self._config.supports_prompt: if prompt is not None: raise ValueError(f"Model '{self.model_name}' does not support prompt parameter.") @@ -318,7 +372,7 @@ class SarvamSTTService(STTService): return logger.info(f"Updating {self.model_name} prompt.") - self._prompt = prompt + self._settings.prompt = prompt await self._disconnect() await self._connect() @@ -405,24 +459,25 @@ class SarvamSTTService(STTService): # Enable flush signal when using Pipecat's VAD (not Sarvam's) so that # the flush() call on user-stopped-speaking is honored by the server. - if not self._vad_signals: + if not self._settings.vad_signals: connect_kwargs["flush_signal"] = "true" # Only send vad parameters when explicitly set (avoid overriding server defaults) - if self._vad_signals is not None: - connect_kwargs["vad_signals"] = "true" if self._vad_signals else "false" - if self._high_vad_sensitivity is not None: + if self._settings.vad_signals is not None: + connect_kwargs["vad_signals"] = "true" if self._settings.vad_signals else "false" + if self._settings.high_vad_sensitivity is not None: connect_kwargs["high_vad_sensitivity"] = ( - "true" if self._high_vad_sensitivity else "false" + "true" if self._settings.high_vad_sensitivity else "false" ) # Add language_code for models that support it - if self._language_string is not None: - connect_kwargs["language_code"] = self._language_string + language_string = self._get_language_string() + if language_string is not None: + connect_kwargs["language_code"] = language_string # Add mode for models that support it - if self._config.supports_mode and self._mode is not None: - connect_kwargs["mode"] = self._mode + if self._config.supports_mode and is_given(self._settings.mode): + connect_kwargs["mode"] = self._settings.mode def _connect_with_sdk_headers(connect_fn, **kwargs): # Different SDK versions may use different kwarg names. @@ -449,8 +504,8 @@ class SarvamSTTService(STTService): self._socket_client = await self._websocket_context.__aenter__() # Set prompt if provided (only for models that support prompts) - if self._prompt is not None and self._config.supports_prompt: - await self._socket_client.set_prompt(self._prompt) + if is_given(self._settings.prompt) and self._config.supports_prompt: + await self._socket_client.set_prompt(self._settings.prompt) # Register event handler for incoming messages def _message_handler(message): @@ -544,10 +599,12 @@ class SarvamSTTService(STTService): # Prefer language from message (auto-detected for translate models). Fallback to configured. if language_code: language = self._map_language_code_to_enum(language_code) - elif self._language_string: - language = self._map_language_code_to_enum(self._language_string) else: - language = Language.HI_IN + language_string = self._get_language_string() + if language_string: + language = self._map_language_code_to_enum(language_string) + else: + language = Language.HI_IN # Emit utterance end event await self._call_event_handler("on_utterance_end") diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 753293c75..e28914b4c 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -40,9 +40,9 @@ See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for full API import asyncio import base64 import json -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple +from typing import AsyncGenerator, Dict, List, Optional, Tuple import aiohttp from loguru import logger @@ -62,6 +62,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.sarvam._sdk import sdk_headers +from pipecat.services.settings import NOT_GIVEN, TTSSettings, is_given from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -244,6 +245,80 @@ def language_to_sarvam_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) +@dataclass +class SarvamHttpTTSSettings(TTSSettings): + """Typed settings for Sarvam HTTP TTS service. + + Parameters: + language: Sarvam language code. + enable_preprocessing: Whether to enable text preprocessing. Defaults to False. + **Note:** Always enabled for bulbul:v3-beta (cannot be disabled). + pace: Speech pace multiplier. Defaults to 1.0. + - bulbul:v2: Range 0.3 to 3.0 + - bulbul:v3-beta: Range 0.5 to 2.0 + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0). + Lower values = more deterministic, higher = more random. Defaults to 0.6. + **Note:** Only supported for bulbul:v3-beta. Ignored for v2. + sample_rate: Audio sample rate. + """ + + language: str = field(default_factory=lambda: NOT_GIVEN) + enable_preprocessing: bool = field(default_factory=lambda: NOT_GIVEN) + pace: float = field(default_factory=lambda: NOT_GIVEN) + pitch: float = field(default_factory=lambda: NOT_GIVEN) + loudness: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + sarvam_sample_rate: int = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class SarvamWSTTSSettings(TTSSettings): + """Typed settings for Sarvam WebSocket TTS service. + + Parameters: + target_language_code: Sarvam language code. + speaker: Voice speaker ID. + speech_sample_rate: Audio sample rate as string. + enable_preprocessing: Enable text preprocessing. Defaults to False. + **Note:** Always enabled for bulbul:v3-beta. + min_buffer_size: Minimum characters to buffer before generating audio. + Lower values reduce latency but may affect quality. Defaults to 50. + max_chunk_length: Maximum characters processed in a single chunk. + Controls memory usage and processing efficiency. Defaults to 150. + output_audio_codec: Audio codec format. Options: linear16, mulaw, alaw, + opus, flac, aac, wav, mp3. Defaults to "linear16". + output_audio_bitrate: Audio bitrate (32k, 64k, 96k, 128k, 192k). + Defaults to "128k". + pace: Speech pace multiplier. Defaults to 1.0. + - bulbul:v2: Range 0.3 to 3.0 + - bulbul:v3-beta: Range 0.5 to 2.0 + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0. + **Note:** Only supported for bulbul:v2. Ignored for v3 models. + temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0). + Lower = more deterministic, higher = more random. Defaults to 0.6. + **Note:** Only supported for bulbul:v3-beta. Ignored for v2. + """ + + target_language_code: str = field(default_factory=lambda: NOT_GIVEN) + speaker: str = field(default_factory=lambda: NOT_GIVEN) + speech_sample_rate: str = field(default_factory=lambda: NOT_GIVEN) + enable_preprocessing: bool = field(default_factory=lambda: NOT_GIVEN) + min_buffer_size: int = field(default_factory=lambda: NOT_GIVEN) + max_chunk_length: int = field(default_factory=lambda: NOT_GIVEN) + output_audio_codec: str = field(default_factory=lambda: NOT_GIVEN) + output_audio_bitrate: str = field(default_factory=lambda: NOT_GIVEN) + pace: float = field(default_factory=lambda: NOT_GIVEN) + pitch: float = field(default_factory=lambda: NOT_GIVEN) + loudness: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + + class SarvamHttpTTSService(TTSService): """Text-to-Speech service using Sarvam AI's API. @@ -403,35 +478,35 @@ class SarvamHttpTTSService(TTSService): pace = max(pace_min, min(pace_max, pace)) # Build base settings - self._settings = { - "language": ( + self._settings: SarvamHttpTTSSettings = SarvamHttpTTSSettings( + language=( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "enable_preprocessing": ( + enable_preprocessing=( True if self._config.preprocessing_always_enabled else params.enable_preprocessing ), - "pace": pace, - "model": model, - } + pace=pace, + model=model, + ) # Add parameters based on model support if self._config.supports_pitch: - self._settings["pitch"] = params.pitch + self._settings.pitch = params.pitch elif params.pitch != 0.0: logger.warning(f"pitch parameter is ignored for {model}") if self._config.supports_loudness: - self._settings["loudness"] = params.loudness + self._settings.loudness = params.loudness elif params.loudness != 1.0: logger.warning(f"loudness parameter is ignored for {model}") if self._config.supports_temperature: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature elif params.temperature != 0.6: logger.warning(f"temperature parameter is ignored for {model}") self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -459,7 +534,7 @@ class SarvamHttpTTSService(TTSService): frame: The start frame containing initialization parameters. """ await super().start(frame) - self._settings["sample_rate"] = self.sample_rate + self._settings.sarvam_sample_rate = self.sample_rate @traced_tts async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: @@ -480,21 +555,25 @@ class SarvamHttpTTSService(TTSService): # Build payload with common parameters payload = { "text": text, - "target_language_code": self._settings["language"], + "target_language_code": self._settings.language, "speaker": self._voice_id, "sample_rate": self.sample_rate, - "enable_preprocessing": self._settings["enable_preprocessing"], + "enable_preprocessing": self._settings.enable_preprocessing, "model": self._model_name, - "pace": self._settings.get("pace", 1.0), + "pace": self._settings.pace if is_given(self._settings.pace) else 1.0, } # Add model-specific parameters based on config if self._config.supports_pitch: - payload["pitch"] = self._settings.get("pitch", 0.0) + payload["pitch"] = self._settings.pitch if is_given(self._settings.pitch) else 0.0 if self._config.supports_loudness: - payload["loudness"] = self._settings.get("loudness", 1.0) + payload["loudness"] = ( + self._settings.loudness if is_given(self._settings.loudness) else 1.0 + ) if self._config.supports_temperature: - payload["temperature"] = self._settings.get("temperature", 0.6) + payload["temperature"] = ( + self._settings.temperature if is_given(self._settings.temperature) else 0.6 + ) headers = { "api-subscription-key": self._api_key, @@ -748,7 +827,7 @@ class SarvamTTSService(InterruptibleTTSService): self._websocket_url = f"{url}?model={model}" self._api_key = api_key self.set_model_name(model) - self.set_voice(voice_id) + self._voice_id = voice_id # Validate and clamp pace to model's valid range pace = params.pace @@ -758,36 +837,36 @@ class SarvamTTSService(InterruptibleTTSService): pace = max(pace_min, min(pace_max, pace)) # Build base settings - self._settings = { - "target_language_code": ( + self._settings: SarvamWSTTSSettings = SarvamWSTTSSettings( + target_language_code=( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "speaker": voice_id, - "speech_sample_rate": str(sample_rate), - "enable_preprocessing": ( + speaker=voice_id, + speech_sample_rate=str(sample_rate), + enable_preprocessing=( True if self._config.preprocessing_always_enabled else params.enable_preprocessing ), - "min_buffer_size": params.min_buffer_size, - "max_chunk_length": params.max_chunk_length, - "output_audio_codec": params.output_audio_codec, - "output_audio_bitrate": params.output_audio_bitrate, - "pace": pace, - "model": model, - } + min_buffer_size=params.min_buffer_size, + max_chunk_length=params.max_chunk_length, + output_audio_codec=params.output_audio_codec, + output_audio_bitrate=params.output_audio_bitrate, + pace=pace, + model=model, + ) # Add parameters based on model support if self._config.supports_pitch: - self._settings["pitch"] = params.pitch + self._settings.pitch = params.pitch elif params.pitch != 0.0: logger.warning(f"pitch parameter is ignored for {model}") if self._config.supports_loudness: - self._settings["loudness"] = params.loudness + self._settings.loudness = params.loudness elif params.loudness != 1.0: logger.warning(f"loudness parameter is ignored for {model}") if self._config.supports_temperature: - self._settings["temperature"] = params.temperature + self._settings.temperature = params.temperature elif params.temperature != 0.6: logger.warning(f"temperature parameter is ignored for {model}") @@ -823,7 +902,7 @@ class SarvamTTSService(InterruptibleTTSService): await super().start(frame) # WebSocket API expects sample rate as string - self._settings["speech_sample_rate"] = str(self.sample_rate) + self._settings.speech_sample_rate = str(self.sample_rate) await self._connect() async def stop(self, frame: EndFrame): @@ -870,13 +949,12 @@ class SarvamTTSService(InterruptibleTTSService): if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): await self.flush_audio() - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if voice changed.""" - prev_voice = self._voice_id - await super()._update_settings(settings) - if not prev_voice == self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed settings update and resend config if voice changed.""" + changed = await super()._update_settings_from_typed(update) + if "voice" in changed: await self._send_config() + return changed async def _connect(self): """Connect to Sarvam WebSocket and start background tasks.""" @@ -934,9 +1012,28 @@ class SarvamTTSService(InterruptibleTTSService): """Send initial configuration message.""" if not self._websocket: raise Exception("WebSocket not connected") - self._settings["speaker"] = self._voice_id - logger.debug(f"Config being sent is {self._settings}") - config_message = {"type": "config", "data": self._settings} + self._settings.speaker = self._voice_id + # Build config dict for the API + config_data = { + "target_language_code": self._settings.target_language_code, + "speaker": self._settings.speaker, + "speech_sample_rate": self._settings.speech_sample_rate, + "enable_preprocessing": self._settings.enable_preprocessing, + "min_buffer_size": self._settings.min_buffer_size, + "max_chunk_length": self._settings.max_chunk_length, + "output_audio_codec": self._settings.output_audio_codec, + "output_audio_bitrate": self._settings.output_audio_bitrate, + "pace": self._settings.pace, + "model": self._settings.model, + } + if is_given(self._settings.pitch): + config_data["pitch"] = self._settings.pitch + if is_given(self._settings.loudness): + config_data["loudness"] = self._settings.loudness + if is_given(self._settings.temperature): + config_data["temperature"] = self._settings.temperature + logger.debug(f"Config being sent is {config_data}") + config_message = {"type": "config", "data": config_data} try: await self._websocket.send(json.dumps(config_message)) diff --git a/src/pipecat/services/settings.py b/src/pipecat/services/settings.py new file mode 100644 index 000000000..fbec5cdf8 --- /dev/null +++ b/src/pipecat/services/settings.py @@ -0,0 +1,297 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Typed settings infrastructure for Pipecat AI services. + +This module provides typed dataclass-based settings objects that replace the +stringly-typed ``Mapping[str, Any]`` dictionaries previously used for service +configuration. Each service type has a corresponding settings class (e.g. +``TTSSettings``, ``LLMSettings``) whose fields use the ``NOT_GIVEN`` sentinel +to distinguish "leave unchanged" from an explicit ``None``. + +Key concepts: + +- **NOT_GIVEN sentinel**: A value meaning "this field was not provided in the + update". Distinct from ``None`` (which may be a valid value for a setting). +- **Settings as both state and delta**: The same class is used for the + service's current settings *and* for update objects. Fields set to + ``NOT_GIVEN`` are simply skipped when applying an update. +- **apply_update**: Applies a delta onto a target settings object and returns + the set of field names that actually changed. +- **from_mapping**: Constructs a typed settings object from a plain dict, + supporting field aliases (e.g. ``"voice_id"`` → ``"voice"``). +- **Extras**: Unknown keys land in the ``extra`` dict so services that have + non-standard settings don't lose data. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass, field, fields +from typing import Any, ClassVar, Dict, Mapping, Optional, Set, Type, TypeVar + +from loguru import logger + +# --------------------------------------------------------------------------- +# NOT_GIVEN sentinel +# --------------------------------------------------------------------------- + + +class _NotGiven: + """Sentinel indicating a settings field was not provided. + + ``NOT_GIVEN`` means "the caller did not supply this value" — distinct from + ``None``, which may be a legitimate setting value. It is used as the + default for every settings field so that ``apply_update`` can tell which + fields the caller actually wants to change. + """ + + _instance: Optional[_NotGiven] = None + + def __new__(cls) -> _NotGiven: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __repr__(self) -> str: + return "NOT_GIVEN" + + def __bool__(self) -> bool: + return False + + +NOT_GIVEN: _NotGiven = _NotGiven() +"""Singleton sentinel meaning "this field was not included in the update".""" + + +def is_given(value: Any) -> bool: + """Check whether a value was explicitly provided (i.e. is not ``NOT_GIVEN``). + + Args: + value: The value to check. + + Returns: + ``True`` if *value* is anything other than ``NOT_GIVEN``. + """ + return not isinstance(value, _NotGiven) + + +# --------------------------------------------------------------------------- +# Base ServiceSettings +# --------------------------------------------------------------------------- + +_S = TypeVar("_S", bound="ServiceSettings") + + +@dataclass +class ServiceSettings: + """Base class for typed service settings. + + Every AI service type (LLM, TTS, STT) extends this with its own fields. + Fields default to ``NOT_GIVEN`` so that an instance can represent either + the full current state **or** a sparse update delta. + + Parameters: + model: The model identifier used by the service. + extra: Overflow dict for service-specific keys that don't map to a + declared field. + """ + + # -- common fields ------------------------------------------------------- + + model: Any = field(default_factory=lambda: NOT_GIVEN) + """AI model identifier (e.g. ``"gpt-4o"``, ``"eleven_turbo_v2_5"``).""" + + extra: Dict[str, Any] = field(default_factory=dict) + """Catch-all for service-specific keys that have no declared field.""" + + # -- class-level configuration ------------------------------------------- + + _aliases: ClassVar[Dict[str, str]] = {} + """Map of alternative key names to canonical field names. + + For example ``{"voice_id": "voice"}`` lets callers use either spelling. + Subclasses should override this as needed. + """ + + # -- public API ---------------------------------------------------------- + + def given_fields(self) -> Dict[str, Any]: + """Return a dict of only the fields that were explicitly provided. + + Skips ``NOT_GIVEN`` values and the ``extra`` field itself. Entries + from ``extra`` are included at the top level. + + Returns: + Dictionary mapping field names to their provided values. + """ + result: Dict[str, Any] = {} + for f in fields(self): + if f.name == "extra": + continue + val = getattr(self, f.name) + if is_given(val): + result[f.name] = val + result.update(self.extra) + return result + + def apply_update(self: _S, update: _S) -> Set[str]: + """Apply *update* onto this settings object, returning changed field names. + + Only fields in *update* that are **given** (i.e. not ``NOT_GIVEN``) + are considered. A field is "changed" if its new value differs from + the current value. + + The ``extra`` dicts are merged: keys present in the update overwrite + keys in the target. + + Args: + update: A settings object of the same type containing the delta. + + Returns: + The set of field names whose values actually changed. + + Examples:: + + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="bob") + changed = current.apply_update(delta) + # changed == {"voice"} + # current.voice == "bob", current.language == "en" + """ + changed: Set[str] = set() + for f in fields(self): + if f.name == "extra": + continue + new_val = getattr(update, f.name) + if not is_given(new_val): + continue + old_val = getattr(self, f.name) + if old_val != new_val: + setattr(self, f.name, new_val) + changed.add(f.name) + + # Merge extra + for key, new_val in update.extra.items(): + old_val = self.extra.get(key, NOT_GIVEN) + if old_val != new_val: + self.extra[key] = new_val + changed.add(key) + + return changed + + @classmethod + def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S: + """Construct a typed settings object from a plain dictionary. + + Keys are matched to dataclass fields by name. Keys listed in + ``_aliases`` are translated to their canonical name first. Any + remaining unrecognized keys are placed into ``extra``. + + Args: + settings: A dictionary of setting names to values. + + Returns: + A new settings instance with the corresponding fields populated. + + Examples:: + + update = TTSSettings.from_mapping({"voice_id": "alice", "speed": 1.2}) + # update.voice == "alice" (via alias) + # update.extra == {"speed": 1.2} + """ + field_names = {f.name for f in fields(cls)} - {"extra"} + kwargs: Dict[str, Any] = {} + extra: Dict[str, Any] = {} + + for key, value in settings.items(): + # Resolve aliases first + canonical = cls._aliases.get(key, key) + if canonical in field_names: + kwargs[canonical] = value + else: + extra[key] = value + + instance = cls(**kwargs) + instance.extra = extra + return instance + + def to_dict(self) -> Dict[str, Any]: + """Serialize to a flat dictionary, including extra. + + Only given (non-``NOT_GIVEN``) values are included. This is the + inverse of ``from_mapping`` and useful for passing settings to APIs + that expect plain dicts. + + Returns: + A flat dictionary of all given settings. + """ + return self.given_fields() + + def copy(self: _S) -> _S: + """Return a deep copy of this settings instance. + + Returns: + A new settings object with the same field values. + """ + return copy.deepcopy(self) + + +# --------------------------------------------------------------------------- +# Service-specific settings +# --------------------------------------------------------------------------- + + +@dataclass +class LLMSettings(ServiceSettings): + """Typed settings for LLM services. + + Parameters: + model: LLM model identifier. + temperature: Sampling temperature. + max_tokens: Maximum tokens to generate. + top_p: Nucleus sampling probability. + top_k: Top-k sampling parameter. + frequency_penalty: Frequency penalty. + presence_penalty: Presence penalty. + seed: Random seed for reproducibility. + """ + + temperature: Any = field(default_factory=lambda: NOT_GIVEN) + max_tokens: Any = field(default_factory=lambda: NOT_GIVEN) + top_p: Any = field(default_factory=lambda: NOT_GIVEN) + top_k: Any = field(default_factory=lambda: NOT_GIVEN) + frequency_penalty: Any = field(default_factory=lambda: NOT_GIVEN) + presence_penalty: Any = field(default_factory=lambda: NOT_GIVEN) + seed: Any = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class TTSSettings(ServiceSettings): + """Typed settings for TTS services. + + Parameters: + model: TTS model identifier. + voice: Voice identifier or name. + language: Language for speech synthesis. + """ + + voice: Any = field(default_factory=lambda: NOT_GIVEN) + language: Any = field(default_factory=lambda: NOT_GIVEN) + + _aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"} + + +@dataclass +class STTSettings(ServiceSettings): + """Typed settings for STT services. + + Parameters: + model: STT model identifier. + language: Language for speech recognition. + """ + + language: Any = field(default_factory=lambda: NOT_GIVEN) diff --git a/src/pipecat/services/soniox/stt.py b/src/pipecat/services/soniox/stt.py index c9184ba4c..9d732a356 100644 --- a/src/pipecat/services/soniox/stt.py +++ b/src/pipecat/services/soniox/stt.py @@ -8,7 +8,8 @@ import json import time -from typing import AsyncGenerator, List, Optional +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, List, Optional from loguru import logger from pydantic import BaseModel @@ -23,6 +24,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import SONIOX_TTFS_P99 from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language @@ -134,6 +136,17 @@ def _prepare_language_hints( return list(set(prepared_languages)) +@dataclass +class SonioxSTTSettings(STTSettings): + """Typed settings for Soniox STT service. + + Parameters: + input_params: Soniox ``SonioxInputParams`` for detailed configuration. + """ + + input_params: SonioxInputParams = field(default_factory=lambda: NOT_GIVEN) + + class SonioxSTTService(WebsocketSTTService): """Speech-to-Text service using Soniox's WebSocket API. @@ -181,9 +194,13 @@ class SonioxSTTService(WebsocketSTTService): self._api_key = api_key self._url = url self.set_model_name(params.model) - self._params = params self._vad_force_turn_endpoint = vad_force_turn_endpoint + self._settings = SonioxSTTSettings( + model=params.model, + input_params=params, + ) + self._final_transcription_buffer = [] self._last_tokens_received: Optional[float] = None @@ -198,6 +215,43 @@ class SonioxSTTService(WebsocketSTTService): await super().start(frame) await self._connect() + async def _update_settings_from_typed(self, update: SonioxSTTSettings) -> set[str]: + """Apply a typed settings update, keeping ``input_params`` in sync. + + Top-level ``model`` is the source of truth. When it is given in + *update* its value is propagated into ``input_params``. When only + ``input_params`` is given, its ``model`` is propagated *up* to the + top-level field. + + Any change triggers a WebSocket reconnect. + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + model_given = is_given(getattr(update, "model", NOT_GIVEN)) + + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + # --- Sync model -------------------------------------------------- + if model_given: + # Top-level model wins → push into input_params. + self._settings.input_params.model = self._settings.model + elif "input_params" in changed and self._settings.input_params.model is not None: + # Only input_params was given → pull model up. + self._settings.model = self._settings.input_params.model + self.set_model_name(self._settings.model) + + await self._disconnect() + await self._connect() + + return changed + async def stop(self, frame: EndFrame): """Stop the Soniox STT websocket connection. @@ -311,7 +365,9 @@ class SonioxSTTService(WebsocketSTTService): # Either one or the other is required. enable_endpoint_detection = not self._vad_force_turn_endpoint - context = self._params.context + params = self._settings.input_params + + context = params.context if isinstance(context, SonioxContextObject): context = context.model_dump() @@ -319,16 +375,16 @@ class SonioxSTTService(WebsocketSTTService): config = { "api_key": self._api_key, "model": self._model_name, - "audio_format": self._params.audio_format, - "num_channels": self._params.num_channels or 1, + "audio_format": params.audio_format, + "num_channels": params.num_channels or 1, "enable_endpoint_detection": enable_endpoint_detection, "sample_rate": self.sample_rate, - "language_hints": _prepare_language_hints(self._params.language_hints), - "language_hints_strict": self._params.language_hints_strict, + "language_hints": _prepare_language_hints(params.language_hints), + "language_hints_strict": params.language_hints_strict, "context": context, - "enable_speaker_diarization": self._params.enable_speaker_diarization, - "enable_language_identification": self._params.enable_language_identification, - "client_reference_id": self._params.client_reference_id, + "enable_speaker_diarization": params.enable_speaker_diarization, + "enable_language_identification": params.enable_language_identification, + "client_reference_id": params.client_reference_id, } # Send the configuration message. diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index ca949a9fd..d04bb564d 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -8,8 +8,10 @@ import asyncio import os +import warnings +from dataclasses import dataclass, field from enum import Enum -from typing import Any, AsyncGenerator +from typing import Any, AsyncGenerator, ClassVar from dotenv import load_dotenv from loguru import logger @@ -31,6 +33,7 @@ from pipecat.frames.frames import ( VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.settings import NOT_GIVEN, STTSettings, is_given from pipecat.services.stt_latency import SPEECHMATICS_TTFS_P99 from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language, resolve_language @@ -80,6 +83,81 @@ class TurnDetectionMode(str, Enum): SMART_TURN = "smart_turn" +@dataclass +class SpeechmaticsSTTSettings(STTSettings): + """Typed settings for Speechmatics STT service. + + See ``SpeechmaticsSTTService.InputParams`` for detailed descriptions of each field. + + Parameters: + model: The operating point / model name. + domain: Domain for Speechmatics API. + turn_detection_mode: Endpoint handling mode. + speaker_active_format: Formatter for active speaker ID. + speaker_passive_format: Formatter for passive speaker ID. + focus_speakers: List of speaker IDs to focus on. + ignore_speakers: List of speaker IDs to ignore. + focus_mode: Speaker focus mode for diarization. + known_speakers: List of known speaker labels and identifiers. + additional_vocab: List of additional vocabulary entries. + audio_encoding: Audio encoding format. + operating_point: Operating point for accuracy vs. latency. + max_delay: Maximum delay in seconds for transcription. + end_of_utterance_silence_trigger: Maximum delay for end of utterance trigger. + end_of_utterance_max_delay: Maximum delay for end of utterance. + punctuation_overrides: Punctuation overrides. + include_partials: Include partial segment fragments. + split_sentences: Emit finalized sentences mid-turn. + enable_diarization: Enable speaker diarization. + speaker_sensitivity: Diarization sensitivity. + max_speakers: Maximum number of speakers to detect. + prefer_current_speaker: Prefer current speaker ID. + extra_params: Extra parameters for the STT engine. + """ + + domain: str = field(default_factory=lambda: NOT_GIVEN) + turn_detection_mode: TurnDetectionMode = field(default_factory=lambda: NOT_GIVEN) + speaker_active_format: str = field(default_factory=lambda: NOT_GIVEN) + speaker_passive_format: str = field(default_factory=lambda: NOT_GIVEN) + focus_speakers: list = field(default_factory=lambda: NOT_GIVEN) + ignore_speakers: list = field(default_factory=lambda: NOT_GIVEN) + focus_mode: Any = field(default_factory=lambda: NOT_GIVEN) + known_speakers: list = field(default_factory=lambda: NOT_GIVEN) + additional_vocab: list = field(default_factory=lambda: NOT_GIVEN) + audio_encoding: Any = field(default_factory=lambda: NOT_GIVEN) + operating_point: Any = field(default_factory=lambda: NOT_GIVEN) + max_delay: float = field(default_factory=lambda: NOT_GIVEN) + end_of_utterance_silence_trigger: float = field(default_factory=lambda: NOT_GIVEN) + end_of_utterance_max_delay: float = field(default_factory=lambda: NOT_GIVEN) + punctuation_overrides: dict = field(default_factory=lambda: NOT_GIVEN) + include_partials: bool = field(default_factory=lambda: NOT_GIVEN) + split_sentences: bool = field(default_factory=lambda: NOT_GIVEN) + enable_diarization: bool = field(default_factory=lambda: NOT_GIVEN) + speaker_sensitivity: float = field(default_factory=lambda: NOT_GIVEN) + max_speakers: int = field(default_factory=lambda: NOT_GIVEN) + prefer_current_speaker: bool = field(default_factory=lambda: NOT_GIVEN) + extra_params: dict = field(default_factory=lambda: NOT_GIVEN) + + #: Fields that can be updated on a live connection via the Speechmatics + #: diarization-config API — no reconnect needed. + HOT_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "focus_speakers", + "ignore_speakers", + "focus_mode", + } + ) + + #: Fields that are purely local (formatting templates) — no reconnect + #: and no API call needed. + LOCAL_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "speaker_active_format", + "speaker_passive_format", + } + ) + + class SpeechmaticsSTTService(STTService): """Speechmatics STT service implementation. @@ -327,30 +405,56 @@ class SpeechmaticsSTTService(STTService): # Deprecation check self._check_deprecated_args(kwargs, params) - # Voice agent + # Output formatting defaults + speaker_active_format = params.speaker_active_format + if speaker_active_format is None: + speaker_active_format = ( + "@{speaker_id}: {text}" if params.enable_diarization else "{text}" + ) + speaker_passive_format = params.speaker_passive_format or speaker_active_format + + # Typed settings — seeded from InputParams + self._settings = SpeechmaticsSTTSettings( + language=params.language, + domain=params.domain, + turn_detection_mode=params.turn_detection_mode, + speaker_active_format=speaker_active_format, + speaker_passive_format=speaker_passive_format, + focus_speakers=params.focus_speakers, + ignore_speakers=params.ignore_speakers, + focus_mode=params.focus_mode, + known_speakers=params.known_speakers, + additional_vocab=params.additional_vocab, + audio_encoding=params.audio_encoding, + operating_point=params.operating_point, + max_delay=params.max_delay, + end_of_utterance_silence_trigger=params.end_of_utterance_silence_trigger, + end_of_utterance_max_delay=params.end_of_utterance_max_delay, + punctuation_overrides=params.punctuation_overrides, + include_partials=params.include_partials, + split_sentences=params.split_sentences, + enable_diarization=params.enable_diarization, + speaker_sensitivity=params.speaker_sensitivity, + max_speakers=params.max_speakers, + prefer_current_speaker=params.prefer_current_speaker, + extra_params=params.extra_params, + ) + + # Build SDK config from settings self._client: VoiceAgentClient | None = None - self._config: VoiceAgentConfig = self._prepare_config(params) + self._config: VoiceAgentConfig = self._build_config() # Outbound frame queue self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue() - # Output formatting - if params.speaker_active_format is None: - params.speaker_active_format = ( - "@{speaker_id}: {text}" if params.enable_diarization else "{text}" - ) - # Framework options self._enable_vad: bool = self._config.end_of_utterance_mode not in [ EndOfUtteranceMode.FIXED, EndOfUtteranceMode.EXTERNAL, ] - self._speaker_active_format: str = params.speaker_active_format - self._speaker_passive_format: str = ( - params.speaker_passive_format or params.speaker_active_format - ) - # Model + metrics + # Model + metrics (operating_point comes from the SDK config/preset) + self._settings.model = self._config.operating_point.value self.set_model_name(self._config.operating_point.value) # Message queue @@ -374,6 +478,56 @@ class SpeechmaticsSTTService(STTService): await super().start(frame) await self._connect() + async def _update_settings_from_typed(self, update: SpeechmaticsSTTSettings) -> set[str]: + """Apply typed settings update, reconnecting only when necessary. + + Fields are classified into three categories (see + ``SpeechmaticsSTTSettings``): + + * **HOT_FIELDS** – diarization speaker settings that can be pushed + to a live Speechmatics connection without reconnecting. + * **LOCAL_FIELDS** – formatting templates evaluated locally; no + reconnect or API call needed. + * Everything else – baked into ``VoiceAgentConfig`` at connection + time and therefore require a full disconnect / reconnect. + + Args: + update: A typed settings delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + + if not changed: + return changed + + no_reconnect = SpeechmaticsSTTSettings.HOT_FIELDS | SpeechmaticsSTTSettings.LOCAL_FIELDS + needs_reconnect = bool(changed - no_reconnect) + + if needs_reconnect: + # Connection-level fields changed — rebuild the SDK config + # from the now-updated self._settings, then reconnect. + self._config = self._build_config() + await self._disconnect() + await self._connect() + elif changed & SpeechmaticsSTTSettings.HOT_FIELDS: + if self._config.enable_diarization: + # Only hot-updatable fields changed — push to the live session. + self._config.speaker_config.focus_speakers = self._settings.focus_speakers + self._config.speaker_config.ignore_speakers = self._settings.ignore_speakers + self._config.speaker_config.focus_mode = self._settings.focus_mode + if self._client: + self._client.update_diarization_config(self._config.speaker_config) + else: + # Diarization not enabled — need a full reconnect to apply. + self._config = self._build_config() + await self._disconnect() + await self._connect() + # LOCAL_FIELDS: already applied by super(); nothing else to do. + + return changed + async def stop(self, frame: EndFrame): """Called when the session ends.""" await super().stop(frame) @@ -484,28 +638,35 @@ class SpeechmaticsSTTService(STTService): # CONFIGURATION # ============================================================================ - def _prepare_config(self, params: InputParams) -> VoiceAgentConfig: - """Parse the InputParams into VoiceAgentConfig.""" - # Preset - config = VoiceAgentConfigPreset.load(params.turn_detection_mode.value) + def _build_config(self) -> VoiceAgentConfig: + """Build a ``VoiceAgentConfig`` from the current ``self._settings``. + + Used both at init time and before reconnecting so the connection + always reflects the latest settings. + """ + s = self._settings + + # Preset from turn detection mode + config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value) # Language + domain - config.language = self._language_to_speechmatics_language(params.language) - config.domain = params.domain - config.output_locale = self._locale_to_speechmatics_locale(config.language, params.language) + language = s.language + config.language = self._language_to_speechmatics_language(language) + config.domain = s.domain if is_given(s.domain) else None + config.output_locale = self._locale_to_speechmatics_locale(config.language, language) # Speaker config config.speaker_config = SpeakerFocusConfig( - focus_speakers=params.focus_speakers, - ignore_speakers=params.ignore_speakers, - focus_mode=params.focus_mode, + focus_speakers=s.focus_speakers if is_given(s.focus_speakers) else [], + ignore_speakers=s.ignore_speakers if is_given(s.ignore_speakers) else [], + focus_mode=s.focus_mode if is_given(s.focus_mode) else SpeakerFocusMode.RETAIN, ) - config.known_speakers = params.known_speakers + config.known_speakers = s.known_speakers if is_given(s.known_speakers) else [] # Custom dictionary - config.additional_vocab = params.additional_vocab + config.additional_vocab = s.additional_vocab if is_given(s.additional_vocab) else [] - # Advanced parameters + # Advanced parameters — only set if given (not NOT_GIVEN or None) for param in [ "operating_point", "max_delay", @@ -519,21 +680,20 @@ class SpeechmaticsSTTService(STTService): "max_speakers", "prefer_current_speaker", ]: - if getattr(params, param) is not None: - setattr(config, param, getattr(params, param)) + val = getattr(s, param) + if is_given(val) and val is not None: + setattr(config, param, val) # Extra parameters - if isinstance(params.extra_params, dict): - for key, value in params.extra_params.items(): + if is_given(s.extra_params) and isinstance(s.extra_params, dict): + for key, value in s.extra_params.items(): if hasattr(config, key): setattr(config, key, value) # Enable sentences - config.speech_segment_config = SpeechSegmentConfig( - emit_sentences=params.split_sentences or False - ) + split = s.split_sentences if is_given(s.split_sentences) else False + config.speech_segment_config = SpeechSegmentConfig(emit_sentences=split or False) - # Return the complete config return config def update_params( @@ -542,12 +702,23 @@ class SpeechmaticsSTTService(STTService): ) -> None: """Updates the speaker configuration. + .. deprecated:: + Use ``STTUpdateSettingsFrame`` with + ``SpeechmaticsSTTSettings(...)`` instead. + This can update the speakers to listen to or ignore during an in-flight transcription. Only available if diarization is enabled. Args: params: Update parameters for the service. """ + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "update_params() is deprecated. Use STTUpdateSettingsFrame with " + "SpeechmaticsSTTSettings(...) instead.", + DeprecationWarning, + ) # Check possible if not self._config.enable_diarization: raise ValueError("Diarization is not enabled") @@ -717,9 +888,9 @@ class SpeechmaticsSTTService(STTService): def attr_from_segment(segment: dict[str, Any]) -> dict[str, Any]: # Formats the output text based on the speaker and defined formats from the config. text = ( - self._speaker_active_format + self._settings.speaker_active_format if segment.get("is_active", True) - else self._speaker_passive_format + else self._settings.speaker_passive_format ).format( **{ "speaker_id": segment.get("speaker_id", "UU"), diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 0f3ff0cb6..0907b4e26 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -95,7 +95,7 @@ class SpeechmaticsTTSService(TTSService): self._params = params or SpeechmaticsTTSService.InputParams() # Set voice from constructor parameter - self.set_voice(voice_id) + self._voice_id = voice_id def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index b556bb23a..d4e5f4cb5 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -34,6 +34,7 @@ from pipecat.frames.frames import ( from pipecat.metrics.metrics import TTFBMetricsData from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import ServiceSettings, STTSettings from pipecat.services.stt_latency import DEFAULT_TTFS_P99 from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language @@ -101,7 +102,6 @@ class STTService(AIService): self._audio_passthrough = audio_passthrough self._init_sample_rate = sample_rate self._sample_rate = 0 - self._settings: Dict[str, Any] = {} self._tracing_enabled: bool = False self._muted: bool = False self._user_id: str = "" @@ -166,18 +166,36 @@ class STTService(AIService): async def set_model(self, model: str): """Set the speech recognition model. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + Args: model: The name of the model to use for speech recognition. """ - self.set_model_name(model) + logger.info(f"Switching STT model to: [{model}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(model=model)) + else: + self.set_model_name(model) async def set_language(self, language: Language): """Set the language for speech recognition. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + Args: language: The language to use for speech recognition. """ - pass + logger.info(f"Switching STT language to: [{language}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(language=language)) + else: + pass @abstractmethod async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: @@ -224,6 +242,23 @@ class STTService(AIService): else: logger.warning(f"Unknown setting for STT service: {key}") + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed STT settings update. + + Handles ``model`` (via parent). Does **not** call ``set_language`` + — concrete services should override this method and handle language + changes (including any reconnect logic) based on the returned + changed-field set. + + Args: + update: A typed STT settings delta. + + Returns: + Set of field names whose values actually changed. + """ + changed = await super()._update_settings_from_typed(update) + return changed + async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): """Process an audio frame for speech recognition. @@ -285,7 +320,16 @@ class STTService(AIService): await self._handle_vad_user_stopped_speaking(frame) await self.push_frame(frame, direction) elif isinstance(frame, STTUpdateSettingsFrame): - await self._update_settings(frame.settings) + # New path: typed settings update object. + if frame.update is not None: + await self._update_settings_from_typed(frame.update) + # Legacy path: plain dict, but service uses typed settings — convert. + elif isinstance(self._settings, ServiceSettings): + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings_from_typed(update) + # Legacy path: plain dict, service still uses dict-based settings. + else: + await self._update_settings(frame.settings) elif isinstance(frame, STTMuteFrame): self._muted = frame.mute logger.debug(f"STT service {'muted' if frame.mute else 'unmuted'}") diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 239d2398b..4196e7872 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -52,6 +52,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService +from pipecat.services.settings import ServiceSettings, TTSSettings, is_given from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -189,7 +190,6 @@ class TTSService(AIService): self._init_sample_rate = sample_rate self._sample_rate = 0 self._voice_id: str = "" - self._settings: Dict[str, Any] = {} self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() if text_aggregator: import warnings @@ -263,18 +263,40 @@ class TTSService(AIService): async def set_model(self, model: str): """Set the TTS model to use. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + Args: model: The name of the TTS model. """ - self.set_model_name(model) + logger.info(f"Switching TTS model to: [{model}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(model=model)) + else: + self.set_model_name(model) - def set_voice(self, voice: str): + async def set_voice(self, voice: str): """Set the voice for speech synthesis. + When the service has been migrated to typed settings this routes + through :meth:`_update_settings_from_typed` so that concrete + services can react (e.g. reconnect) in a single place. + + .. versionchanged:: 0.0.103 + Now ``async``. In ``__init__`` methods, set + ``self._voice_id`` directly instead of calling this method. + Args: voice: The voice identifier or name. """ - self._voice_id = voice + logger.info(f"Switching TTS voice to: [{voice}]") + if isinstance(self._settings, ServiceSettings): + settings_cls = type(self._settings) + await self._update_settings_from_typed(settings_cls(voice=voice)) + else: + self._voice_id = voice def create_context_id(self) -> str: """Generate a unique context ID for a TTS request. @@ -416,13 +438,42 @@ class TTSService(AIService): elif key == "model": self.set_model_name(value) elif key == "voice" or key == "voice_id": - self.set_voice(value) + self._voice_id = value elif key == "text_filter": for filter in self._text_filters: await filter.update_settings(value) else: logger.warning(f"Unknown setting for TTS service: {key}") + async def _update_settings_from_typed(self, update: TTSSettings) -> set[str]: + """Apply a typed TTS settings update. + + Handles ``model`` (via parent) and syncs ``_voice_id`` when voice + changes. Translates language values before applying. Does **not** + call ``set_voice`` or ``set_model`` directly — concrete services + should override this method and handle reconnect logic based on the + returned changed-field set. + + Args: + update: A typed TTS settings delta. + + Returns: + Set of field names whose values actually changed. + """ + # Translate language *before* applying so the stored value is canonical + if is_given(update.language) and update.language is not None: + converted = self.language_to_service_language(update.language) + if converted is not None: + update.language = converted + + changed = await super()._update_settings_from_typed(update) + + # Keep _voice_id in sync for code that reads it directly + if "voice" in changed and isinstance(self._settings, TTSSettings): + self._voice_id = self._settings.voice + + return changed + async def say(self, text: str): """Immediately speak the provided text. @@ -504,7 +555,16 @@ class TTSService(AIService): await self.flush_audio() self._processing_text = processing_text elif isinstance(frame, TTSUpdateSettingsFrame): - await self._update_settings(frame.settings) + # New path: typed settings update object. + if frame.update is not None: + await self._update_settings_from_typed(frame.update) + # Legacy path: plain dict, but service uses typed settings — convert. + elif isinstance(self._settings, ServiceSettings): + update = type(self._settings).from_mapping(frame.settings) + await self._update_settings_from_typed(update) + # Legacy path: plain dict, service still uses dict-based settings. + else: + await self._update_settings(frame.settings) elif isinstance(frame, BotStoppedSpeakingFrame): await self._maybe_resume_frame_processing() await self.push_frame(frame, direction) diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index d549b11e5..9f0658486 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -15,6 +15,7 @@ import asyncio import datetime import json import uuid +from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Union import aiohttp @@ -34,7 +35,6 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, - LLMUpdateSettingsFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, @@ -56,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.settings import NOT_GIVEN, LLMSettings from pipecat.utils.time import time_now_iso8601 try: @@ -66,6 +67,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class UltravoxRealtimeLLMSettings(LLMSettings): + """Settings for UltravoxRealtimeLLMService. + + Parameters: + output_medium: The output medium for the model ("voice" or "text"). + """ + + output_medium: str = field(default=NOT_GIVEN) + + class AgentInputParams(BaseModel): """Input parameters for Ultravox Realtime generation using a pre-defined Agent. @@ -163,6 +175,7 @@ class UltravoxRealtimeLLMService(LLMService): **kwargs: Additional arguments passed to parent LLMService. """ super().__init__(**kwargs) + self._settings = UltravoxRealtimeLLMSettings() self._params = params if one_shot_selected_tools: if not isinstance(self._params, OneShotInputParams): @@ -310,6 +323,12 @@ class UltravoxRealtimeLLMService(LLMService): await self.cancel_task(self._receive_task, timeout=1.0) self._receive_task = None + async def _update_settings_from_typed(self, update: UltravoxRealtimeLLMSettings): + changed = await super()._update_settings_from_typed(update) + if "output_medium" in changed: + await self._update_output_medium(self._settings.output_medium) + return changed + # # frame processing # StartFrame, StopFrame, CancelFrame implemented in base class @@ -331,9 +350,6 @@ class UltravoxRealtimeLLMService(LLMService): else LLMContext.from_openai_context(frame.context) ) await self._handle_context(context) - elif isinstance(frame, LLMUpdateSettingsFrame): - if "output_medium" in frame.settings: - await self._update_output_medium(frame.settings.get("output_medium")) elif isinstance(frame, InputTextRawFrame): await self._send_user_text(frame.text) await self.push_frame(frame, direction) diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index bc999dba4..2a02c6ce7 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -10,6 +10,7 @@ This module provides common functionality for services implementing the Whisper interface, including language mapping, metrics generation, and error handling. """ +from dataclasses import dataclass, field from typing import AsyncGenerator, Optional from loguru import logger @@ -17,6 +18,7 @@ from openai import AsyncOpenAI from openai.types.audio import Transcription from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_latency import WHISPER_TTFS_P99 from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language @@ -24,6 +26,22 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt +@dataclass +class BaseWhisperSTTSettings(STTSettings): + """Typed settings for Whisper API-based STT services. + + Parameters: + base_url: API base URL. + prompt: Optional text to guide the model's style or continue + a previous segment. + temperature: Sampling temperature between 0 and 1. + """ + + base_url: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + prompt: Optional[str] = field(default_factory=lambda: NOT_GIVEN) + temperature: Optional[float] = field(default_factory=lambda: NOT_GIVEN) + + def language_to_whisper_language(language: Language) -> Optional[str]: """Maps pipecat Language enum to Whisper API language codes. @@ -143,26 +161,36 @@ class BaseWhisperSTTService(SegmentedSTTService): self._temperature = temperature self._include_prob_metrics = include_prob_metrics - self._settings = { - "base_url": base_url, - "language": self._language, - "prompt": self._prompt, - "temperature": self._temperature, - } + self._settings: BaseWhisperSTTSettings = BaseWhisperSTTSettings( + model=model, + language=self._language, + base_url=base_url, + prompt=self._prompt, + temperature=self._temperature, + ) def _create_client(self, api_key: Optional[str], base_url: Optional[str]): return AsyncOpenAI(api_key=api_key, base_url=base_url) - async def set_model(self, model: str): - """Set the model name for transcription. + async def _update_settings_from_typed(self, update: STTSettings) -> set[str]: + """Apply a typed settings update, syncing instance variables. - Args: - model: The name of the model to use. + Keeps ``_language``, ``_prompt``, and ``_temperature`` in sync with + the typed settings fields. """ - self.set_model_name(model) + changed = await super()._update_settings_from_typed(update) + + if "language" in changed: + self._language = self.language_to_service_language(Language(self._settings.language)) + if "prompt" in changed: + self._prompt = self._settings.prompt + if "temperature" in changed: + self._temperature = self._settings.temperature + + return changed def can_generate_metrics(self) -> bool: - """Indicates whether this service can generate metrics. + """Whether this service can generate processing metrics. Returns: bool: True, as this service supports metric generation. @@ -180,15 +208,6 @@ class BaseWhisperSTTService(SegmentedSTTService): """ return language_to_whisper_language(language) - async def set_language(self, language: Language): - """Set the language for transcription. - - Args: - language: The Language enum value to use for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._language = self.language_to_service_language(language) - @traced_stt async def _handle_transcription( self, transcript: str, is_final: bool, language: Optional[Language] = None diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index f11978cc2..30451e6d0 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -11,6 +11,7 @@ supporting both Faster Whisper and MLX Whisper backends for efficient inference. """ import asyncio +from dataclasses import dataclass, field from enum import Enum from typing import AsyncGenerator, Optional @@ -19,6 +20,7 @@ from loguru import logger from typing_extensions import TYPE_CHECKING, override from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import NOT_GIVEN, STTSettings from pipecat.services.stt_service import SegmentedSTTService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.time import time_now_iso8601 @@ -172,6 +174,36 @@ def language_to_whisper_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class WhisperSTTSettings(STTSettings): + """Typed settings for the local Whisper (Faster Whisper) STT service. + + Parameters: + device: Inference device ('cpu', 'cuda', or 'auto'). + compute_type: Compute type for inference ('default', 'int8', etc.). + no_speech_prob: Probability threshold for filtering non-speech segments. + """ + + device: str = field(default_factory=lambda: NOT_GIVEN) + compute_type: str = field(default_factory=lambda: NOT_GIVEN) + no_speech_prob: float = field(default_factory=lambda: NOT_GIVEN) + + +@dataclass +class WhisperMLXSTTSettings(STTSettings): + """Typed settings for the MLX Whisper STT service. + + Parameters: + no_speech_prob: Probability threshold for filtering non-speech segments. + temperature: Sampling temperature (0.0-1.0). + engine: Whisper engine identifier. + """ + + no_speech_prob: float = field(default_factory=lambda: NOT_GIVEN) + temperature: float = field(default_factory=lambda: NOT_GIVEN) + engine: str = field(default_factory=lambda: NOT_GIVEN) + + class WhisperSTTService(SegmentedSTTService): """Class to transcribe audio with a locally-downloaded Whisper model. @@ -206,12 +238,13 @@ class WhisperSTTService(SegmentedSTTService): self._no_speech_prob = no_speech_prob self._model: Optional[WhisperModel] = None - self._settings = { - "language": language, - "device": self._device, - "compute_type": self._compute_type, - "no_speech_prob": self._no_speech_prob, - } + self._settings: WhisperSTTSettings = WhisperSTTSettings( + model=model if isinstance(model, str) else model.value, + language=language, + device=self._device, + compute_type=self._compute_type, + no_speech_prob=self._no_speech_prob, + ) self._load() @@ -234,15 +267,6 @@ class WhisperSTTService(SegmentedSTTService): """ return language_to_whisper_language(language) - async def set_language(self, language: Language): - """Set the language for transcription. - - Args: - language: The Language enum value to use for transcription. - """ - logger.info(f"Switching STT language to: [{language}]") - self._settings["language"] = language - def _load(self): """Loads the Whisper model. @@ -293,7 +317,7 @@ class WhisperSTTService(SegmentedSTTService): # Divide by 32768 because we have signed 16-bit data. audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0 - whisper_lang = self.language_to_service_language(self._settings["language"]) + whisper_lang = self.language_to_service_language(self._settings.language) segments, _ = await asyncio.to_thread( self._model.transcribe, audio_float, language=whisper_lang ) @@ -305,13 +329,13 @@ class WhisperSTTService(SegmentedSTTService): await self.stop_processing_metrics() if text: - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, ) @@ -347,12 +371,13 @@ class WhisperSTTServiceMLX(WhisperSTTService): self._no_speech_prob = no_speech_prob self._temperature = temperature - self._settings = { - "language": language, - "no_speech_prob": self._no_speech_prob, - "temperature": self._temperature, - "engine": "mlx", - } + self._settings: WhisperMLXSTTSettings = WhisperMLXSTTSettings( + model=model if isinstance(model, str) else model.value, + language=language, + no_speech_prob=self._no_speech_prob, + temperature=self._temperature, + engine="mlx", + ) # No need to call _load() as MLX Whisper loads models on demand @@ -390,7 +415,7 @@ class WhisperSTTServiceMLX(WhisperSTTService): # Divide by 32768 because we have signed 16-bit data. audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0 - whisper_lang = self.language_to_service_language(self._settings["language"]) + whisper_lang = self.language_to_service_language(self._settings.language) chunk = await asyncio.to_thread( mlx_whisper.transcribe, audio_float, @@ -413,13 +438,13 @@ class WhisperSTTServiceMLX(WhisperSTTService): await self.stop_processing_metrics() if text: - await self._handle_transcription(text, True, self._settings["language"]) + await self._handle_transcription(text, True, self._settings.language) logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame( text, self._user_id, time_now_iso8601(), - self._settings["language"], + self._settings.language, ) except Exception as e: diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index bf4eb4f03..664d3d4be 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -10,7 +10,8 @@ This module provides integration with Coqui XTTS streaming server for text-to-speech synthesis using local Docker deployment. """ -from typing import Any, AsyncGenerator, Dict, Optional +from dataclasses import dataclass, field +from typing import AsyncGenerator, Dict, Optional import aiohttp from loguru import logger @@ -24,6 +25,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, ) +from pipecat.services.settings import NOT_GIVEN, TTSSettings from pipecat.services.tts_service import TTSService from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts @@ -68,6 +70,17 @@ def language_to_xtts_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +@dataclass +class XTTSTTSSettings(TTSSettings): + """Typed settings for XTTS TTS service. + + Parameters: + base_url: Base URL of the XTTS streaming server. + """ + + base_url: str = field(default_factory=lambda: NOT_GIVEN) + + class XTTSService(TTSService): """Coqui XTTS text-to-speech service. @@ -98,11 +111,12 @@ class XTTSService(TTSService): """ super().__init__(sample_rate=sample_rate, **kwargs) - self._settings = { - "language": self.language_to_service_language(language), - "base_url": base_url, - } - self.set_voice(voice_id) + self._settings: XTTSTTSSettings = XTTSTTSSettings( + voice=voice_id, + language=self.language_to_service_language(language), + base_url=base_url, + ) + self._voice_id = voice_id self._studio_speakers: Optional[Dict[str, Any]] = None self._aiohttp_session = aiohttp_session @@ -138,7 +152,7 @@ class XTTSService(TTSService): if self._studio_speakers: return - async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r: + async with self._aiohttp_session.get(self._settings.base_url + "/studio_speakers") as r: if r.status != 200: text = await r.text() await self.push_error( @@ -166,11 +180,11 @@ class XTTSService(TTSService): embeddings = self._studio_speakers[self._voice_id] - url = self._settings["base_url"] + "/tts_stream" + url = self._settings.base_url + "/tts_stream" payload = { "text": text.replace(".", "").replace("*", ""), - "language": self._settings["language"], + "language": self._settings.language, "speaker_embedding": embeddings["speaker_embedding"], "gpt_cond_latent": embeddings["gpt_cond_latent"], "add_wav_header": False, diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 000000000..62583b00b --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,308 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Tests for the typed settings infrastructure in pipecat.services.settings.""" + +import pytest + +from pipecat.services.settings import ( + NOT_GIVEN, + LLMSettings, + ServiceSettings, + STTSettings, + TTSSettings, + _NotGiven, + is_given, +) + +# --------------------------------------------------------------------------- +# NOT_GIVEN sentinel +# --------------------------------------------------------------------------- + + +class TestNotGiven: + def test_singleton(self): + """NOT_GIVEN is a singleton — every reference is the same object.""" + assert _NotGiven() is _NotGiven() + assert NOT_GIVEN is _NotGiven() + + def test_repr(self): + assert repr(NOT_GIVEN) == "NOT_GIVEN" + + def test_bool_is_false(self): + assert not NOT_GIVEN + assert bool(NOT_GIVEN) is False + + def test_is_given_with_not_given(self): + assert is_given(NOT_GIVEN) is False + + def test_is_given_with_none(self): + assert is_given(None) is True + + def test_is_given_with_values(self): + assert is_given(0) is True + assert is_given("") is True + assert is_given(False) is True + assert is_given(42) is True + assert is_given("hello") is True + + +# --------------------------------------------------------------------------- +# ServiceSettings base +# --------------------------------------------------------------------------- + + +class TestServiceSettings: + def test_default_fields_are_not_given(self): + s = ServiceSettings() + assert not is_given(s.model) + assert s.extra == {} + + def test_given_fields_empty_by_default(self): + s = ServiceSettings() + assert s.given_fields() == {} + + def test_given_fields_includes_set_values(self): + s = ServiceSettings(model="gpt-4o") + assert s.given_fields() == {"model": "gpt-4o"} + + def test_given_fields_includes_extra(self): + s = ServiceSettings(model="gpt-4o") + s.extra = {"custom_key": 42} + result = s.given_fields() + assert result == {"model": "gpt-4o", "custom_key": 42} + + def test_to_dict(self): + s = ServiceSettings(model="gpt-4o") + assert s.to_dict() == {"model": "gpt-4o"} + + def test_copy_is_deep(self): + s = ServiceSettings(model="gpt-4o") + s.extra = {"nested": {"a": 1}} + c = s.copy() + assert c.model == "gpt-4o" + assert c.extra == {"nested": {"a": 1}} + # Mutating the copy shouldn't affect the original + c.extra["nested"]["a"] = 999 + assert s.extra["nested"]["a"] == 1 + + +# --------------------------------------------------------------------------- +# apply_update +# --------------------------------------------------------------------------- + + +class TestApplyUpdate: + def test_apply_update_basic(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="bob") + changed = current.apply_update(delta) + assert changed == {"voice"} + assert current.voice == "bob" + assert current.language == "en" + + def test_apply_update_no_change(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings(voice="alice") + changed = current.apply_update(delta) + assert changed == set() + assert current.voice == "alice" + + def test_apply_update_not_given_skipped(self): + current = TTSSettings(voice="alice", language="en") + delta = TTSSettings() # all NOT_GIVEN + changed = current.apply_update(delta) + assert changed == set() + assert current.voice == "alice" + assert current.language == "en" + + def test_apply_update_multiple_fields(self): + current = LLMSettings(temperature=0.7, max_tokens=100) + delta = LLMSettings(temperature=0.9, max_tokens=200, top_p=0.95) + changed = current.apply_update(delta) + assert changed == {"temperature", "max_tokens", "top_p"} + assert current.temperature == 0.9 + assert current.max_tokens == 200 + assert current.top_p == 0.95 + + def test_apply_update_extra_merged(self): + current = TTSSettings(voice="alice") + current.extra = {"speed": 1.0, "stability": 0.5} + delta = TTSSettings() + delta.extra = {"speed": 1.2} + changed = current.apply_update(delta) + assert "speed" in changed + assert current.extra == {"speed": 1.2, "stability": 0.5} + + def test_apply_update_extra_no_change(self): + current = TTSSettings(voice="alice") + current.extra = {"speed": 1.0} + delta = TTSSettings() + delta.extra = {"speed": 1.0} + changed = current.apply_update(delta) + assert changed == set() + + def test_apply_update_model_field(self): + current = ServiceSettings(model="old-model") + delta = ServiceSettings(model="new-model") + changed = current.apply_update(delta) + assert changed == {"model"} + assert current.model == "new-model" + + def test_apply_update_none_is_a_valid_value(self): + """Setting a field to None should be treated as a change from NOT_GIVEN.""" + current = TTSSettings() + delta = TTSSettings(language=None) + changed = current.apply_update(delta) + assert "language" in changed + assert current.language is None + + def test_apply_update_none_to_value(self): + current = TTSSettings(language=None) + delta = TTSSettings(language="en") + changed = current.apply_update(delta) + assert "language" in changed + assert current.language == "en" + + +# --------------------------------------------------------------------------- +# from_mapping +# --------------------------------------------------------------------------- + + +class TestFromMapping: + def test_basic_mapping(self): + s = TTSSettings.from_mapping({"voice": "alice", "language": "en"}) + assert s.voice == "alice" + assert s.language == "en" + assert not is_given(s.model) + + def test_alias_resolution(self): + """'voice_id' is an alias for 'voice' in TTSSettings.""" + s = TTSSettings.from_mapping({"voice_id": "alice"}) + assert s.voice == "alice" + + def test_unknown_keys_go_to_extra(self): + s = TTSSettings.from_mapping({"voice": "alice", "speed": 1.2, "stability": 0.5}) + assert s.voice == "alice" + assert s.extra == {"speed": 1.2, "stability": 0.5} + + def test_model_field(self): + s = LLMSettings.from_mapping({"model": "gpt-4o", "temperature": 0.7}) + assert s.model == "gpt-4o" + assert s.temperature == 0.7 + + def test_empty_mapping(self): + s = ServiceSettings.from_mapping({}) + assert s.given_fields() == {} + + def test_all_unknown_keys(self): + s = ServiceSettings.from_mapping({"foo": 1, "bar": 2}) + assert not is_given(s.model) + assert s.extra == {"foo": 1, "bar": 2} + + def test_llm_settings_from_mapping(self): + s = LLMSettings.from_mapping({"temperature": 0.5, "max_tokens": 1000, "custom_param": True}) + assert s.temperature == 0.5 + assert s.max_tokens == 1000 + assert s.extra == {"custom_param": True} + + def test_stt_settings_from_mapping(self): + s = STTSettings.from_mapping({"language": "fr", "model": "whisper-large"}) + assert s.language == "fr" + assert s.model == "whisper-large" + + +# --------------------------------------------------------------------------- +# LLMSettings specifics +# --------------------------------------------------------------------------- + + +class TestLLMSettings: + def test_all_fields_not_given_by_default(self): + s = LLMSettings() + for name in ( + "model", + "temperature", + "max_tokens", + "top_p", + "top_k", + "frequency_penalty", + "presence_penalty", + "seed", + ): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_given_fields(self): + s = LLMSettings(temperature=0.7, seed=42) + assert s.given_fields() == {"temperature": 0.7, "seed": 42} + + +# --------------------------------------------------------------------------- +# TTSSettings specifics +# --------------------------------------------------------------------------- + + +class TestTTSSettings: + def test_all_fields_not_given_by_default(self): + s = TTSSettings() + for name in ("model", "voice", "language"): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_aliases_class_var(self): + assert TTSSettings._aliases == {"voice_id": "voice"} + + def test_given_fields(self): + s = TTSSettings(voice="alice") + assert s.given_fields() == {"voice": "alice"} + + +# --------------------------------------------------------------------------- +# STTSettings specifics +# --------------------------------------------------------------------------- + + +class TestSTTSettings: + def test_all_fields_not_given_by_default(self): + s = STTSettings() + for name in ("model", "language"): + assert not is_given(getattr(s, name)), f"{name} should be NOT_GIVEN" + + def test_given_fields(self): + s = STTSettings(language="en", model="whisper-large") + assert s.given_fields() == {"language": "en", "model": "whisper-large"} + + +# --------------------------------------------------------------------------- +# Integration: roundtrip from_mapping → apply_update +# --------------------------------------------------------------------------- + + +class TestRoundtrip: + def test_from_mapping_then_apply_update(self): + """Simulate the real flow: dict arrives via frame, gets converted, applied.""" + # Simulating current service state + current = TTSSettings(model="eleven_turbo_v2_5", voice="alice", language="en") + current.extra = {"stability": 0.5, "speed": 1.0} + + # Incoming dict-based update + raw = {"voice_id": "bob", "speed": 1.2} + delta = TTSSettings.from_mapping(raw) + + changed = current.apply_update(delta) + assert changed == {"voice", "speed"} + assert current.voice == "bob" + assert current.language == "en" + assert current.extra["speed"] == 1.2 + assert current.extra["stability"] == 0.5 + + def test_from_mapping_preserves_model(self): + current = LLMSettings(model="gpt-4o", temperature=0.7) + delta = LLMSettings.from_mapping({"model": "gpt-4o-mini", "temperature": 0.9}) + changed = current.apply_update(delta) + assert changed == {"model", "temperature"} + assert current.model == "gpt-4o-mini" + assert current.temperature == 0.9