wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3714 from pipecat-ai/pk/service-settings-refactor

Browse Source 
Broad refactor of service settings and how they’re updated at runtime
This commit is contained in:
kompfner
2026-02-23 17:15:15 -05:00
committed by GitHub
parent 6a7e9358c6 71fc078c24
commit cdd65b6c0a
172 changed files with 16024 additions and 2155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/pipecat/frames/frames.py
View File

@@ -42,6 +42,7 @@ from pipecat.utils.utils import obj_count, obj_id
if TYPE_CHECKING:
from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.services.settings import ServiceSettings
from pipecat.utils.tracing.tracing_context import TracingContext
@@ -2120,13 +2121,21 @@ class TTSStoppedFrame(ControlFrame):
class ServiceUpdateSettingsFrame(ControlFrame):
"""Base frame for updating service settings.
A control frame containing a request to update service settings.
Supports both a ``settings`` dict (for backward compatibility) and an
``update`` object. When both are provided, ``update`` takes precedence.
Parameters:
settings: Dictionary of setting name to value mappings.
.. deprecated:: 0.0.103
Use ``update`` with a typed settings object instead.
update: :class:`~pipecat.services.settings.ServiceSettings` object
describing the delta to apply.
"""
settings: Mapping[str, Any]
settings: Mapping[str, Any] = field(default_factory=dict)
update: Optional["ServiceSettings"] = None
@dataclass

92
src/pipecat/services/ai_service.py
View File

@@ -10,7 +10,7 @@ Provides the foundation for all AI services in the Pipecat framework, including
model management, settings handling, and frame processing lifecycle methods.
"""
from typing import Any, AsyncGenerator, Dict, Mapping
from typing import Any, AsyncGenerator, Dict
from loguru import logger
@@ -23,6 +23,7 @@ from pipecat.frames.frames import (
)
from pipecat.metrics.metrics import MetricsData
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.settings import ServiceSettings
class AIService(FrameProcessor):
@@ -41,29 +42,27 @@ class AIService(FrameProcessor):
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
self._model_name: str = ""
self._settings: Dict[str, Any] = {}
self._settings: ServiceSettings = ServiceSettings(model="")
self._session_properties: Dict[str, Any] = {}
self._tracing_enabled: bool = False
self._tracing_context = None
@property
def model_name(self) -> str:
"""Get the current model name.
def _sync_model_name_to_metrics(self):
"""Sync the current AI model name (in `self._settings.model`) for usage in metrics.
Returns:
The name of the AI model being used.
"""
return self._model_name
We don't store model name here because there's already a single source
of truth for it in `self._settings.model`. This method is just for
syncing the model name to the metrics data.
def set_model_name(self, model: str):
"""Set the AI model name and update metrics.
TODO: as a next step we should make it so that service classes pass
model into `super().__init__` and `AIService` can be responsible for
syncing its initial value to metrics, just as it's responsible for
syncing any updates to its value to metrics via `_update_settings`.
Args:
model: The name of the AI model to use.
"""
self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._settings.model))
async def start(self, frame: StartFrame):
"""Start the AI service.
@@ -99,44 +98,45 @@ class AIService(FrameProcessor):
"""
pass
async def _update_settings(self, settings: Mapping[str, Any]):
from pipecat.services.openai.realtime.events import SessionProperties
async def _update_settings(self, update: ServiceSettings) -> Dict[str, Any]:
"""Apply a settings update and return the changed fields.
for key, value in settings.items():
logger.debug("Update request for:", key, value)
The update is applied to ``_settings`` and a dict mapping each changed
field name to its **pre-update** value is returned. The ``model``
field is handled specially: when it changes, ``set_model_name`` is
called.
if key in self._settings:
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._settings[key] = value
elif key in SessionProperties.model_fields:
logger.debug("Attempting to update", key, value)
Concrete services should override this method (calling ``super()``)
to react to specific changed fields (e.g. reconnect on voice change).
try:
from pipecat.services.openai.realtime.events import TurnDetection
Args:
update: A settings delta.
if isinstance(self._session_properties, SessionProperties):
current_properties = self._session_properties
else:
current_properties = SessionProperties(**self._session_properties)
Returns:
Dict mapping changed field names to their previous values.
"""
changed = self._settings.apply_update(update)
if key == "turn_detection" and isinstance(value, dict):
turn_detection = TurnDetection(**value)
setattr(current_properties, key, turn_detection)
else:
setattr(current_properties, key, value)
if "model" in changed:
self._sync_model_name_to_metrics()
validated_properties = SessionProperties.model_validate(
current_properties.model_dump()
)
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._session_properties = validated_properties.model_dump()
except Exception as e:
logger.warning(f"Unexpected error updating session property {key}: {e}")
elif key == "model":
logger.info(f"Updating LLM setting {key} to: [{value}]")
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for {self.name} service: {key}")
if changed:
logger.info(f"{self.name}: updated settings fields: {set(changed)}")
return changed
def _warn_unhandled_updated_settings(self, unhandled):
"""Log a warning for settings changes that won't take effect at runtime.
Convenience helper for ``_update_settings`` overrides. Accepts any
iterable of field names (a ``dict``, ``set``, ``dict_keys``, etc.).
Args:
unhandled: Field names that changed but are not applied.
"""
if unhandled:
fields = ", ".join(sorted(unhandled))
logger.warning(f"{self.name}: runtime update of [{fields}] is not currently supported")
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and handle service lifecycle.

139
src/pipecat/services/anthropic/llm.py
View File

@@ -16,8 +16,8 @@ import copy
import io
import json
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, Union
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Literal, Optional, Union
import httpx
from loguru import logger
@@ -42,7 +42,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
UserImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -59,6 +58,8 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
from pipecat.services.settings import LLMSettings, _NotGiven, is_given
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -69,6 +70,50 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
class AnthropicThinkingConfig(BaseModel):
"""Configuration for extended thinking.
Parameters:
type: Type of thinking mode (currently only "enabled" or "disabled").
budget_tokens: Maximum number of tokens for thinking.
With today's models, the minimum is 1024.
Only allowed if type is "enabled".
"""
# Why `| str` here? To not break compatibility in case Anthropic adds
# more types in the future.
type: Literal["enabled", "disabled"] | str
# Why not enforce minimnum of 1024 here? To not break compatibility in
# case Anthropic changes this requirement in the future.
budget_tokens: int
@dataclass
class AnthropicLLMSettings(LLMSettings):
"""Settings for Anthropic LLM services.
Parameters:
enable_prompt_caching: Whether to enable prompt caching.
thinking: Extended thinking configuration.
"""
enable_prompt_caching: bool | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
thinking: AnthropicThinkingConfig | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
@classmethod
def from_mapping(cls, settings):
"""Convert a plain dict to settings, coercing thinking dicts.
For backward compatibility, a ``thinking`` value that is a plain dict
is converted to a :class:`AnthropicThinkingConfig`.
"""
instance = super().from_mapping(settings)
if is_given(instance.thinking) and isinstance(instance.thinking, dict):
instance.thinking = AnthropicThinkingConfig(**instance.thinking)
return instance
@dataclass
class AnthropicContextAggregatorPair:
"""Pair of context aggregators for Anthropic conversations.
@@ -115,26 +160,13 @@ class AnthropicLLMService(LLMService):
Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex.
"""
_settings: AnthropicLLMSettings
# Overriding the default adapter to use the Anthropic one.
adapter_class = AnthropicLLMAdapter
class ThinkingConfig(BaseModel):
"""Configuration for extended thinking.
Parameters:
type: Type of thinking mode (currently only "enabled" or "disabled").
budget_tokens: Maximum number of tokens for thinking.
With today's models, the minimum is 1024.
Only allowed if type is "enabled".
"""
# Why `| str` here? To not break compatibility in case Anthropic adds
# more types in the future.
type: Literal["enabled", "disabled"] | str
# Why not enforce minimnum of 1024 here? To not break compatibility in
# case Anthropic changes this requirement in the future.
budget_tokens: int
# Backward compatibility: ThinkingConfig used to be defined inline here.
ThinkingConfig = AnthropicThinkingConfig
class InputParams(BaseModel):
"""Input parameters for Anthropic model inference.
@@ -163,9 +195,7 @@ class AnthropicLLMService(LLMService):
temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
top_k: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0)
top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
thinking: Optional["AnthropicLLMService.ThinkingConfig"] = Field(
default_factory=lambda: NOT_GIVEN
)
thinking: Optional[AnthropicThinkingConfig] = Field(default_factory=lambda: NOT_GIVEN)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def model_post_init(self, __context):
@@ -207,12 +237,12 @@ class AnthropicLLMService(LLMService):
self._client = client or AsyncAnthropic(
api_key=api_key
) # if the client is provided, use it and remove it, otherwise create a new one
self.set_model_name(model)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = {
"max_tokens": params.max_tokens,
"enable_prompt_caching": (
self._settings = AnthropicLLMSettings(
model=model,
max_tokens=params.max_tokens,
enable_prompt_caching=(
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
@@ -221,12 +251,13 @@ class AnthropicLLMService(LLMService):
else False
)
),
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"thinking": params.thinking,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate usage metrics.
@@ -280,7 +311,7 @@ class AnthropicLLMService(LLMService):
if isinstance(context, LLMContext):
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
invocation_params = adapter.get_llm_invocation_params(
context, enable_prompt_caching=self._settings["enable_prompt_caching"]
context, enable_prompt_caching=self._settings.enable_prompt_caching
)
messages = invocation_params["messages"]
system = invocation_params["system"]
@@ -293,21 +324,21 @@ class AnthropicLLMService(LLMService):
# Build params using the same method as streaming completions
params = {
"model": self.model_name,
"max_tokens": max_tokens if max_tokens is not None else self._settings["max_tokens"],
"model": self._settings.model,
"max_tokens": max_tokens if max_tokens is not None else self._settings.max_tokens,
"stream": False,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"temperature": self._settings.temperature,
"top_k": self._settings.top_k,
"top_p": self._settings.top_p,
"messages": messages,
"system": system,
"tools": tools,
"betas": ["interleaved-thinking-2025-05-14"],
}
if self._settings["thinking"]:
params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
if self._settings.thinking:
params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
params.update(self._settings["extra"])
params.update(self._settings.extra)
# LLM completion
response = await self._client.beta.messages.create(**params)
@@ -358,14 +389,14 @@ class AnthropicLLMService(LLMService):
if isinstance(context, LLMContext):
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
params = adapter.get_llm_invocation_params(
context, enable_prompt_caching=self._settings["enable_prompt_caching"]
context, enable_prompt_caching=self._settings.enable_prompt_caching
)
return params
# Anthropic-specific context
messages = (
context.get_messages_with_cache_control_markers()
if self._settings["enable_prompt_caching"]
if self._settings.enable_prompt_caching
else context.messages
)
return AnthropicLLMInvocationParams(
@@ -407,22 +438,22 @@ class AnthropicLLMService(LLMService):
await self.start_ttfb_metrics()
params = {
"model": self.model_name,
"max_tokens": self._settings["max_tokens"],
"model": self._settings.model,
"max_tokens": self._settings.max_tokens,
"stream": True,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"temperature": self._settings.temperature,
"top_k": self._settings.top_k,
"top_p": self._settings.top_p,
}
# Add thinking parameter if set
if self._settings["thinking"]:
params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
if self._settings.thinking:
params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
# Messages, system, tools
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
# "Interleaved thinking" needed to allow thinking between sequences
# of function calls, when extended thinking is enabled.
@@ -576,11 +607,9 @@ class AnthropicLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = AnthropicLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMEnablePromptCachingFrame):
logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
self._settings["enable_prompt_caching"] = frame.enable
self._settings.enable_prompt_caching = frame.enable
else:
await self.push_frame(frame, direction)

66
src/pipecat/services/assemblyai/stt.py
View File

@@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription.
import asyncio
import json
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional
from urllib.parse import urlencode
@@ -29,6 +30,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import ASSEMBLYAI_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -52,6 +54,21 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AssemblyAISTTSettings(STTSettings):
"""Settings for the AssemblyAI STT service.
See :class:`AssemblyAIConnectionParams` for detailed parameter descriptions.
Parameters:
connection_params: Connection configuration parameters.
"""
connection_params: AssemblyAIConnectionParams | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class AssemblyAISTTService(WebsocketSTTService):
"""AssemblyAI real-time speech-to-text service.
@@ -60,6 +77,8 @@ class AssemblyAISTTService(WebsocketSTTService):
for audio processing and connection management.
"""
_settings: AssemblyAISTTSettings
def __init__(
self,
*,
@@ -96,9 +115,11 @@ class AssemblyAISTTService(WebsocketSTTService):
)
self._api_key = api_key
self._language = language
self._settings = AssemblyAISTTSettings(
language=language,
connection_params=connection_params,
)
self._api_endpoint_base_url = api_endpoint_base_url
self._connection_params = connection_params
self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._termination_event = asyncio.Event()
@@ -165,6 +186,37 @@ class AssemblyAISTTService(WebsocketSTTService):
"""
return True
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
Args:
update: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# # Re-apply manual turn mode config if vad_force_turn_endpoint is active
# # and connection_params were updated.
# if self._vad_force_turn_endpoint and "connection_params" in changed:
# self._settings.connection_params = self._configure_manual_turn_mode(
# self._settings.connection_params
# )
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
@@ -239,7 +291,7 @@ class AssemblyAISTTService(WebsocketSTTService):
def _build_ws_url(self) -> str:
"""Build WebSocket URL with query parameters using urllib.parse.urlencode."""
params = {}
for k, v in self._connection_params.model_dump().items():
for k, v in self._settings.connection_params.model_dump().items():
if v is not None:
if k == "keyterms_prompt":
params[k] = json.dumps(v)
@@ -415,18 +467,18 @@ class AssemblyAISTTService(WebsocketSTTService):
if not message.transcript:
return
if message.end_of_turn and (
not self._connection_params.formatted_finals or message.turn_is_formatted
not self._settings.connection_params.formatted_finals or message.turn_is_formatted
):
await self.push_frame(
TranscriptionFrame(
message.transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
message,
)
)
await self._trace_transcription(message.transcript, True, self._language)
await self._trace_transcription(message.transcript, True, self._settings.language)
await self.stop_processing_metrics()
else:
await self.push_frame(
@@ -434,7 +486,7 @@ class AssemblyAISTTService(WebsocketSTTService):
message.transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
message,
)
)

115
src/pipecat/services/asyncai/tts.py
View File

@@ -9,7 +9,8 @@
import asyncio
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Mapping, Optional
import aiohttp
from loguru import logger
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -72,12 +74,40 @@ def language_to_async_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class AsyncAITTSSettings(TTSSettings):
"""Settings for Async AI TTS services.
Parameters:
output_container: Audio container format (e.g. "raw").
output_encoding: Audio encoding format (e.g. "pcm_s16le").
output_sample_rate: Audio sample rate in Hz.
"""
output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "AsyncAITTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``output_format``."""
flat = dict(settings)
nested = flat.pop("output_format", None)
if isinstance(nested, dict):
flat.setdefault("output_container", nested.get("container"))
flat.setdefault("output_encoding", nested.get("encoding"))
flat.setdefault("output_sample_rate", nested.get("sample_rate"))
return super().from_mapping(flat)
class AsyncAITTSService(AudioContextTTSService):
"""Async TTS service with WebSocket streaming.
Provides text-to-speech using Async's streaming WebSocket API.
"""
_settings: AsyncAITTSSettings
class InputParams(BaseModel):
"""Input parameters for Async TTS configuration.
@@ -131,23 +161,35 @@ class AsyncAITTSService(AudioContextTTSService):
self._api_key = api_key
self._api_version = version
self._url = url
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
self._settings = AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
}
self.set_model_name(model)
self.set_voice(voice_id)
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._keepalive_task = None
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
self._warn_unhandled_updated_settings(changed)
return changed
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -178,7 +220,7 @@ class AsyncAITTSService(AudioContextTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -232,10 +274,14 @@ class AsyncAITTSService(AudioContextTTSService):
f"{self._url}?api_key={self._api_key}&version={self._api_version}"
)
init_msg = {
"model_id": self._model_name,
"voice": {"mode": "id", "id": self._voice_id},
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"model_id": self._settings.model,
"voice": {"mode": "id", "id": self._settings.voice},
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"language": self._settings.language,
}
await self._get_websocket().send(json.dumps(init_msg))
@@ -404,6 +450,8 @@ class AsyncAIHttpTTSService(TTSService):
connection is not required or desired.
"""
_settings: AsyncAITTSSettings
class InputParams(BaseModel):
"""Input parameters for Async API.
@@ -450,18 +498,17 @@ class AsyncAIHttpTTSService(TTSService):
self._api_key = api_key
self._base_url = url
self._api_version = version
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
self._settings = AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
}
self.set_voice(voice_id)
self.set_model_name(model)
)
self._sync_model_name_to_metrics()
self._session = aiohttp_session
@@ -491,7 +538,7 @@ class AsyncAIHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -507,14 +554,18 @@ class AsyncAIHttpTTSService(TTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
voice_config = {"mode": "id", "id": self._voice_id}
voice_config = {"mode": "id", "id": self._settings.voice}
await self.start_ttfb_metrics()
payload = {
"model_id": self._model_name,
"model_id": self._settings.model,
"transcript": text,
"voice": voice_config,
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"language": self._settings.language,
}
yield TTSStartedFrame(context_id=context_id)
headers = {

66
src/pipecat/services/aws/llm.py
View File

@@ -18,8 +18,8 @@ import io
import json
import os
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Optional
from loguru import logger
from PIL import Image
@@ -40,7 +40,6 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
UserImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -57,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -71,6 +71,21 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AWSBedrockLLMSettings(LLMSettings):
"""Settings for AWS Bedrock LLM services.
Parameters:
latency: Performance mode - "standard" or "optimized".
additional_model_request_fields: Additional model-specific parameters.
"""
latency: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
additional_model_request_fields: Dict[str, Any] | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
@dataclass
class AWSBedrockContextAggregatorPair:
"""Container for AWS Bedrock context aggregators.
@@ -730,6 +745,8 @@ class AWSBedrockLLMService(LLMService):
vision capabilities.
"""
_settings: AWSBedrockLLMSettings
# Overriding the default adapter to use the Anthropic one.
adapter_class = AWSBedrockLLMAdapter
@@ -803,18 +820,19 @@ class AWSBedrockLLMService(LLMService):
"config": client_config,
}
self.set_model_name(model)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
"top_p": params.top_p,
"latency": params.latency,
"additional_model_request_fields": params.additional_model_request_fields
self._settings = AWSBedrockLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_p=params.top_p,
latency=params.latency,
additional_model_request_fields=params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
}
)
self._sync_model_name_to_metrics()
logger.info(f"Using AWS Bedrock model: {model}")
@@ -836,12 +854,12 @@ class AWSBedrockLLMService(LLMService):
Dictionary containing only the inference parameters that are not None.
"""
inference_config = {}
if self._settings["max_tokens"] is not None:
inference_config["maxTokens"] = self._settings["max_tokens"]
if self._settings["temperature"] is not None:
inference_config["temperature"] = self._settings["temperature"]
if self._settings["top_p"] is not None:
inference_config["topP"] = self._settings["top_p"]
if self._settings.max_tokens is not None:
inference_config["maxTokens"] = self._settings.max_tokens
if self._settings.temperature is not None:
inference_config["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
inference_config["topP"] = self._settings.top_p
return inference_config
async def run_inference(
@@ -877,9 +895,9 @@ class AWSBedrockLLMService(LLMService):
inference_config["maxTokens"] = max_tokens
request_params = {
"modelId": self.model_name,
"modelId": self._settings.model,
"messages": messages,
"additionalModelRequestFields": self._settings["additional_model_request_fields"],
"additionalModelRequestFields": self._settings.additional_model_request_fields,
}
if inference_config:
@@ -1034,9 +1052,9 @@ class AWSBedrockLLMService(LLMService):
# Prepare request parameters
request_params = {
"modelId": self.model_name,
"modelId": self._settings.model,
"messages": messages,
"additionalModelRequestFields": self._settings["additional_model_request_fields"],
"additionalModelRequestFields": self._settings.additional_model_request_fields,
}
# Only add inference config if it has parameters
@@ -1081,8 +1099,8 @@ class AWSBedrockLLMService(LLMService):
request_params["toolConfig"] = tool_config
# Add performance config if latency is specified
if self._settings["latency"] in ["standard", "optimized"]:
request_params["performanceConfig"] = {"latency": self._settings["latency"]}
if self._settings.latency in ["standard", "optimized"]:
request_params["performanceConfig"] = {"latency": self._settings.latency}
# Log request params with messages redacted for logging
if isinstance(context, LLMContext):
@@ -1207,8 +1225,6 @@ class AWSBedrockLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = AWSBedrockLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

107
src/pipecat/services/aws/nova_sonic/llm.py
View File

@@ -16,7 +16,7 @@ import json
import time
import uuid
import wave
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from importlib.resources import files
from typing import Any, List, Optional
@@ -60,6 +60,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
try:
@@ -185,6 +186,20 @@ class Params(BaseModel):
endpointing_sensitivity: Optional[str] = Field(default=None)
@dataclass
class AWSNovaSonicLLMSettings(LLMSettings):
"""Settings for AWS Nova Sonic LLM service.
Parameters:
voice_id: Voice for speech synthesis.
endpointing_sensitivity: Controls how quickly Nova Sonic decides the
user has stopped speaking. Can be "LOW", "MEDIUM", or "HIGH".
"""
voice_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSNovaSonicLLMService(LLMService):
"""AWS Nova Sonic speech-to-speech LLM service.
@@ -192,6 +207,8 @@ class AWSNovaSonicLLMService(LLMService):
and function calling capabilities using AWS Nova Sonic model.
"""
_settings: AWSNovaSonicLLMSettings
# Override the default adapter to use the AWSNovaSonicLLMAdapter one
adapter_class = AWSNovaSonicLLMAdapter
@@ -242,23 +259,38 @@ class AWSNovaSonicLLMService(LLMService):
self._access_key_id = access_key_id
self._session_token = session_token
self._region = region
self._model = model
self._client: Optional[BedrockRuntimeClient] = None
self._voice_id = voice_id
self._params = params or Params()
params = params or Params()
self._settings = AWSNovaSonicLLMSettings(
model=model,
voice_id=voice_id,
temperature=params.temperature,
max_tokens=params.max_tokens,
top_p=params.top_p,
endpointing_sensitivity=params.endpointing_sensitivity,
)
self._sync_model_name_to_metrics()
# Audio I/O config (hardware settings, not runtime-tunable)
self._input_sample_rate = params.input_sample_rate
self._input_sample_size = params.input_sample_size
self._input_channel_count = params.input_channel_count
self._output_sample_rate = params.output_sample_rate
self._output_sample_size = params.output_sample_size
self._output_channel_count = params.output_channel_count
self._system_instruction = system_instruction
self._tools = tools
# Validate endpointing_sensitivity parameter
if (
self._params.endpointing_sensitivity
self._settings.endpointing_sensitivity
and not self._is_endpointing_sensitivity_supported()
):
logger.warning(
f"endpointing_sensitivity is not supported for model '{model}' and will be ignored. "
"This parameter is only supported starting with Nova 2 Sonic (amazon.nova-2-sonic-v1:0)."
)
self._params.endpointing_sensitivity = None
self._settings.endpointing_sensitivity = None
if not send_transcription_frames:
import warnings
@@ -302,6 +334,29 @@ class AWSNovaSonicLLMService(LLMService):
with wave.open(file_path.open("rb"), "rb") as wav_file:
self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes())
#
# settings
#
async def _update_settings(self, update: AWSNovaSonicLLMSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._start_connecting()
self._warn_unhandled_updated_settings(changed)
return changed
#
# standard AIService frame handling
#
@@ -472,7 +527,7 @@ class AWSNovaSonicLLMService(LLMService):
# Start the bidirectional stream
self._stream = await self._client.invoke_model_with_bidirectional_stream(
InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model)
InvokeModelWithBidirectionalStreamOperationInput(model_id=self._settings.model)
)
# Send session start event
@@ -639,7 +694,7 @@ class AWSNovaSonicLLMService(LLMService):
def _is_first_generation_sonic_model(self) -> bool:
# Nova Sonic (the older model) is identified by "amazon.nova-sonic-v1:0"
return self._model == "amazon.nova-sonic-v1:0"
return self._settings.model == "amazon.nova-sonic-v1:0"
def _is_endpointing_sensitivity_supported(self) -> bool:
# endpointing_sensitivity is only supported with Nova 2 Sonic (and,
@@ -658,9 +713,9 @@ class AWSNovaSonicLLMService(LLMService):
turn_detection_config = (
f""",
"turnDetectionConfiguration": {{
"endpointingSensitivity": "{self._params.endpointing_sensitivity}"
"endpointingSensitivity": "{self._settings.endpointing_sensitivity}"
}}"""
if self._params.endpointing_sensitivity
if self._settings.endpointing_sensitivity
else ""
)
@@ -669,9 +724,9 @@ class AWSNovaSonicLLMService(LLMService):
"event": {{
"sessionStart": {{
"inferenceConfiguration": {{
"maxTokens": {self._params.max_tokens},
"topP": {self._params.top_p},
"temperature": {self._params.temperature}
"maxTokens": {self._settings.max_tokens},
"topP": {self._settings.top_p},
"temperature": {self._settings.temperature}
}}{turn_detection_config}
}}
}}
@@ -706,10 +761,10 @@ class AWSNovaSonicLLMService(LLMService):
}},
"audioOutputConfiguration": {{
"mediaType": "audio/lpcm",
"sampleRateHertz": {self._params.output_sample_rate},
"sampleSizeBits": {self._params.output_sample_size},
"channelCount": {self._params.output_channel_count},
"voiceId": "{self._voice_id}",
"sampleRateHertz": {self._output_sample_rate},
"sampleSizeBits": {self._output_sample_size},
"channelCount": {self._output_channel_count},
"voiceId": "{self._settings.voice_id}",
"encoding": "base64",
"audioType": "SPEECH"
}}{tools_config}
@@ -734,9 +789,9 @@ class AWSNovaSonicLLMService(LLMService):
"role": "USER",
"audioInputConfiguration": {{
"mediaType": "audio/lpcm",
"sampleRateHertz": {self._params.input_sample_rate},
"sampleSizeBits": {self._params.input_sample_size},
"channelCount": {self._params.input_channel_count},
"sampleRateHertz": {self._input_sample_rate},
"sampleSizeBits": {self._input_sample_size},
"channelCount": {self._input_channel_count},
"audioType": "SPEECH",
"encoding": "base64"
}}
@@ -1019,8 +1074,8 @@ class AWSNovaSonicLLMService(LLMService):
audio = base64.b64decode(audio_content)
frame = TTSAudioRawFrame(
audio=audio,
sample_rate=self._params.output_sample_rate,
num_channels=self._params.output_channel_count,
sample_rate=self._output_sample_rate,
num_channels=self._output_channel_count,
)
await self.push_frame(frame)
@@ -1304,7 +1359,7 @@ class AWSNovaSonicLLMService(LLMService):
"""
if not self._is_assistant_response_trigger_needed():
logger.warning(
f"Assistant response trigger not needed for model '{self._model}'; skipping. "
f"Assistant response trigger not needed for model '{self._settings.model}'; skipping. "
"An LLMRunFrame() should be sufficient to prompt the assistant to respond, "
"assuming the context ends in a user message."
)
@@ -1332,9 +1387,9 @@ class AWSNovaSonicLLMService(LLMService):
chunk_duration = 0.02 # what we might get from InputAudioRawFrame
chunk_size = int(
chunk_duration
* self._params.input_sample_rate
* self._params.input_channel_count
* (self._params.input_sample_size / 8)
* self._input_sample_rate
* self._input_channel_count
* (self._input_sample_size / 8)
) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes
# Lead with a bit of blank audio, if needed.

83
src/pipecat/services/aws/stt.py
View File

@@ -14,7 +14,8 @@ import json
import os
import random
import string
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -28,6 +29,7 @@ from pipecat.frames.frames import (
TranscriptionFrame,
)
from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import AWS_TRANSCRIBE_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -43,6 +45,25 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AWSTranscribeSTTSettings(STTSettings):
"""Settings for the AWS Transcribe STT service.
Parameters:
sample_rate: Audio sample rate in Hz (8000 or 16000).
media_encoding: Audio encoding format (e.g. "linear16").
number_of_channels: Number of audio channels.
show_speaker_label: Whether to show speaker labels.
enable_channel_identification: Whether to enable channel identification.
"""
sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
media_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
number_of_channels: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
show_speaker_label: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_channel_identification: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSTranscribeSTTService(WebsocketSTTService):
"""AWS Transcribe Speech-to-Text service using WebSocket streaming.
@@ -51,6 +72,8 @@ class AWSTranscribeSTTService(WebsocketSTTService):
final transcription results.
"""
_settings: AWSTranscribeSTTSettings
def __init__(
self,
*,
@@ -78,21 +101,21 @@ class AWSTranscribeSTTService(WebsocketSTTService):
"""
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._settings = {
"sample_rate": sample_rate,
"language": language,
"media_encoding": "linear16", # AWS expects raw PCM
"number_of_channels": 1,
"show_speaker_label": False,
"enable_channel_identification": False,
}
self._settings = AWSTranscribeSTTSettings(
language=self.language_to_service_language(language) or "en-US",
sample_rate=sample_rate,
media_encoding="linear16",
number_of_channels=1,
show_speaker_label=False,
enable_channel_identification=False,
)
# Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz
if sample_rate not in [8000, 16000]:
logger.warning(
f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz."
)
self._settings["sample_rate"] = 16000
self._settings.sample_rate = 16000
self._credentials = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -117,6 +140,26 @@ class AWSTranscribeSTTService(WebsocketSTTService):
}
return encoding_map.get(encoding, encoding)
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if changed and self._websocket:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Initialize the connection when the service starts.
@@ -208,9 +251,9 @@ class AWSTranscribeSTTService(WebsocketSTTService):
logger.debug("Connecting to AWS Transcribe WebSocket")
language_code = self.language_to_service_language(Language(self._settings["language"]))
language_code = self._settings.language
if not language_code:
raise ValueError(f"Unsupported language: {self._settings['language']}")
raise ValueError(f"Unsupported language: {language_code}")
# Generate random websocket key
websocket_key = "".join(
@@ -237,14 +280,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
},
language_code=language_code,
media_encoding=self.get_service_encoding(
self._settings["media_encoding"]
self._settings.media_encoding
), # Convert to AWS format
sample_rate=self._settings["sample_rate"],
number_of_channels=self._settings["number_of_channels"],
sample_rate=self._settings.sample_rate,
number_of_channels=self._settings.number_of_channels,
enable_partial_results_stabilization=True,
partial_results_stability="high",
show_speaker_label=self._settings["show_speaker_label"],
enable_channel_identification=self._settings["enable_channel_identification"],
show_speaker_label=self._settings.show_speaker_label,
enable_channel_identification=self._settings.enable_channel_identification,
)
logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...")
@@ -479,14 +522,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
result=result,
)
)
await self._handle_transcription(
transcript,
is_final,
self._settings["language"],
self._settings.language,
)
await self.stop_processing_metrics()
else:
@@ -495,7 +538,7 @@ class AWSTranscribeSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
result=result,
)
)

64
src/pipecat/services/aws/tts.py
View File

@@ -11,6 +11,7 @@ supporting multiple languages, voices, and SSML features.
"""
import os
from dataclasses import dataclass, field
from typing import AsyncGenerator, List, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -121,6 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class AWSPollyTTSSettings(TTSSettings):
"""Settings for AWS Polly TTS service.
Parameters:
engine: TTS engine to use ('standard', 'neural', etc.).
pitch: Voice pitch adjustment (for standard engine only).
rate: Speech rate adjustment.
volume: Voice volume adjustment.
lexicon_names: List of pronunciation lexicons to apply.
"""
engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
lexicon_names: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSPollyTTSService(TTSService):
"""AWS Polly text-to-speech service.
@@ -129,6 +150,8 @@ class AWSPollyTTSService(TTSService):
options including prosody controls.
"""
_settings: AWSPollyTTSSettings
class InputParams(BaseModel):
"""Input parameters for AWS Polly TTS configuration.
@@ -185,21 +208,20 @@ class AWSPollyTTSService(TTSService):
}
self._aws_session = aioboto3.Session()
self._settings = {
"engine": params.engine,
"language": self.language_to_service_language(params.language)
self._settings = AWSPollyTTSSettings(
voice=voice_id,
engine=params.engine,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
"lexicon_names": params.lexicon_names,
}
pitch=params.pitch,
rate=params.rate,
volume=params.volume,
lexicon_names=params.lexicon_names,
)
self._resampler = create_stream_resampler()
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -222,19 +244,19 @@ class AWSPollyTTSService(TTSService):
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
language = self._settings["language"]
language = self._settings.language
ssml += f"<lang xml:lang='{language}'>"
prosody_attrs = []
# Prosody tags are only supported for standard and neural engines
if self._settings["engine"] == "standard":
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings.engine == "standard":
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
@@ -275,11 +297,11 @@ class AWSPollyTTSService(TTSService):
"Text": ssml,
"TextType": "ssml",
"OutputFormat": "pcm",
"VoiceId": self._voice_id,
"Engine": self._settings["engine"],
"VoiceId": self._settings.voice,
"Engine": self._settings.engine,
# AWS only supports 8000 and 16000 for PCM. We select 16000.
"SampleRate": "16000",
"LexiconNames": self._settings["lexicon_names"],
"LexiconNames": self._settings.lexicon_names,
}
# Filter out None values

3
src/pipecat/services/azure/image.py
View File

@@ -54,7 +54,8 @@ class AzureImageGenServiceREST(ImageGenService):
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self.set_model_name(model)
self._settings.model = model
self._sync_model_name_to_metrics()
self._image_size = image_size
self._aiohttp_session = aiohttp_session

65
src/pipecat/services/azure/stt.py
View File

@@ -11,7 +11,8 @@ Speech SDK for real-time audio transcription.
"""
import asyncio
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -25,6 +26,7 @@ from pipecat.frames.frames import (
TranscriptionFrame,
)
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import AZURE_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -48,6 +50,19 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AzureSTTSettings(STTSettings):
"""Settings for the Azure STT service.
Parameters:
region: Azure region for the Speech service.
sample_rate: Audio sample rate in Hz.
"""
region: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AzureSTTService(STTService):
"""Azure Speech-to-Text service for real-time audio transcription.
@@ -56,6 +71,8 @@ class AzureSTTService(STTService):
provides real-time transcription results with timing information.
"""
_settings: AzureSTTSettings
def __init__(
self,
*,
@@ -92,11 +109,11 @@ class AzureSTTService(STTService):
self._audio_stream = None
self._speech_recognizer = None
self._settings = {
"region": region,
"language": language_to_azure_language(language),
"sample_rate": sample_rate,
}
self._settings = AzureSTTSettings(
region=region,
language=language_to_azure_language(language),
sample_rate=sample_rate,
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate performance metrics.
@@ -106,6 +123,38 @@ class AzureSTTService(STTService):
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Azure service-specific language code.
Args:
language: The language to convert.
Returns:
The Azure-specific language identifier, or None if not supported.
"""
return language_to_azure_language(language)
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active recognizer.
"""
changed = await super()._update_settings(update)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if "language" in changed:
# self._speech_config.speech_recognition_language = self._settings.language
# if self._speech_recognizer:
# # Requires refactoring to set up and tear down recognizer, as
# # language is applied at recognizer initialization
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text conversion.
@@ -198,7 +247,7 @@ class AzureSTTService(STTService):
def _on_handle_recognized(self, event):
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
language = getattr(event.result, "language", None) or self._settings.get("language")
language = getattr(event.result, "language", None) or self._settings.language
frame = TranscriptionFrame(
event.result.text,
self._user_id,
@@ -213,7 +262,7 @@ class AzureSTTService(STTService):
def _on_handle_recognizing(self, event):
if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0:
language = getattr(event.result, "language", None) or self._settings.get("language")
language = getattr(event.result, "language", None) or self._settings.language
frame = InterimTranscriptionFrame(
event.result.text,
self._user_id,

93
src/pipecat/services/azure/tts.py
View File

@@ -7,6 +7,7 @@
"""Azure Cognitive Services Text-to-Speech service implementations."""
import asyncio
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -25,6 +26,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService, WordTTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -65,6 +67,31 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma
return sample_rate_map.get(sample_rate, SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm)
@dataclass
class AzureTTSSettings(TTSSettings):
"""Settings for Azure TTS services.
Parameters:
emphasis: Emphasis level for speech ("strong", "moderate", "reduced").
language: Language for synthesis. Defaults to English (US).
pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high").
rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast").
role: Voice role for expression (e.g., "YoungAdultFemale").
style: Speaking style (e.g., "cheerful", "sad", "excited").
style_degree: Intensity of the speaking style (0.01 to 2.0).
volume: Volume level (e.g., "+20%", "loud", "x-soft").
"""
emphasis: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
role: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style_degree: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AzureBaseTTSService:
"""Base mixin class for Azure Cognitive Services text-to-speech implementations.
@@ -73,6 +100,8 @@ class AzureBaseTTSService:
This is a mixin class and should be used alongside TTSService or its subclasses.
"""
_settings: AzureTTSSettings
# Define SSML escape mappings based on SSML reserved characters
# See - https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-structure
SSML_ESCAPE_CHARS = {
@@ -126,22 +155,22 @@ class AzureBaseTTSService:
"""
params = params or AzureBaseTTSService.InputParams()
self._settings = {
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
self._settings = AzureTTSSettings(
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"role": params.role,
"style": params.style,
"style_degree": params.style_degree,
"volume": params.volume,
}
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
)
self._api_key = api_key
self._region = region
self._voice_id = voice
self._speech_synthesizer = None
def language_to_service_language(self, language: Language) -> Optional[str]:
@@ -156,7 +185,7 @@ class AzureBaseTTSService:
return language_to_azure_language(language)
def _construct_ssml(self, text: str) -> str:
language = self._settings["language"]
language = self._settings.language
# Escape special characters
escaped_text = self._escape_text(text)
@@ -165,42 +194,42 @@ class AzureBaseTTSService:
f"<speak version='1.0' xml:lang='{language}' "
"xmlns='http://www.w3.org/2001/10/synthesis' "
"xmlns:mstts='http://www.w3.org/2001/mstts'>"
f"<voice name='{self._voice_id}'>"
f"<voice name='{self._settings.voice}'>"
"<mstts:silence type='Sentenceboundary' value='20ms' />"
)
if self._settings["style"]:
ssml += f"<mstts:express-as style='{self._settings['style']}'"
if self._settings["style_degree"]:
ssml += f" styledegree='{self._settings['style_degree']}'"
if self._settings["role"]:
ssml += f" role='{self._settings['role']}'"
if self._settings.style:
ssml += f"<mstts:express-as style='{self._settings.style}'"
if self._settings.style_degree:
ssml += f" styledegree='{self._settings.style_degree}'"
if self._settings.role:
ssml += f" role='{self._settings.role}'"
ssml += ">"
prosody_attrs = []
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
# Only wrap in prosody tag if there are prosody attributes
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
if self._settings.emphasis:
ssml += f"<emphasis level='{self._settings.emphasis}'>"
ssml += escaped_text
if self._settings["emphasis"]:
if self._settings.emphasis:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
if self._settings["style"]:
if self._settings.style:
ssml += "</mstts:express-as>"
ssml += "</voice></speak>"
@@ -314,7 +343,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
subscription=self._api_key,
region=self._region,
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.speech_synthesis_language = self._settings.language
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)
@@ -364,7 +393,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
Returns:
True if the language is CJK, False otherwise.
"""
language = self._settings.get("language", "").lower()
language = (self._settings.language if self._settings.language else "").lower()
# Check if language starts with CJK language codes
return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu"))
@@ -735,7 +764,7 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
subscription=self._api_key,
region=self._region,
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.speech_synthesis_language = self._settings.language
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)

43
src/pipecat/services/camb/tts.py
View File

@@ -16,6 +16,7 @@ Features:
- Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz)
"""
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional
from camb import StreamTtsOutputConfiguration
@@ -31,6 +32,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -133,6 +135,18 @@ def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]:
return buffer[:aligned_size], buffer[aligned_size:]
@dataclass
class CambTTSSettings(TTSSettings):
"""Settings for Camb.ai TTS service.
Parameters:
user_instructions: Custom instructions for mars-instruct model only.
Ignored for other models. Max 1000 characters.
"""
user_instructions: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class CambTTSService(TTSService):
"""Camb.ai MARS text-to-speech service using the official SDK.
@@ -156,6 +170,8 @@ class CambTTSService(TTSService):
)
"""
_settings: CambTTSSettings
class InputParams(BaseModel):
"""Input parameters for Camb.ai TTS configuration.
@@ -212,16 +228,15 @@ class CambTTSService(TTSService):
)
# Build settings
self._settings = {
"language": (
self._settings = CambTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else "en-us"
),
"user_instructions": params.user_instructions,
}
self.set_model_name(model)
self.set_voice(str(voice_id))
self._voice_id = voice_id
user_instructions=params.user_instructions,
)
self._sync_model_name_to_metrics()
self._client = None
@@ -256,7 +271,7 @@ class CambTTSService(TTSService):
# Use model-specific sample rate if not explicitly specified
if not self._init_sample_rate:
self._sample_rate = MODEL_SAMPLE_RATES.get(self.model_name, 22050)
self._sample_rate = MODEL_SAMPLE_RATES.get(self._settings.model, 22050)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -282,15 +297,15 @@ class CambTTSService(TTSService):
# Build SDK parameters
tts_kwargs: Dict[str, Any] = {
"text": text,
"voice_id": self._voice_id,
"language": self._settings["language"],
"speech_model": self.model_name,
"voice_id": self._settings.voice,
"language": self._settings.language,
"speech_model": self._settings.model,
"output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"),
}
# Add user instructions if using mars-instruct model
if self._model_name == "mars-instruct" and self._settings.get("user_instructions"):
tts_kwargs["user_instructions"] = self._settings["user_instructions"]
if self._settings.model == "mars-instruct" and self._settings.user_instructions:
tts_kwargs["user_instructions"] = self._settings.user_instructions
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame(context_id=context_id)

53
src/pipecat/services/cartesia/stt.py
View File

@@ -12,7 +12,8 @@ the Cartesia Live transcription API for real-time speech recognition.
import json
import urllib.parse
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import CARTESIA_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -42,6 +44,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class CartesiaSTTSettings(STTSettings):
"""Settings for the Cartesia STT service.
Parameters:
encoding: Audio encoding format (e.g. ``"pcm_s16le"``).
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class CartesiaLiveOptions:
"""Configuration options for Cartesia Live STT service.
@@ -136,6 +149,8 @@ class CartesiaSTTService(WebsocketSTTService):
See: https://docs.cartesia.ai/api-reference/stt/stt
"""
_settings: CartesiaSTTSettings
def __init__(
self,
*,
@@ -181,8 +196,12 @@ class CartesiaSTTService(WebsocketSTTService):
k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None"
}
self._settings = merged_options
self.set_model_name(merged_options["model"])
self._settings = CartesiaSTTSettings(
model=merged_options["model"],
language=merged_options.get("language"),
encoding=merged_options.get("encoding", "pcm_s16le"),
)
self._sync_model_name_to_metrics()
self._api_key = api_key
self._base_url = base_url or "api.cartesia.ai"
self._receive_task = None
@@ -275,13 +294,39 @@ class CartesiaSTTService(WebsocketSTTService):
await self._disconnect_websocket()
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update.
Args:
update: A :class:`STTSettings` (or ``CartesiaSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if changed:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def _connect_websocket(self):
try:
if self._websocket and self._websocket.state is State.OPEN:
return
logger.debug("Connecting to Cartesia STT")
params = self._settings
params = {
"model": self._settings.model,
"language": self._settings.language,
"encoding": self._settings.encoding,
"sample_rate": str(self.sample_rate),
}
ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}"
headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key}

188
src/pipecat/services/cartesia/tts.py
View File

@@ -9,8 +9,9 @@
import base64
import json
import warnings
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, List, Literal, Optional
from typing import Any, AsyncGenerator, ClassVar, Dict, List, Literal, Mapping, Optional
from loguru import logger
from pydantic import BaseModel, Field
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
@@ -191,6 +193,42 @@ class CartesiaEmotion(str, Enum):
DETERMINED = "determined"
@dataclass
class CartesiaTTSSettings(TTSSettings):
"""Settings for Cartesia TTS services.
Parameters:
output_container: Audio container format (e.g. "raw").
output_encoding: Audio encoding format (e.g. "pcm_s16le").
output_sample_rate: Audio sample rate in Hz.
speed: Voice speed control for non-Sonic-3 models (literal values).
emotion: List of emotion controls for non-Sonic-3 models.
generation_config: Generation configuration for Sonic-3 models. Includes volume,
speed (numeric), and emotion (string) parameters.
pronunciation_dict_id: The ID of the pronunciation dictionary to use for
custom pronunciations.
"""
output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: Literal["slow", "normal", "fast"] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emotion: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
generation_config: GenerationConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pronunciation_dict_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "CartesiaTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``output_format``."""
flat = dict(settings)
nested = flat.pop("output_format", None)
if isinstance(nested, dict):
flat.setdefault("output_container", nested.get("container"))
flat.setdefault("output_encoding", nested.get("encoding"))
flat.setdefault("output_sample_rate", nested.get("sample_rate"))
return super().from_mapping(flat)
class CartesiaTTSService(AudioContextWordTTSService):
"""Cartesia TTS service with WebSocket streaming and word timestamps.
@@ -199,6 +237,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
customization options including speed and emotion controls.
"""
_settings: CartesiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Cartesia TTS configuration.
@@ -289,22 +329,21 @@ class CartesiaTTSService(AudioContextWordTTSService):
self._api_key = api_key
self._cartesia_version = cartesia_version
self._url = url
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
self._settings = CartesiaTTSSettings(
model=model,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
"speed": params.speed,
"emotion": params.emotion,
"generation_config": params.generation_config,
"pronunciation_dict_id": params.pronunciation_dict_id,
}
self.set_model_name(model)
self.set_voice(voice_id)
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
voice=voice_id,
)
self._sync_model_name_to_metrics()
self._receive_task = None
@@ -316,16 +355,6 @@ class CartesiaTTSService(AudioContextWordTTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model.
Args:
model: The model name to use for synthesis.
"""
self._model_id = model
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Cartesia language format.
@@ -390,7 +419,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
Returns:
List of (word, start_time) tuples processed for the language.
"""
current_language = self._settings.get("language")
current_language = self._settings.language
# Check if this is a CJK language (if language is None, treat as non-CJK)
if current_language and self._is_cjk_language(current_language):
@@ -411,9 +440,9 @@ class CartesiaTTSService(AudioContextWordTTSService):
):
voice_config = {}
voice_config["mode"] = "id"
voice_config["id"] = self._voice_id
voice_config["id"] = self._settings.voice
if self._settings["emotion"]:
if is_given(self._settings.emotion) and self._settings.emotion:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -422,33 +451,36 @@ class CartesiaTTSService(AudioContextWordTTSService):
stacklevel=2,
)
voice_config["__experimental_controls"] = {}
if self._settings["emotion"]:
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
voice_config["__experimental_controls"]["emotion"] = self._settings.emotion
msg = {
"transcript": text,
"continue": continue_transcript,
"context_id": self.get_active_audio_context_id(),
"model_id": self.model_name,
"model_id": self._settings.model,
"voice": voice_config,
"output_format": self._settings["output_format"],
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"add_timestamps": add_timestamps,
"use_original_timestamps": False if self.model_name == "sonic" else True,
"use_original_timestamps": False if self._settings.model == "sonic" else True,
}
if self._settings["language"]:
msg["language"] = self._settings["language"]
if is_given(self._settings.language) and self._settings.language:
msg["language"] = self._settings.language
if self._settings["speed"]:
msg["speed"] = self._settings["speed"]
if is_given(self._settings.speed) and self._settings.speed:
msg["speed"] = self._settings.speed
if self._settings["generation_config"]:
msg["generation_config"] = self._settings["generation_config"].model_dump(
if is_given(self._settings.generation_config) and self._settings.generation_config:
msg["generation_config"] = self._settings.generation_config.model_dump(
exclude_none=True
)
if self._settings["pronunciation_dict_id"]:
msg["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
if is_given(self._settings.pronunciation_dict_id) and self._settings.pronunciation_dict_id:
msg["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
return json.dumps(msg)
@@ -459,7 +491,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -636,6 +668,8 @@ class CartesiaHttpTTSService(TTSService):
integration is preferred.
"""
_settings: CartesiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Cartesia HTTP TTS configuration.
@@ -693,22 +727,21 @@ class CartesiaHttpTTSService(TTSService):
self._api_key = api_key
self._base_url = base_url
self._cartesia_version = cartesia_version
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
self._settings = CartesiaTTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
"speed": params.speed,
"emotion": params.emotion,
"generation_config": params.generation_config,
"pronunciation_dict_id": params.pronunciation_dict_id,
}
self.set_voice(voice_id)
self.set_model_name(model)
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
)
self._sync_model_name_to_metrics()
self._client = AsyncCartesia(
api_key=api_key,
@@ -741,7 +774,7 @@ class CartesiaHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
async def stop(self, frame: EndFrame):
"""Stop the Cartesia HTTP TTS service.
@@ -775,9 +808,9 @@ class CartesiaHttpTTSService(TTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
voice_config = {"mode": "id", "id": self._voice_id}
voice_config = {"mode": "id", "id": self._settings.voice}
if self._settings["emotion"]:
if is_given(self._settings.emotion) and self._settings.emotion:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -785,30 +818,39 @@ class CartesiaHttpTTSService(TTSService):
DeprecationWarning,
stacklevel=2,
)
voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]}
voice_config["__experimental_controls"] = {"emotion": self._settings.emotion}
await self.start_ttfb_metrics()
payload = {
"model_id": self._model_name,
"transcript": text,
"voice": voice_config,
"output_format": self._settings["output_format"],
output_format = {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
}
if self._settings["language"]:
payload["language"] = self._settings["language"]
payload = {
"model_id": self._settings.model,
"transcript": text,
"voice": voice_config,
"output_format": output_format,
}
if self._settings["speed"]:
payload["speed"] = self._settings["speed"]
if is_given(self._settings.language) and self._settings.language:
payload["language"] = self._settings.language
if self._settings["generation_config"]:
payload["generation_config"] = self._settings["generation_config"].model_dump(
if is_given(self._settings.speed) and self._settings.speed:
payload["speed"] = self._settings.speed
if is_given(self._settings.generation_config) and self._settings.generation_config:
payload["generation_config"] = self._settings.generation_config.model_dump(
exclude_none=True
)
if self._settings["pronunciation_dict_id"]:
payload["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
if (
is_given(self._settings.pronunciation_dict_id)
and self._settings.pronunciation_dict_id
):
payload["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
yield TTSStartedFrame(context_id=context_id)

12
src/pipecat/services/cerebras/llm.py
View File

@@ -66,16 +66,16 @@ class CerebrasLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"seed": self._settings.seed,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_completion_tokens": self._settings.max_completion_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

102
src/pipecat/services/deepgram/flux/stt.py
View File

@@ -9,6 +9,7 @@
import asyncio
import json
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator, Dict, Optional
from urllib.parse import urlencode
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
@@ -67,6 +69,34 @@ class FluxEventType(str, Enum):
UPDATE = "Update"
@dataclass
class DeepgramFluxSTTSettings(STTSettings):
"""Settings for the Deepgram Flux STT service.
Parameters:
eager_eot_threshold: EagerEndOfTurn/TurnResumed threshold. Off by default.
Lower values = more aggressive (faster response, more LLM calls).
Higher values = more conservative (slower response, fewer LLM calls).
eot_threshold: End-of-turn confidence required to finish a turn (default 0.7).
eot_timeout_ms: Time in ms after speech to finish a turn regardless of EOT
confidence (default 5000).
keyterm: Keyterms to boost recognition accuracy for specialized terminology.
mip_opt_out: Opt out of the Deepgram Model Improvement Program (default False).
tag: Tags to label requests for identification during usage reporting.
min_confidence: Minimum confidence required to create a TranscriptionFrame.
encoding: Audio encoding format (e.g. ``"linear16"``).
"""
eager_eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
eot_timeout_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
keyterm: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
mip_opt_out: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
tag: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_confidence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramFluxSTTService(WebsocketSTTService):
"""Deepgram Flux speech-to-text service.
@@ -89,6 +119,8 @@ class DeepgramFluxSTTService(WebsocketSTTService):
...
"""
_settings: DeepgramFluxSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Deepgram Flux API.
@@ -181,14 +213,23 @@ class DeepgramFluxSTTService(WebsocketSTTService):
**kwargs,
)
params = params or DeepgramFluxSTTService.InputParams()
self._settings = DeepgramFluxSTTSettings(
model=model,
language=Language.EN,
encoding=flux_encoding,
eager_eot_threshold=params.eager_eot_threshold,
eot_threshold=params.eot_threshold,
eot_timeout_ms=params.eot_timeout_ms,
keyterm=params.keyterm or [],
mip_opt_out=params.mip_opt_out,
tag=params.tag or [],
min_confidence=params.min_confidence,
)
self._sync_model_name_to_metrics()
self._api_key = api_key
self._url = url
self._model = model
self._params = params or DeepgramFluxSTTService.InputParams()
self._should_interrupt = should_interrupt
self._flux_encoding = flux_encoding
# This is the currently only supported language
self._language = Language.EN
self._websocket_url = None
self._receive_task = None
# Flux event handlers
@@ -343,6 +384,25 @@ class DeepgramFluxSTTService(WebsocketSTTService):
"""
return True
async def _update_settings(self, update: DeepgramFluxSTTSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram Flux STT service.
@@ -355,29 +415,29 @@ class DeepgramFluxSTTService(WebsocketSTTService):
await super().start(frame)
url_params = [
f"model={self._model}",
f"model={self._settings.model}",
f"sample_rate={self.sample_rate}",
f"encoding={self._flux_encoding}",
f"encoding={self._settings.encoding}",
]
if self._params.eager_eot_threshold is not None:
url_params.append(f"eager_eot_threshold={self._params.eager_eot_threshold}")
if self._settings.eager_eot_threshold is not None:
url_params.append(f"eager_eot_threshold={self._settings.eager_eot_threshold}")
if self._params.eot_threshold is not None:
url_params.append(f"eot_threshold={self._params.eot_threshold}")
if self._settings.eot_threshold is not None:
url_params.append(f"eot_threshold={self._settings.eot_threshold}")
if self._params.eot_timeout_ms is not None:
url_params.append(f"eot_timeout_ms={self._params.eot_timeout_ms}")
if self._settings.eot_timeout_ms is not None:
url_params.append(f"eot_timeout_ms={self._settings.eot_timeout_ms}")
if self._params.mip_opt_out is not None:
url_params.append(f"mip_opt_out={str(self._params.mip_opt_out).lower()}")
if self._settings.mip_opt_out is not None:
url_params.append(f"mip_opt_out={str(self._settings.mip_opt_out).lower()}")
# Add keyterm parameters (can have multiple)
for keyterm in self._params.keyterm:
for keyterm in self._settings.keyterm:
url_params.append(urlencode({"keyterm": keyterm}))
# Add tag parameters (can have multiple)
for tag_value in self._params.tag:
for tag_value in self._settings.tag:
url_params.append(urlencode({"tag": tag_value}))
self._websocket_url = f"{self._url}?{'&'.join(url_params)}"
@@ -676,7 +736,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
# Compute the average confidence
average_confidence = self._calculate_average_confidence(data)
if not self._params.min_confidence or average_confidence > self._params.min_confidence:
if not self._settings.min_confidence or average_confidence > self._settings.min_confidence:
# EndOfTurn means Flux has determined the turn is complete,
# so this TranscriptionFrame is always finalized
await self.push_frame(
@@ -684,7 +744,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
result=data,
finalized=True,
)
@@ -694,7 +754,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
f"Transcription confidence below min_confidence threshold: {average_confidence}"
)
await self._handle_transcription(transcript, True, self._language)
await self._handle_transcription(transcript, True, self._settings.language)
await self.stop_processing_metrics()
await self.broadcast_frame(UserStoppedSpeakingFrame)
await self._call_event_handler("on_end_of_turn", transcript)
@@ -738,7 +798,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
result=data,
)
)

91
src/pipecat/services/deepgram/stt.py
View File

@@ -6,7 +6,8 @@
"""Deepgram speech-to-text service implementation."""
from typing import AsyncGenerator, Dict, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional
from loguru import logger
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import DEEPGRAM_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -45,6 +47,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class DeepgramSTTSettings(STTSettings):
"""Settings for the Deepgram STT service.
Parameters:
live_options: Deepgram ``LiveOptions`` for detailed configuration.
"""
live_options: LiveOptions | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramSTTService(STTService):
"""Deepgram speech-to-text service.
@@ -63,6 +76,8 @@ class DeepgramSTTService(STTService):
...
"""
_settings: DeepgramSTTSettings
def __init__(
self,
*,
@@ -139,12 +154,18 @@ class DeepgramSTTService(STTService):
if "language" in merged_options and isinstance(merged_options["language"], Language):
merged_options["language"] = merged_options["language"].value
self.set_model_name(merged_options["model"])
self._settings = merged_options
merged_live_options = LiveOptions(**merged_options)
self._settings = DeepgramSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
)
self._sync_model_name_to_metrics()
self._addons = addons
self._should_interrupt = should_interrupt
if merged_options.get("vad_events"):
if merged_live_options.vad_events:
import warnings
with warnings.catch_warnings():
@@ -175,7 +196,7 @@ class DeepgramSTTService(STTService):
Returns:
True if VAD events are enabled in the current settings.
"""
return self._settings["vad_events"]
return self._settings.live_options.vad_events
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -185,28 +206,48 @@ class DeepgramSTTService(STTService):
"""
return True
async def set_model(self, model: str):
"""Set the Deepgram model and reconnect.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update, keeping ``live_options`` in sync.
Args:
model: The Deepgram model name to use.
Top-level ``model`` and ``language`` are the source of truth. When
they are given in *update* their values are propagated into
``live_options``. When only ``live_options`` is given, its ``model``
and ``language`` are propagated *up* to the top-level fields.
Any change triggers a WebSocket reconnect.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._settings["model"] = model
# Determine which top-level fields are explicitly provided.
model_given = isinstance(update, DeepgramSTTSettings) and is_given(
getattr(update, "model", NOT_GIVEN)
)
language_given = isinstance(update, DeepgramSTTSettings) and is_given(
getattr(update, "language", NOT_GIVEN)
)
changed = await super()._update_settings(update)
if not changed:
return changed
# --- Sync model --------------------------------------------------
if model_given:
# Top-level model wins → push into live_options.
self._settings.live_options.model = self._settings.model
elif "live_options" in changed and self._settings.live_options.model is not None:
# Only live_options was given → pull model up.
self._settings.model = self._settings.live_options.model
self._sync_model_name_to_metrics()
# --- Sync language -----------------------------------------------
if language_given:
self._settings.live_options.language = self._settings.language
elif "live_options" in changed and self._settings.live_options.language is not None:
self._settings.language = self._settings.live_options.language
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
Args:
language: The language to use for speech recognition.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram STT service.
@@ -215,7 +256,7 @@ class DeepgramSTTService(STTService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.live_options.sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -268,7 +309,9 @@ class DeepgramSTTService(STTService):
self._on_utterance_end,
)
if not await self._connection.start(options=self._settings, addons=self._addons):
if not await self._connection.start(
options=self._settings.live_options, addons=self._addons
):
await self.push_error(error_msg=f"Unable to connect to Deepgram")
else:
headers = {

102
src/pipecat/services/deepgram/stt_sagemaker.py
View File

@@ -14,7 +14,8 @@ languages, and various Deepgram features.
import asyncio
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -31,6 +32,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import DEEPGRAM_SAGEMAKER_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -47,6 +49,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class DeepgramSageMakerSTTSettings(STTSettings):
"""Settings for the Deepgram SageMaker STT service.
Parameters:
live_options: Deepgram ``LiveOptions`` for detailed configuration.
"""
live_options: LiveOptions | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramSageMakerSTTService(STTService):
"""Deepgram speech-to-text service for AWS SageMaker.
@@ -75,6 +88,8 @@ class DeepgramSageMakerSTTService(STTService):
)
"""
_settings: DeepgramSageMakerSTTSettings
def __init__(
self,
*,
@@ -128,8 +143,13 @@ class DeepgramSageMakerSTTService(STTService):
if "language" in merged_options and isinstance(merged_options["language"], Language):
merged_options["language"] = merged_options["language"].value
self.set_model_name(merged_options["model"])
self._settings = merged_options
merged_live_options = LiveOptions(**merged_options)
self._settings = DeepgramSageMakerSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
)
self._sync_model_name_to_metrics()
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None
@@ -143,35 +163,55 @@ class DeepgramSageMakerSTTService(STTService):
"""
return True
async def set_model(self, model: str):
"""Set the Deepgram model and reconnect.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update, keeping ``live_options`` in sync.
Disconnects from the current session, updates the model setting, and
establishes a new connection with the updated model.
Top-level ``model`` and ``language`` are the source of truth. When
they are given in *update* their values are propagated into
``live_options``. When only ``live_options`` is given, its ``model``
and ``language`` are propagated *up* to the top-level fields.
Args:
model: The Deepgram model name to use (e.g., "nova-3").
Any change triggers a reconnect.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._settings["model"] = model
await self._disconnect()
await self._connect()
# Determine which top-level fields are explicitly provided.
model_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given(
getattr(update, "model", NOT_GIVEN)
)
language_given = isinstance(update, DeepgramSageMakerSTTSettings) and is_given(
getattr(update, "language", NOT_GIVEN)
)
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
changed = await super()._update_settings(update)
Disconnects from the current session, updates the language setting, and
establishes a new connection with the updated language.
if not changed:
return changed
Args:
language: The language to use for speech recognition (e.g., Language.EN,
Language.ES).
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
await self._disconnect()
await self._connect()
# --- Sync model --------------------------------------------------
if model_given:
# Top-level model wins → push into live_options.
self._settings.live_options.model = self._settings.model
elif "live_options" in changed and self._settings.live_options.model is not None:
# Only live_options was given → pull model up.
self._settings.model = self._settings.live_options.model
self._sync_model_name_to_metrics()
# --- Sync language -----------------------------------------------
if language_given:
lang = self._settings.language
if isinstance(lang, Language):
lang = lang.value
self._settings.live_options.language = lang
elif "live_options" in changed and self._settings.live_options.language is not None:
self._settings.language = self._settings.live_options.language
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram SageMaker STT service.
@@ -180,7 +220,7 @@ class DeepgramSageMakerSTTService(STTService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.live_options.sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -226,12 +266,12 @@ class DeepgramSageMakerSTTService(STTService):
"""
logger.debug("Connecting to Deepgram on SageMaker...")
# Update sample rate in settings
self._settings["sample_rate"] = self.sample_rate
# Update sample rate in live_options
self._settings.live_options.sample_rate = self.sample_rate
# Build query string from settings, converting booleans to strings
# Build query string from live_options, converting booleans to strings
query_params = {}
for key, value in self._settings.items():
for key, value in self._settings.live_options.to_dict().items():
if value is not None:
# Convert boolean values to lowercase strings for Deepgram API
if isinstance(value, bool):

69
src/pipecat/services/deepgram/tts.py
View File

@@ -11,7 +11,8 @@ for generating speech from text using various voice models.
"""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -29,6 +30,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService, WebsocketTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -43,6 +45,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class DeepgramTTSSettings(TTSSettings):
"""Settings for Deepgram TTS service.
Parameters:
encoding: Audio encoding format (linear16, mulaw, alaw).
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramTTSService(WebsocketTTSService):
"""Deepgram WebSocket-based text-to-speech service.
@@ -51,6 +64,8 @@ class DeepgramTTSService(WebsocketTTSService):
message for conversational AI use cases.
"""
_settings: DeepgramTTSSettings
SUPPORTED_ENCODINGS = ("linear16", "mulaw", "alaw")
def __init__(
@@ -91,10 +106,12 @@ class DeepgramTTSService(WebsocketTTSService):
self._api_key = api_key
self._base_url = base_url
self._settings = {
"encoding": encoding,
}
self.set_voice(voice)
self._settings = DeepgramTTSSettings(
model=voice,
voice=voice,
encoding=encoding,
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._context_id: Optional[str] = None
@@ -166,6 +183,28 @@ class DeepgramTTSService(WebsocketTTSService):
await self._disconnect_websocket()
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update.
Args:
update: A :class:`TTSSettings` (or ``DeepgramTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
# Deepgram uses voice as the model, so keep them in sync for metrics
if "voice" in changed:
self._settings.model = self._settings.voice
self._sync_model_name_to_metrics()
if changed:
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to Deepgram WebSocket API with configured settings."""
try:
@@ -176,8 +215,8 @@ class DeepgramTTSService(WebsocketTTSService):
# Build WebSocket URL with query parameters
params = []
params.append(f"model={self._voice_id}")
params.append(f"encoding={self._settings['encoding']}")
params.append(f"model={self._settings.voice}")
params.append(f"encoding={self._settings.encoding}")
params.append(f"sample_rate={self.sample_rate}")
url = f"{self._base_url}/v1/speak?{'&'.join(params)}"
@@ -330,6 +369,8 @@ class DeepgramHttpTTSService(TTSService):
configurable sample rates and quality settings.
"""
_settings: DeepgramTTSSettings
def __init__(
self,
*,
@@ -357,10 +398,12 @@ class DeepgramHttpTTSService(TTSService):
self._api_key = api_key
self._session = aiohttp_session
self._base_url = base_url
self._settings = {
"encoding": encoding,
}
self.set_voice(voice)
self._settings = DeepgramTTSSettings(
model=voice,
voice=voice,
encoding=encoding,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
@@ -389,8 +432,8 @@ class DeepgramHttpTTSService(TTSService):
headers = {"Authorization": f"Token {self._api_key}", "Content-Type": "application/json"}
params = {
"model": self._voice_id,
"encoding": self._settings["encoding"],
"model": self._settings.voice,
"encoding": self._settings.encoding,
"sample_rate": self.sample_rate,
"container": "none",
}

53
src/pipecat/services/deepgram/tts_sagemaker.py
View File

@@ -14,7 +14,8 @@ streaming audio output.
import asyncio
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -32,10 +33,22 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@dataclass
class DeepgramSageMakerTTSSettings(TTSSettings):
"""Settings for Deepgram SageMaker TTS service.
Parameters:
encoding: Audio encoding format (e.g. "linear16").
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramSageMakerTTSService(TTSService):
"""Deepgram text-to-speech service for AWS SageMaker.
@@ -58,6 +71,8 @@ class DeepgramSageMakerTTSService(TTSService):
)
"""
_settings: DeepgramSageMakerTTSSettings
def __init__(
self,
*,
@@ -89,8 +104,12 @@ class DeepgramSageMakerTTSService(TTSService):
self._endpoint_name = endpoint_name
self._region = region
self._encoding = encoding
self.set_voice(voice)
self._settings = DeepgramSageMakerTTSSettings(
model=voice,
voice=voice,
encoding=encoding,
)
self._sync_model_name_to_metrics()
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None
@@ -156,7 +175,8 @@ class DeepgramSageMakerTTSService(TTSService):
logger.debug("Connecting to Deepgram TTS on SageMaker...")
query_string = (
f"model={self._voice_id}&encoding={self._encoding}&sample_rate={self.sample_rate}"
f"model={self._settings.voice}&encoding={self._settings.encoding}"
f"&sample_rate={self.sample_rate}"
)
self._client = SageMakerBidiClient(
@@ -200,6 +220,31 @@ class DeepgramSageMakerTTSService(TTSService):
logger.debug("Disconnected from Deepgram TTS on SageMaker")
await self._call_event_handler("on_disconnected")
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and reconnect if necessary.
Since all settings are part of the SageMaker session query string,
any setting change requires reconnecting to apply the new values.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# Deepgram uses voice as the model, so keep them in sync for metrics
if "voice" in changed:
self._settings.model = self._settings.voice
self._sync_model_name_to_metrics()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def _process_responses(self):
"""Process streaming responses from Deepgram TTS on SageMaker.

14
src/pipecat/services/deepseek/llm.py
View File

@@ -65,18 +65,18 @@ class DeepSeekLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

234
src/pipecat/services/elevenlabs/stt.py
View File

@@ -11,11 +11,13 @@ using segmented audio processing. The service uploads audio files and receives
transcription results directly.
"""
import asyncio
import base64
import io
import json
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, Optional
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -33,6 +35,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import ELEVENLABS_REALTIME_TTFS_P99, ELEVENLABS_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -167,6 +170,51 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
class CommitStrategy(str, Enum):
"""Commit strategies for transcript segmentation."""
MANUAL = "manual"
VAD = "vad"
@dataclass
class ElevenLabsSTTSettings(STTSettings):
"""Settings for the ElevenLabs file-based STT service.
Parameters:
tag_audio_events: Whether to include audio event tags in transcription.
"""
tag_audio_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class ElevenLabsRealtimeSTTSettings(STTSettings):
"""Settings for the ElevenLabs Realtime STT service.
See ``ElevenLabsRealtimeSTTService.InputParams`` for detailed descriptions.
Parameters:
commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD).
vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0).
vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive).
min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms).
min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms).
include_timestamps: Whether to include word-level timestamps in transcripts.
enable_logging: Whether to enable logging on ElevenLabs' side.
include_language_detection: Whether to include language detection in transcripts.
"""
commit_strategy: CommitStrategy | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_silence_threshold_secs: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_speech_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_silence_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_timestamps: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_logging: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_language_detection: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class ElevenLabsSTTService(SegmentedSTTService):
"""Speech-to-text service using ElevenLabs' file-based API.
@@ -175,6 +223,8 @@ class ElevenLabsSTTService(SegmentedSTTService):
The service uploads audio files to ElevenLabs and receives transcription results directly.
"""
_settings: ElevenLabsSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for ElevenLabs STT API.
@@ -223,13 +273,15 @@ class ElevenLabsSTTService(SegmentedSTTService):
self._base_url = base_url
self._session = aiohttp_session
self._model_id = model
self._tag_audio_events = params.tag_audio_events
self._settings = {
"language": self.language_to_service_language(params.language)
self._settings = ElevenLabsSTTSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
}
tag_audio_events=params.tag_audio_events,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -250,27 +302,24 @@ class ElevenLabsSTTService(SegmentedSTTService):
"""
return language_to_elevenlabs_language(language)
async def set_language(self, language: Language):
"""Set the transcription language.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update.
Converts language to ElevenLabs format before applying and keeps
``_model_id`` in sync with the model setting.
Args:
language: The language to use for speech-to-text transcription.
update: A :class:`STTSettings` (or ``ElevenLabsSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = self.language_to_service_language(language)
changed = await super()._update_settings(update)
async def set_model(self, model: str):
"""Set the STT model.
if "model" in changed:
self._model_id = self._settings.model
Args:
model: The model name to use for transcription.
Note:
ElevenLabs STT API does not currently support model selection.
This method is provided for interface compatibility.
"""
await super().set_model(model)
logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model")
return changed
async def _transcribe_audio(self, audio_data: bytes) -> dict:
"""Upload audio data to ElevenLabs and get transcription result.
@@ -298,8 +347,8 @@ class ElevenLabsSTTService(SegmentedSTTService):
# Add required model_id, language_code, and tag_audio_events
data.add_field("model_id", self._model_id)
data.add_field("language_code", self._settings["language"])
data.add_field("tag_audio_events", str(self._tag_audio_events).lower())
data.add_field("language_code", self._settings.language)
data.add_field("tag_audio_events", str(self._settings.tag_audio_events).lower())
async with self._session.post(url, data=data, headers=headers) as response:
if response.status != 200:
@@ -385,13 +434,6 @@ def audio_format_from_sample_rate(sample_rate: int) -> str:
return "pcm_16000"
class CommitStrategy(str, Enum):
"""Commit strategies for transcript segmentation."""
MANUAL = "manual"
VAD = "vad"
class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""Speech-to-text service using ElevenLabs' Realtime WebSocket API.
@@ -404,6 +446,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
commit transcript segments, providing consistency with other STT services.
"""
_settings: ElevenLabsRealtimeSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for ElevenLabs Realtime STT API.
@@ -469,11 +513,25 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
self._api_key = api_key
self._base_url = base_url
self._model_id = model
self._params = params
self._audio_format = "" # initialized in start()
self._receive_task = None
self._settings = {"language": params.language_code}
self._connected_event = asyncio.Event()
self._connected_event.set()
self._settings = ElevenLabsRealtimeSTTSettings(
model=model,
language=params.language_code,
commit_strategy=params.commit_strategy,
vad_silence_threshold_secs=params.vad_silence_threshold_secs,
vad_threshold=params.vad_threshold,
min_speech_duration_ms=params.min_speech_duration_ms,
min_silence_duration_ms=params.min_silence_duration_ms,
include_timestamps=params.include_timestamps,
enable_logging=params.enable_logging,
include_language_detection=params.include_language_detection,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -483,42 +541,29 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the transcription language.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update and reconnect if anything changed.
Converts language to ElevenLabs format before applying and keeps
``_model_id`` in sync.
Args:
language: The language to use for speech-to-text transcription.
update: A :class:`STTSettings` (or ``ElevenLabsRealtimeSTTSettings``) delta.
Note:
Changing language requires reconnecting to the WebSocket.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.info(f"Switching STT language to: [{language}]")
new_language = (
language_to_elevenlabs_language(language)
if isinstance(language, Language)
else language
)
self._params.language_code = new_language
self._settings["language"] = new_language
# Reconnect with new settings
await self._disconnect()
await self._connect()
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
Note:
Changing model requires reconnecting to the WebSocket.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._model_id = model
# Reconnect with new settings
changed = await super()._update_settings(update)
if not changed:
return changed
if "model" in changed:
self._model_id = self._settings.model
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the STT service and establish WebSocket connection.
@@ -566,7 +611,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
# Send commit when user stops speaking (manual commit mode)
if self._params.commit_strategy == CommitStrategy.MANUAL:
if self._settings.commit_strategy == CommitStrategy.MANUAL:
if self._websocket and self._websocket.state is State.OPEN:
try:
commit_message = {
@@ -589,6 +634,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Yields:
None - transcription results are handled via WebSocket responses.
"""
# Wait for any in-flight _connect() to finish before checking state
await self._connected_event.wait()
# Reconnect if connection is closed
if not self._websocket or self._websocket.state is State.CLOSED:
await self._connect()
@@ -613,12 +661,18 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
async def _connect(self):
"""Establish WebSocket connection to ElevenLabs Realtime STT."""
await self._connect_websocket()
self._connected_event.clear()
try:
await self._connect_websocket()
await super()._connect()
await super()._connect()
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(
self._receive_task_handler(self._report_error)
)
finally:
self._connected_event.set()
async def _disconnect(self):
"""Close WebSocket connection and cleanup tasks."""
@@ -656,36 +710,40 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
# Build query parameters
params = [f"model_id={self._model_id}"]
if self._params.language_code:
params.append(f"language_code={self._params.language_code}")
if self._settings.language:
params.append(f"language_code={self._settings.language}")
params.append(f"audio_format={self._audio_format}")
params.append(f"commit_strategy={self._params.commit_strategy.value}")
params.append(f"commit_strategy={self._settings.commit_strategy.value}")
# Add optional parameters
if self._params.include_timestamps:
params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}")
if self._params.enable_logging:
params.append(f"enable_logging={str(self._params.enable_logging).lower()}")
if self._params.include_language_detection:
if self._settings.include_timestamps:
params.append(
f"include_language_detection={str(self._params.include_language_detection).lower()}"
f"include_timestamps={str(self._settings.include_timestamps).lower()}"
)
if self._settings.enable_logging:
params.append(f"enable_logging={str(self._settings.enable_logging).lower()}")
if self._settings.include_language_detection:
params.append(
f"include_language_detection={str(self._settings.include_language_detection).lower()}"
)
# Add VAD parameters if using VAD commit strategy and values are specified
if self._params.commit_strategy == CommitStrategy.VAD:
if self._params.vad_silence_threshold_secs is not None:
if self._settings.commit_strategy == CommitStrategy.VAD:
if self._settings.vad_silence_threshold_secs is not None:
params.append(
f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}"
f"vad_silence_threshold_secs={self._settings.vad_silence_threshold_secs}"
)
if self._settings.vad_threshold is not None:
params.append(f"vad_threshold={self._settings.vad_threshold}")
if self._settings.min_speech_duration_ms is not None:
params.append(f"min_speech_duration_ms={self._settings.min_speech_duration_ms}")
if self._settings.min_silence_duration_ms is not None:
params.append(
f"min_silence_duration_ms={self._settings.min_silence_duration_ms}"
)
if self._params.vad_threshold is not None:
params.append(f"vad_threshold={self._params.vad_threshold}")
if self._params.min_speech_duration_ms is not None:
params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}")
if self._params.min_silence_duration_ms is not None:
params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}")
ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}"
@@ -817,7 +875,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""
# If timestamps are enabled, skip this message and wait for the
# committed_transcript_with_timestamps message which contains all the data
if self._params.include_timestamps:
if self._settings.include_timestamps:
return
text = data.get("text", "").strip()

298
src/pipecat/services/elevenlabs/tts.py
View File

@@ -13,7 +13,19 @@ with support for streaming audio, word timestamps, and voice customization.
import asyncio
import base64
import json
from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union
from dataclasses import dataclass, field
from typing import (
Any,
AsyncGenerator,
ClassVar,
Dict,
List,
Literal,
Mapping,
Optional,
Tuple,
Union,
)
import aiohttp
from loguru import logger
@@ -32,6 +44,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import (
AudioContextWordTTSService,
WordTTSService,
@@ -136,12 +149,12 @@ def output_format_from_sample_rate(sample_rate: int) -> str:
def build_elevenlabs_voice_settings(
settings: Dict[str, Any],
settings: Union[Dict[str, Any], "TTSSettings"],
) -> Optional[Dict[str, Union[float, bool]]]:
"""Build voice settings dictionary for ElevenLabs based on provided settings.
Args:
settings: Dictionary containing voice settings parameters.
settings: Dictionary or settings containing voice settings parameters.
Returns:
Dictionary of voice settings or None if no valid settings are provided.
@@ -150,8 +163,11 @@ def build_elevenlabs_voice_settings(
voice_settings = {}
for key in voice_setting_keys:
if key in settings and settings[key] is not None:
voice_settings[key] = settings[key]
val = (
getattr(settings, key, None) if isinstance(settings, TTSSettings) else settings.get(key)
)
if val is not None and is_given(val):
voice_settings[key] = val
return voice_settings or None
@@ -168,6 +184,79 @@ class PronunciationDictionaryLocator(BaseModel):
version_id: str
@dataclass
class ElevenLabsTTSSettings(TTSSettings):
"""Settings for the ElevenLabs WebSocket TTS service.
Fields that appear in the WebSocket URL (``voice``, ``model``,
``language``) require a full reconnect when changed. Fields that
affect the voice character (``stability``, ``similarity_boost``,
``style``, ``use_speaker_boost``, ``speed``) can be applied by closing
the current audio context so a new one is opened with updated settings.
Parameters:
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.7 to 1.2).
auto_mode: Whether to enable automatic mode optimization.
enable_ssml_parsing: Whether to parse SSML tags in text.
enable_logging: Whether to enable ElevenLabs logging.
apply_text_normalization: Text normalization mode ("auto", "on", "off").
"""
stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
auto_mode: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_ssml_parsing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_logging: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
#: Fields in the WS URL — changing any of these requires a reconnect.
URL_FIELDS: ClassVar[frozenset[str]] = frozenset({"voice", "model", "language"})
#: Fields affecting voice character — changing these requires closing the
#: current audio context so the next one picks up new settings.
VOICE_SETTINGS_FIELDS: ClassVar[frozenset[str]] = frozenset(
{"stability", "similarity_boost", "style", "use_speaker_boost", "speed"}
)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@dataclass
class ElevenLabsHttpTTSSettings(TTSSettings):
"""Settings for the ElevenLabs HTTP TTS service.
Parameters:
optimize_streaming_latency: Latency optimization level (0-4).
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.25 to 4.0).
apply_text_normalization: Text normalization mode ("auto", "on", "off").
"""
optimize_streaming_latency: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
def calculate_word_times(
alignment_info: Mapping[str, Any],
cumulative_time: float,
@@ -236,6 +325,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
customization options including stability, similarity boost, and speed controls.
"""
_settings: ElevenLabsTTSSettings
class InputParams(BaseModel):
"""Input parameters for ElevenLabs TTS configuration.
@@ -316,22 +407,24 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
self._api_key = api_key
self._url = url
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else None,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
"speed": params.speed,
"auto_mode": str(params.auto_mode).lower(),
"enable_ssml_parsing": params.enable_ssml_parsing,
"enable_logging": params.enable_logging,
"apply_text_normalization": params.apply_text_normalization,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._settings = ElevenLabsTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else None
),
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
auto_mode=str(params.auto_mode).lower(),
enable_ssml_parsing=params.enable_ssml_parsing,
enable_logging=params.enable_logging,
apply_text_normalization=params.apply_text_normalization,
)
self._sync_model_name_to_metrics()
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
@@ -365,54 +458,57 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
return language_to_elevenlabs_language(language)
def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings)
ts = self._settings
voice_setting_keys = [
"stability",
"similarity_boost",
"style",
"use_speaker_boost",
"speed",
]
voice_settings = {}
for key in voice_setting_keys:
val = getattr(ts, key, None)
if val is not None and is_given(val):
voice_settings[key] = val
return voice_settings or None
async def set_model(self, model: str):
"""Set the TTS model and reconnect.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update, reconnecting as needed.
Uses the declarative ``URL_FIELDS`` and ``VOICE_SETTINGS_FIELDS``
sets on :class:`ElevenLabsTTSSettings` to decide whether to
reconnect the WebSocket or close the current audio context.
Args:
model: The model name to use for synthesis.
update: A :class:`TTSSettings` (or ``ElevenLabsTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
changed = await super()._update_settings(update)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice, model, or language changed."""
# Track previous values for settings that require reconnection
prev_voice = self._voice_id
prev_model = self.model_name
prev_language = self._settings.get("language")
# Create snapshot of current voice settings to detect changes after update
prev_voice_settings = self._voice_settings.copy() if self._voice_settings else None
if not changed:
return changed
await super()._update_settings(settings)
# Update voice settings for the next context creation
# Rebuild voice settings for next context
self._voice_settings = self._set_voice_settings()
# Check if URL-level settings changed (these require reconnection)
url_changed = (
prev_voice != self._voice_id
or prev_model != self.model_name
or prev_language != self._settings.get("language")
)
# Check if only voice settings changed (speed, stability, etc.)
voice_settings_changed = prev_voice_settings != self._voice_settings
url_changed = bool(changed.keys() & ElevenLabsTTSSettings.URL_FIELDS)
voice_settings_changed = bool(changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS)
if url_changed:
# These settings are in the WebSocket URL, so we need to reconnect
logger.debug(
f"URL-level setting changed (voice/model/language), reconnecting WebSocket"
f"URL-level setting changed ({changed.keys() & ElevenLabsTTSSettings.URL_FIELDS}), "
f"reconnecting WebSocket"
)
await self._disconnect()
await self._connect()
elif voice_settings_changed and self.has_active_audio_context():
# Voice settings can be updated by closing current context
# so new one gets created with updated voice settings
logger.debug(f"Voice settings changed, closing current context to apply changes")
logger.debug(
f"Voice settings changed ({changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS}), "
f"closing current context to apply changes"
)
context_id = self.get_active_audio_context_id()
try:
if self._websocket:
@@ -423,6 +519,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
self.reset_active_audio_context()
if not url_changed:
# Reconnect applies all settings; only warn about fields not handled
# by voice settings or URL changes.
handled = ElevenLabsTTSSettings.URL_FIELDS | ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS
self._warn_unhandled_updated_settings(changed.keys() - handled)
return changed
async def start(self, frame: StartFrame):
"""Start the ElevenLabs TTS service.
@@ -503,22 +607,22 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
logger.debug("Connecting to ElevenLabs")
voice_id = self._voice_id
model = self.model_name
voice_id = self._settings.voice
model = self._settings.model
output_format = self._output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings.auto_mode}"
if self._settings["enable_ssml_parsing"]:
url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}"
if self._settings.enable_ssml_parsing:
url += f"&enable_ssml_parsing={self._settings.enable_ssml_parsing}"
if self._settings["enable_logging"]:
url += f"&enable_logging={self._settings['enable_logging']}"
if self._settings.enable_logging:
url += f"&enable_logging={self._settings.enable_logging}"
if self._settings["apply_text_normalization"] is not None:
url += f"&apply_text_normalization={self._settings['apply_text_normalization']}"
if self._settings.apply_text_normalization is not None:
url += f"&apply_text_normalization={self._settings.apply_text_normalization}"
# Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS
language = self._settings["language"]
language = self._settings.language
if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None:
url += f"&language_code={language}"
logger.debug(f"Using language code: {language}")
@@ -742,6 +846,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
connection is not required or desired.
"""
_settings: ElevenLabsHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for ElevenLabs HTTP TTS configuration.
@@ -808,20 +914,21 @@ class ElevenLabsHttpTTSService(WordTTSService):
self._params = params
self._session = aiohttp_session
self._settings = {
"language": self.language_to_service_language(params.language)
self._settings = ElevenLabsHttpTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(params.language)
if params.language
else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
"speed": params.speed,
"apply_text_normalization": params.apply_text_normalization,
}
self.set_model_name(model)
self.set_voice(voice_id)
optimize_streaming_latency=params.optimize_streaming_latency,
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
apply_text_normalization=params.apply_text_normalization,
)
self._sync_model_name_to_metrics()
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
@@ -858,10 +965,19 @@ class ElevenLabsHttpTTSService(WordTTSService):
def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings)
async def _update_settings(self, settings: Mapping[str, Any]):
await super()._update_settings(settings)
# Update voice settings for the next context creation
self._voice_settings = self._set_voice_settings()
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and rebuild voice settings.
Args:
update: A :class:`TTSSettings` (or ``ElevenLabsHttpTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if changed:
self._voice_settings = self._set_voice_settings()
return changed
def _reset_state(self):
"""Reset internal state variables."""
@@ -979,11 +1095,11 @@ class ElevenLabsHttpTTSService(WordTTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
# Use the with-timestamps endpoint
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream/with-timestamps"
url = f"{self._base_url}/v1/text-to-speech/{self._settings.voice}/stream/with-timestamps"
payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = {
"text": text,
"model_id": self._model_name,
"model_id": self._settings.model,
}
# Include previous text as context if available
@@ -998,11 +1114,14 @@ class ElevenLabsHttpTTSService(WordTTSService):
locator.model_dump() for locator in self._pronunciation_dictionary_locators
]
if self._settings["apply_text_normalization"] is not None:
payload["apply_text_normalization"] = self._settings["apply_text_normalization"]
if (
is_given(self._settings.apply_text_normalization)
and self._settings.apply_text_normalization is not None
):
payload["apply_text_normalization"] = self._settings.apply_text_normalization
language = self._settings["language"]
if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language:
language = self._settings.language
if self._settings.model in ELEVENLABS_MULTILINGUAL_MODELS and language:
payload["language_code"] = language
logger.debug(f"Using language code: {language}")
elif language:
@@ -1019,8 +1138,11 @@ class ElevenLabsHttpTTSService(WordTTSService):
params = {
"output_format": self._output_format,
}
if self._settings["optimize_streaming_latency"] is not None:
params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"]
if (
is_given(self._settings.optimize_streaming_latency)
and self._settings.optimize_streaming_latency is not None
):
params["optimize_streaming_latency"] = self._settings.optimize_streaming_latency
try:
await self.start_ttfb_metrics()

5
src/pipecat/services/fal/image.py
View File

@@ -78,7 +78,8 @@ class FalImageGenService(ImageGenService):
**kwargs: Additional arguments passed to parent ImageGenService.
"""
super().__init__(**kwargs)
self.set_model_name(model)
self._settings.model = model
self._sync_model_name_to_metrics()
self._params = params
self._aiohttp_session = aiohttp_session
if key:
@@ -103,7 +104,7 @@ class FalImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
response = await fal_client.run_async(
self.model_name,
self._settings.model,
arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)},
)

58
src/pipecat/services/fal/stt.py
View File

@@ -11,12 +11,14 @@ transcription using segmented audio processing.
"""
import os
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import FAL_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -146,6 +148,22 @@ def language_to_fal_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class FalSTTSettings(STTSettings):
"""Settings for the Fal Wizper STT service.
Parameters:
task: Task to perform ('transcribe' or 'translate'). Defaults to
'transcribe'.
chunk_level: Level of chunking ('segment'). Defaults to 'segment'.
version: Version of Wizper model to use. Defaults to '3'.
"""
task: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
chunk_level: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
version: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class FalSTTService(SegmentedSTTService):
"""Speech-to-text service using Fal's Wizper API.
@@ -153,6 +171,8 @@ class FalSTTService(SegmentedSTTService):
segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
"""
_settings: FalSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Fal's Wizper API.
@@ -203,14 +223,14 @@ class FalSTTService(SegmentedSTTService):
)
self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY"))
self._settings = {
"task": params.task,
"language": self.language_to_service_language(params.language)
self._settings = FalSTTSettings(
language=self.language_to_service_language(params.language)
if params.language
else "en",
"chunk_level": params.chunk_level,
"version": params.version,
}
task=params.task,
chunk_level=params.chunk_level,
version=params.version,
)
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -231,24 +251,6 @@ class FalSTTService(SegmentedSTTService):
"""
return language_to_fal_language(language)
async def set_language(self, language: Language):
"""Set the transcription language.
Args:
language: The language to use for speech-to-text transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = self.language_to_service_language(language)
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[str] = None
@@ -276,19 +278,19 @@ class FalSTTService(SegmentedSTTService):
data_uri = fal_client.encode(audio, "audio/x-wav")
response = await self._fal_client.run(
"fal-ai/wizper",
arguments={"audio_url": data_uri, **self._settings},
arguments={"audio_url": data_uri, **self._settings.given_fields()},
)
if response and "text" in response:
text = response["text"].strip()
if text: # Only yield non-empty text
await self._handle_transcription(text, True, self._settings["language"])
await self._handle_transcription(text, True, self._settings.language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text,
self._user_id,
time_now_iso8601(),
Language(self._settings["language"]),
Language(self._settings.language),
result=response,
)

14
src/pipecat/services/fireworks/llm.py
View File

@@ -66,17 +66,17 @@ class FireworksLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

105
src/pipecat/services/fish/tts.py
View File

@@ -11,7 +11,8 @@ for streaming text-to-speech synthesis with customizable voice parameters.
"""
import uuid
from typing import AsyncGenerator, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Literal, Mapping, Optional
from loguru import logger
from pydantic import BaseModel
@@ -28,6 +29,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -45,6 +47,41 @@ except ModuleNotFoundError as e:
FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"]
@dataclass
class FishAudioTTSSettings(TTSSettings):
"""Settings for Fish Audio TTS service.
Parameters:
fish_sample_rate: Audio sample rate sent to the API.
latency: Latency mode ("normal" or "balanced"). Defaults to "normal".
format: Audio output format.
normalize: Whether to normalize audio output. Defaults to True.
prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0.
prosody_volume: Volume adjustment in dB. Defaults to 0.
reference_id: Reference ID of the voice model.
"""
fish_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
latency: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
normalize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_volume: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
reference_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "fish_sample_rate"}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "FishAudioTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``prosody``."""
flat = dict(settings)
nested = flat.pop("prosody", None)
if isinstance(nested, dict):
flat.setdefault("prosody_speed", nested.get("speed"))
flat.setdefault("prosody_volume", nested.get("volume"))
return super().from_mapping(flat)
class FishAudioTTSService(InterruptibleTTSService):
"""Fish Audio text-to-speech service with WebSocket streaming.
@@ -53,6 +90,8 @@ class FishAudioTTSService(InterruptibleTTSService):
audio generation with interruption handling.
"""
_settings: FishAudioTTSSettings
class InputParams(BaseModel):
"""Input parameters for Fish Audio TTS configuration.
@@ -136,19 +175,18 @@ class FishAudioTTSService(InterruptibleTTSService):
self._receive_task = None
self._request_id = None
self._settings = {
"sample_rate": 0,
"latency": params.latency,
"format": output_format,
"normalize": params.normalize,
"prosody": {
"speed": params.prosody_speed,
"volume": params.prosody_volume,
},
"reference_id": reference_id,
}
self.set_model_name(model_id)
self._settings = FishAudioTTSSettings(
model=model_id,
voice=reference_id,
fish_sample_rate=0,
latency=params.latency,
format=output_format,
normalize=params.normalize,
prosody_speed=params.prosody_speed,
prosody_volume=params.prosody_volume,
reference_id=reference_id,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -158,16 +196,24 @@ class FishAudioTTSService(InterruptibleTTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model and reconnect.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and reconnect if needed.
Any change to voice or model triggers a WebSocket reconnect.
Args:
model: The model name to use for synthesis.
update: A :class:`TTSSettings` (or ``FishAudioTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
changed = await super()._update_settings(update)
if changed:
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Fish Audio TTS service.
@@ -176,7 +222,7 @@ class FishAudioTTSService(InterruptibleTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.fish_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -221,11 +267,22 @@ class FishAudioTTSService(InterruptibleTTSService):
logger.debug("Connecting to Fish Audio")
headers = {"Authorization": f"Bearer {self._api_key}"}
headers["model"] = self.model_name
headers["model"] = self._settings.model
self._websocket = await websocket_connect(self._base_url, additional_headers=headers)
# Send initial start message with ormsgpack
start_message = {"event": "start", "request": {"text": "", **self._settings}}
request_settings = {
"sample_rate": self._settings.fish_sample_rate,
"latency": self._settings.latency,
"format": self._settings.format,
"normalize": self._settings.normalize,
"prosody": {
"speed": self._settings.prosody_speed,
"volume": self._settings.prosody_volume,
},
"reference_id": self._settings.reference_id,
}
start_message = {"event": "start", "request": {"text": "", **request_settings}}
await self._websocket.send(ormsgpack.packb(start_message))
logger.debug("Sent start message to Fish Audio")

94
src/pipecat/services/gladia/stt.py
View File

@@ -14,6 +14,7 @@ import asyncio
import base64
import json
import warnings
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Literal, Optional
import aiohttp
@@ -32,6 +33,7 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
)
from pipecat.services.gladia.config import GladiaInputParams
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GLADIA_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -178,6 +180,17 @@ class _InputParamsDescriptor:
return GladiaInputParams
@dataclass
class GladiaSTTSettings(STTSettings):
"""Settings for Gladia STT service.
Parameters:
input_params: Gladia ``GladiaInputParams`` for detailed configuration.
"""
input_params: GladiaInputParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GladiaSTTService(WebsocketSTTService):
"""Speech-to-Text service using Gladia's API.
@@ -191,6 +204,8 @@ class GladiaSTTService(WebsocketSTTService):
Use :class:`~pipecat.services.gladia.config.GladiaInputParams` directly instead.
"""
_settings: GladiaSTTSettings
# Maintain backward compatibility
InputParams = _InputParamsDescriptor()
@@ -264,10 +279,9 @@ class GladiaSTTService(WebsocketSTTService):
self._api_key = api_key
self._region = region
self._url = url
self.set_model_name(model)
self._params = params
self._receive_task = None
self._settings = {}
self._settings = GladiaSTTSettings(model=model, input_params=params)
self._sync_model_name_to_metrics()
# Session management
self._session_url = None
@@ -307,31 +321,33 @@ class GladiaSTTService(WebsocketSTTService):
return language_to_gladia_language(language)
def _prepare_settings(self) -> Dict[str, Any]:
params = self._settings.input_params
settings = {
"encoding": self._params.encoding or "wav/pcm",
"bit_depth": self._params.bit_depth or 16,
"encoding": params.encoding or "wav/pcm",
"bit_depth": params.bit_depth or 16,
"sample_rate": self.sample_rate,
"channels": self._params.channels or 1,
"model": self._model_name,
"channels": params.channels or 1,
"model": self._settings.model,
}
# Add custom_metadata if provided
settings["custom_metadata"] = dict(self._params.custom_metadata or {})
settings["custom_metadata"] = dict(params.custom_metadata or {})
settings["custom_metadata"]["pipecat"] = pipecat_version()
# Add endpointing parameters if provided
if self._params.endpointing is not None:
settings["endpointing"] = self._params.endpointing
if self._params.maximum_duration_without_endpointing is not None:
if params.endpointing is not None:
settings["endpointing"] = params.endpointing
if params.maximum_duration_without_endpointing is not None:
settings["maximum_duration_without_endpointing"] = (
self._params.maximum_duration_without_endpointing
params.maximum_duration_without_endpointing
)
# Add language configuration (prioritize language_config over deprecated language)
if self._params.language_config:
settings["language_config"] = self._params.language_config.model_dump(exclude_none=True)
elif self._params.language: # Backward compatibility for deprecated parameter
language_code = self.language_to_service_language(self._params.language)
if params.language_config:
settings["language_config"] = params.language_config.model_dump(exclude_none=True)
elif params.language: # Backward compatibility for deprecated parameter
language_code = self.language_to_service_language(params.language)
if language_code:
settings["language_config"] = {
"languages": [language_code],
@@ -339,21 +355,18 @@ class GladiaSTTService(WebsocketSTTService):
}
# Add pre_processing configuration if provided
if self._params.pre_processing:
settings["pre_processing"] = self._params.pre_processing.model_dump(exclude_none=True)
if params.pre_processing:
settings["pre_processing"] = params.pre_processing.model_dump(exclude_none=True)
# Add realtime_processing configuration if provided
if self._params.realtime_processing:
settings["realtime_processing"] = self._params.realtime_processing.model_dump(
if params.realtime_processing:
settings["realtime_processing"] = params.realtime_processing.model_dump(
exclude_none=True
)
# Add messages_config if provided
if self._params.messages_config:
settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True)
# Store settings for tracing
self._settings = settings
if params.messages_config:
settings["messages_config"] = params.messages_config.model_dump(exclude_none=True)
return settings
@@ -366,6 +379,33 @@ class GladiaSTTService(WebsocketSTTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, update: GladiaSTTSettings) -> dict[str, Any]:
"""Apply settings update.
Settings are stored but not applied to the active session.
Args:
update: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# self._session_url = None
# self._session_id = None
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def stop(self, frame: EndFrame):
"""Stop the Gladia STT websocket connection.
@@ -522,7 +562,7 @@ class GladiaSTTService(WebsocketSTTService):
Broadcasts UserStartedSpeakingFrame and optionally triggers interruption
when VAD is enabled.
"""
if not self._params.enable_vad or self._is_speaking:
if not self._settings.input_params.enable_vad or self._is_speaking:
return
logger.debug(f"{self} User started speaking")
@@ -537,7 +577,7 @@ class GladiaSTTService(WebsocketSTTService):
Broadcasts UserStoppedSpeakingFrame when VAD is enabled.
"""
if not self._params.enable_vad or not self._is_speaking:
if not self._settings.input_params.enable_vad or not self._is_speaking:
return
self._is_speaking = False
await self.broadcast_frame(UserStoppedSpeakingFrame)

149
src/pipecat/services/google/gemini_live/llm.py
View File

@@ -17,9 +17,9 @@ import io
import time
import uuid
import warnings
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from typing import Any, ClassVar, Dict, List, Optional, Union
from loguru import logger
from PIL import Image
@@ -47,7 +47,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
TranscriptionFrame,
TTSAudioRawFrame,
@@ -77,6 +76,7 @@ from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.string import match_endofsentence
from pipecat.utils.time import time_now_iso8601
@@ -602,6 +602,33 @@ class InputParams(BaseModel):
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
@dataclass
class GeminiLiveLLMSettings(LLMSettings):
"""Settings for Gemini Live LLM services.
Parameters:
modalities: Response modalities.
language: Language for generation.
media_resolution: Media resolution setting.
vad: Voice activity detection parameters.
context_window_compression: Context window compression configuration.
thinking: Thinking configuration.
enable_affective_dialog: Whether to enable affective dialog.
proactivity: Proactivity configuration.
"""
modalities: GeminiModalities | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
media_resolution: GeminiMediaResolution | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad: GeminiVADParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
context_window_compression: ContextWindowCompressionParams | dict | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
thinking: ThinkingConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_affective_dialog: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
proactivity: ProactivityConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GeminiLiveLLMService(LLMService):
"""Provides access to Google's Gemini Live API.
@@ -610,6 +637,8 @@ class GeminiLiveLLMService(LLMService):
responses, and tool usage.
"""
_settings: GeminiLiveLLMSettings
# Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter
@@ -672,7 +701,6 @@ class GeminiLiveLLMService(LLMService):
self._last_sent_time = 0
self._base_url = base_url
self.set_model_name(model)
self._voice_id = voice_id
self._language_code = params.language
@@ -714,25 +742,27 @@ class GeminiLiveLLMService(LLMService):
self._consecutive_failures = 0
self._connection_start_time = None
self._settings = {
"frequency_penalty": params.frequency_penalty,
"max_tokens": params.max_tokens,
"presence_penalty": params.presence_penalty,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"modalities": params.modalities,
"language": self._language_code,
"media_resolution": params.media_resolution,
"vad": params.vad,
"context_window_compression": params.context_window_compression.model_dump()
self._settings = GeminiLiveLLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
max_tokens=params.max_tokens,
presence_penalty=params.presence_penalty,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
modalities=params.modalities,
language=self._language_code,
media_resolution=params.media_resolution,
vad=params.vad,
context_window_compression=params.context_window_compression.model_dump()
if params.context_window_compression
else {},
"thinking": params.thinking or {},
"enable_affective_dialog": params.enable_affective_dialog or False,
"proactivity": params.proactivity or {},
"extra": params.extra if isinstance(params.extra, dict) else {},
}
thinking=params.thinking or {},
enable_affective_dialog=params.enable_affective_dialog or False,
proactivity=params.proactivity or {},
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
self._file_api_base_url = file_api_base_url
self._file_api: Optional[GeminiFileAPI] = None
@@ -776,6 +806,25 @@ class GeminiLiveLLMService(LLMService):
"""
return True
async def _update_settings(self, update: LLMSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
def set_audio_input_paused(self, paused: bool):
"""Set the audio input pause state.
@@ -798,7 +847,7 @@ class GeminiLiveLLMService(LLMService):
Args:
modalities: The modalities to use for responses.
"""
self._settings["modalities"] = modalities
self._settings.modalities = modalities
def set_language(self, language: Language):
"""Set the language for generation.
@@ -808,7 +857,7 @@ class GeminiLiveLLMService(LLMService):
"""
self._language = language
self._language_code = language_to_gemini_language(language) or "en-US"
self._settings["language"] = self._language_code
self._settings.language = self._language_code
logger.info(f"Set Gemini language to: {self._language_code}")
async def set_context(self, context: OpenAILLMContext):
@@ -866,7 +915,7 @@ class GeminiLiveLLMService(LLMService):
async def _handle_interruption(self):
if self._bot_is_responding:
await self._set_bot_is_responding(False)
if self._settings.get("modalities") == GeminiModalities.AUDIO:
if self._settings.modalities == GeminiModalities.AUDIO:
await self.push_frame(TTSStoppedFrame())
# Do not send LLMFullResponseEndFrame here - an interruption
# already tells the assistant context aggregator that the response
@@ -947,10 +996,9 @@ class GeminiLiveLLMService(LLMService):
# uses this frame *without* a user context aggregator still works
# (we have an example that does just that, actually).
await self._create_single_response(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
# TODO: implement runtime tool updates for Gemini Live.
pass
else:
await self.push_frame(frame, direction)
@@ -1074,20 +1122,20 @@ class GeminiLiveLLMService(LLMService):
# Assemble basic configuration
config = LiveConnectConfig(
generation_config=GenerationConfig(
frequency_penalty=self._settings["frequency_penalty"],
max_output_tokens=self._settings["max_tokens"],
presence_penalty=self._settings["presence_penalty"],
temperature=self._settings["temperature"],
top_k=self._settings["top_k"],
top_p=self._settings["top_p"],
response_modalities=[Modality(self._settings["modalities"].value)],
frequency_penalty=self._settings.frequency_penalty,
max_output_tokens=self._settings.max_tokens,
presence_penalty=self._settings.presence_penalty,
temperature=self._settings.temperature,
top_k=self._settings.top_k,
top_p=self._settings.top_p,
response_modalities=[Modality(self._settings.modalities.value)],
speech_config=SpeechConfig(
voice_config=VoiceConfig(
prebuilt_voice_config={"voice_name": self._voice_id}
),
language_code=self._settings["language"],
language_code=self._settings.language,
),
media_resolution=MediaResolution(self._settings["media_resolution"].value),
media_resolution=MediaResolution(self._settings.media_resolution.value),
),
input_audio_transcription=AudioTranscriptionConfig(),
output_audio_transcription=AudioTranscriptionConfig(),
@@ -1095,37 +1143,36 @@ class GeminiLiveLLMService(LLMService):
)
# Add context window compression to configuration, if enabled
if self._settings.get("context_window_compression", {}).get("enabled", False):
cwc = self._settings.context_window_compression or {}
if cwc.get("enabled", False):
compression_config = ContextWindowCompressionConfig()
# Add sliding window (always true if compression is enabled)
compression_config.sliding_window = SlidingWindow()
# Add trigger_tokens if specified
trigger_tokens = self._settings.get("context_window_compression", {}).get(
"trigger_tokens"
)
trigger_tokens = cwc.get("trigger_tokens")
if trigger_tokens is not None:
compression_config.trigger_tokens = trigger_tokens
config.context_window_compression = compression_config
# Add thinking configuration to configuration, if provided
if self._settings.get("thinking"):
config.thinking_config = self._settings["thinking"]
if self._settings.thinking:
config.thinking_config = self._settings.thinking
# Add affective dialog setting, if provided
if self._settings.get("enable_affective_dialog", False):
config.enable_affective_dialog = self._settings["enable_affective_dialog"]
if self._settings.enable_affective_dialog:
config.enable_affective_dialog = self._settings.enable_affective_dialog
# Add proactivity configuration to configuration, if provided
if self._settings.get("proactivity"):
config.proactivity = self._settings["proactivity"]
if self._settings.proactivity:
config.proactivity = self._settings.proactivity
# Add VAD configuration to configuration, if provided
if self._settings.get("vad"):
if self._settings.vad:
vad_config = AutomaticActivityDetection()
vad_params = self._settings["vad"]
vad_params = self._settings.vad
has_vad_settings = False
# Only add parameters that are explicitly set
@@ -1183,7 +1230,9 @@ class GeminiLiveLLMService(LLMService):
await self.push_error(error_msg=f"Initialization error: {e}", exception=e)
async def _connection_task_handler(self, config: LiveConnectConfig):
async with self._client.aio.live.connect(model=self._model_name, config=config) as session:
async with self._client.aio.live.connect(
model=self._settings.model, config=config
) as session:
logger.info("Connected to Gemini service")
# Mark connection start time
@@ -1604,7 +1653,7 @@ class GeminiLiveLLMService(LLMService):
text: The transcription text to push
result: Optional LiveServerMessage that triggered this transcription
"""
await self._handle_user_transcription(text, True, self._settings["language"])
await self._handle_user_transcription(text, True, self._settings.language)
await self.push_frame(
TranscriptionFrame(
text=text,

4
src/pipecat/services/google/image.py
View File

@@ -79,7 +79,9 @@ class GoogleImageGenService(ImageGenService):
http_options = update_google_client_http_options(http_options)
self._client = genai.Client(api_key=api_key, http_options=http_options)
self.set_model_name(self._params.model)
self._settings.model = self._params.model
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

152
src/pipecat/services/google/llm.py
View File

@@ -15,8 +15,8 @@ import io
import json
import os
import uuid
from dataclasses import dataclass
from typing import Any, AsyncIterator, Dict, List, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncIterator, ClassVar, Dict, List, Literal, Optional
from loguru import logger
from PIL import Image
@@ -39,7 +39,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -59,6 +58,7 @@ from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven, is_given
from pipecat.utils.tracing.service_decorators import traced_llm
# Suppress gRPC fork warnings
@@ -673,6 +673,62 @@ class GoogleLLMContext(OpenAILLMContext):
self._messages = [m for m in self._messages if m.parts]
class GoogleThinkingConfig(BaseModel):
"""Configuration for controlling the model's internal "thinking" process used before generating a response.
Gemini 2.5 and 3 series models have this thinking process.
Parameters:
thinking_level: Thinking level for Gemini 3 models.
For Gemini 3 Pro, this can be "low" or "high".
For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high".
If not provided, Gemini 3 models default to "high".
Note: Gemini 2.5 series must use thinking_budget instead.
thinking_budget: Token budget for thinking, for Gemini 2.5 series.
-1 for dynamic thinking (model decides), 0 to disable thinking,
or a specific token count (e.g., 128-32768 for 2.5 Pro).
If not provided, most models today default to dynamic thinking.
See https://ai.google.dev/gemini-api/docs/thinking#set-budget
for default values and allowed ranges.
Note: Gemini 3 models must use thinking_level instead.
include_thoughts: Whether to include thought summaries in the response.
Today's models default to not including thoughts (False).
"""
thinking_budget: Optional[int] = Field(default=None)
# Why `| str` here? To not break compatibility in case Google adds more
# levels in the future.
thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field(
default=None
)
include_thoughts: Optional[bool] = Field(default=None)
@dataclass
class GoogleLLMSettings(LLMSettings):
"""Settings for Google LLM services.
Parameters:
thinking: Thinking configuration.
"""
thinking: GoogleThinkingConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings):
"""Convert a plain dict to settings, coercing thinking dicts.
For backward compatibility, a ``thinking`` value that is a plain dict
is converted to a :class:`GoogleThinkingConfig`.
"""
instance = super().from_mapping(settings)
if is_given(instance.thinking) and isinstance(instance.thinking, dict):
instance.thinking = GoogleThinkingConfig(**instance.thinking)
return instance
class GoogleLLMService(LLMService):
"""Google AI (Gemini) LLM service implementation.
@@ -681,40 +737,13 @@ class GoogleLLMService(LLMService):
expected by the Google AI model.
"""
_settings: GoogleLLMSettings
# Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter
class ThinkingConfig(BaseModel):
"""Configuration for controlling the model's internal "thinking" process used before generating a response.
Gemini 2.5 and 3 series models have this thinking process.
Parameters:
thinking_level: Thinking level for Gemini 3 models.
For Gemini 3 Pro, this can be "low" or "high".
For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high".
If not provided, Gemini 3 models default to "high".
Note: Gemini 2.5 series must use thinking_budget instead.
thinking_budget: Token budget for thinking, for Gemini 2.5 series.
-1 for dynamic thinking (model decides), 0 to disable thinking,
or a specific token count (e.g., 128-32768 for 2.5 Pro).
If not provided, most models today default to dynamic thinking.
See https://ai.google.dev/gemini-api/docs/thinking#set-budget
for default values and allowed ranges.
Note: Gemini 3 models must use thinking_level instead.
include_thoughts: Whether to include thought summaries in the response.
Today's models default to not including thoughts (False).
"""
thinking_budget: Optional[int] = Field(default=None)
# Why `| str` here? To not break compatibility in case Google adds more
# levels in the future.
thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field(
default=None
)
include_thoughts: Optional[bool] = Field(default=None)
# Backward compatibility: ThinkingConfig used to be defined inline here.
ThinkingConfig = GoogleThinkingConfig
class InputParams(BaseModel):
"""Input parameters for Google AI models.
@@ -737,7 +766,7 @@ class GoogleLLMService(LLMService):
temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
top_k: Optional[int] = Field(default=None, ge=0)
top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0)
thinking: Optional["GoogleLLMService.ThinkingConfig"] = Field(default=None)
thinking: Optional[GoogleThinkingConfig] = Field(default=None)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def __init__(
@@ -768,19 +797,20 @@ class GoogleLLMService(LLMService):
params = params or GoogleLLMService.InputParams()
self.set_model_name(model)
self._api_key = api_key
self._system_instruction = system_instruction
self._http_options = update_google_client_http_options(http_options)
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"thinking": params.thinking,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self._settings = GoogleLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
self._tools = tools
self._tool_config = tool_config
@@ -840,7 +870,7 @@ class GoogleLLMService(LLMService):
# Use the new google-genai client's async method
response = await self._client.aio.models.generate_content(
model=self._model_name,
model=self._settings.model,
contents=messages,
config=generation_config,
)
@@ -874,10 +904,10 @@ class GoogleLLMService(LLMService):
k: v
for k, v in {
"system_instruction": system_instruction,
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"top_k": self._settings["top_k"],
"max_output_tokens": self._settings["max_tokens"],
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"top_k": self._settings.top_k,
"max_output_tokens": self._settings.max_tokens,
"tools": tools,
"tool_config": tool_config,
}.items()
@@ -885,13 +915,13 @@ class GoogleLLMService(LLMService):
}
# Add thinking parameters if configured
if self._settings["thinking"]:
generation_params["thinking_config"] = self._settings["thinking"].model_dump(
if self._settings.thinking:
generation_params["thinking_config"] = self._settings.thinking.model_dump(
exclude_unset=True
)
if self._settings["extra"]:
generation_params.update(self._settings["extra"])
if self._settings.extra:
generation_params.update(self._settings.extra)
return generation_params
@@ -900,10 +930,10 @@ class GoogleLLMService(LLMService):
# There's no way to introspect on model capabilities, so
# to check for models that we know default to thinkin on
# and can be configured to turn it off.
if not self._model_name.startswith("gemini-2.5-flash"):
if not self._settings.model.startswith("gemini-2.5-flash"):
return
# If we have an image model, we don't use a budget either.
if "image" in self._model_name:
if "image" in self._settings.model:
return
# If thinking_config is already set, don't override it.
if "thinking_config" in generation_params:
@@ -944,7 +974,7 @@ class GoogleLLMService(LLMService):
await self.start_ttfb_metrics()
return await self._client.aio.models.generate_content_stream(
model=self._model_name,
model=self._settings.model,
contents=messages,
config=generation_config,
)
@@ -1190,8 +1220,6 @@ class GoogleLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = GoogleLLMContext(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -1215,14 +1243,6 @@ class GoogleLLMService(LLMService):
# Do nothing - we're shutting down anyway
pass
async def _update_settings(self, settings):
"""Override to handle ThinkingConfig validation."""
# Convert thinking dict to ThinkingConfig if needed
if "thinking" in settings and isinstance(settings["thinking"], dict):
settings = dict(settings) # Make a copy to avoid modifying the original
settings["thinking"] = self.ThinkingConfig(**settings["thinking"])
await super()._update_settings(settings)
def create_context_aggregator(
self,
context: OpenAILLMContext,

245
src/pipecat/services/google/stt.py
View File

@@ -15,13 +15,15 @@ import asyncio
import json
import os
import time
import warnings
from dataclasses import dataclass, field
from pipecat.utils.tracing.service_decorators import traced_stt
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from typing import AsyncGenerator, List, Optional, Union
from typing import Any, AsyncGenerator, List, Optional, Union
from loguru import logger
from pydantic import BaseModel, Field, field_validator
@@ -34,6 +36,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GOOGLE_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -355,6 +358,46 @@ def language_to_google_stt_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class GoogleSTTSettings(STTSettings):
"""Settings for Google Cloud Speech-to-Text V2.
Parameters:
languages: List of ``Language`` enums for recognition
(e.g. ``[Language.EN_US]``). Preferred over ``language_codes``.
language_codes: List of Google STT language code strings
(e.g. ``["en-US"]``).
.. deprecated:: 0.0.103
Use ``languages`` instead. If both are provided, ``languages``
takes precedence. This field is here just for backward
compatibility with dict-based settings updates.
use_separate_recognition_per_channel: Process each audio channel separately.
enable_automatic_punctuation: Add punctuation to transcripts.
enable_spoken_punctuation: Include spoken punctuation in transcript.
enable_spoken_emojis: Include spoken emojis in transcript.
profanity_filter: Filter profanity from transcript.
enable_word_time_offsets: Include timing information for each word.
enable_word_confidence: Include confidence scores for each word.
enable_interim_results: Stream partial recognition results.
enable_voice_activity_events: Detect voice activity in audio.
"""
languages: List[Language] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_codes: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_separate_recognition_per_channel: bool | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
enable_automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_spoken_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_spoken_emojis: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_word_time_offsets: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_word_confidence: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_interim_results: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_voice_activity_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GoogleSTTService(STTService):
"""Google Cloud Speech-to-Text V2 service implementation.
@@ -371,6 +414,8 @@ class GoogleSTTService(STTService):
ValueError: If project ID is not found in credentials.
"""
_settings: GoogleSTTSettings
# Google Cloud's STT service has a connection time limit of 5 minutes per stream.
# They've shared an "endless streaming" example that guided this implementation:
# https://cloud.google.com/speech-to-text/docs/transcribe-streaming-audio#endless-streaming
@@ -508,21 +553,19 @@ class GoogleSTTService(STTService):
self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options)
self._settings = {
"language_codes": [
self.language_to_service_language(lang) for lang in params.language_list
],
"model": params.model,
"use_separate_recognition_per_channel": params.use_separate_recognition_per_channel,
"enable_automatic_punctuation": params.enable_automatic_punctuation,
"enable_spoken_punctuation": params.enable_spoken_punctuation,
"enable_spoken_emojis": params.enable_spoken_emojis,
"profanity_filter": params.profanity_filter,
"enable_word_time_offsets": params.enable_word_time_offsets,
"enable_word_confidence": params.enable_word_confidence,
"enable_interim_results": params.enable_interim_results,
"enable_voice_activity_events": params.enable_voice_activity_events,
}
self._settings = GoogleSTTSettings(
languages=list(params.language_list),
model=params.model,
use_separate_recognition_per_channel=params.use_separate_recognition_per_channel,
enable_automatic_punctuation=params.enable_automatic_punctuation,
enable_spoken_punctuation=params.enable_spoken_punctuation,
enable_spoken_emojis=params.enable_spoken_emojis,
profanity_filter=params.profanity_filter,
enable_word_time_offsets=params.enable_word_time_offsets,
enable_word_confidence=params.enable_word_confidence,
enable_interim_results=params.enable_interim_results,
enable_voice_activity_events=params.enable_voice_activity_events,
)
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
@@ -545,6 +588,23 @@ class GoogleSTTService(STTService):
return [language_to_google_stt_language(lang) or "en-US" for lang in language]
return language_to_google_stt_language(language) or "en-US"
def _get_language_codes(self) -> List[str]:
"""Resolve the current language settings to Google STT language code strings.
Prefers ``languages`` (``Language`` enums) over the deprecated
``language_codes`` (raw strings). Falls back to ``["en-US"]``.
Returns:
List[str]: Google STT language code strings.
"""
from pipecat.services.settings import is_given
if is_given(self._settings.languages):
return [self.language_to_service_language(lang) for lang in self._settings.languages]
if is_given(self._settings.language_codes):
return list(self._settings.language_codes)
return ["en-US"]
async def _reconnect_if_needed(self):
"""Reconnect the stream if it's currently active."""
if self._streaming_task:
@@ -552,41 +612,65 @@ class GoogleSTTService(STTService):
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
"""Update the service's recognition language.
A convenience method for setting a single language.
Args:
language: New language for recognition.
"""
logger.debug(f"Switching STT language to: {language}")
await self.set_languages([language])
async def set_languages(self, languages: List[Language]):
"""Update the service's recognition languages.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(languages=...)``
instead.
Args:
languages: List of languages for recognition. First language is primary.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"set_languages() is deprecated. Use STTUpdateSettingsFrame with "
"GoogleSTTSettings(languages=...) instead.",
DeprecationWarning,
)
logger.debug(f"Switching STT languages to: {languages}")
self._settings["language_codes"] = [
self.language_to_service_language(lang) for lang in languages
]
# Recreate stream with new languages
await self._reconnect_if_needed()
await self._update_settings(GoogleSTTSettings(languages=list(languages)))
async def set_model(self, model: str):
"""Update the service's recognition model.
async def _update_settings(self, update: GoogleSTTSettings) -> dict[str, Any]:
"""Apply settings update and reconnect if anything changed.
Handles ``language`` from base ``set_language`` by converting it to
``languages``. Emits a deprecation warning if ``language_codes`` is
used. All other fields (model, boolean flags) are applied directly.
Reconnects the stream on any change.
Args:
model: The new recognition model to use.
update: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.debug(f"Switching STT model to: {model}")
await super().set_model(model)
self._settings["model"] = model
# Recreate stream with new model
await self._reconnect_if_needed()
from pipecat.services.settings import is_given
# If base set_language sent a Language value, convert to languages list
if is_given(update.language):
update.languages = [update.language]
# Clear language so the base class doesn't try to store it
update.language = NOT_GIVEN
# Warn on deprecated language_codes usage
if is_given(update.language_codes):
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"GoogleSTTSettings.language_codes is deprecated. "
"Use GoogleSTTSettings.languages (List[Language]) instead.",
DeprecationWarning,
stacklevel=2,
)
changed = await super()._update_settings(update)
if changed:
await self._reconnect_if_needed()
return changed
async def start(self, frame: StartFrame):
"""Start the STT service and establish connection.
@@ -632,6 +716,10 @@ class GoogleSTTService(STTService):
) -> None:
"""Update service options dynamically.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(...)``
instead.
Args:
languages: New list of recognition languages.
model: New recognition model.
@@ -649,55 +737,42 @@ class GoogleSTTService(STTService):
Changes that affect the streaming configuration will cause
the stream to be reconnected.
"""
# Update settings with new values
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"update_options() is deprecated. Use STTUpdateSettingsFrame with "
"GoogleSTTSettings(...) instead.",
DeprecationWarning,
)
# Build a settings delta from the provided options
update = GoogleSTTSettings()
if languages is not None:
logger.debug(f"Updating language to: {languages}")
self._settings["language_codes"] = [
self.language_to_service_language(lang) for lang in languages
]
update.languages = list(languages)
if model is not None:
logger.debug(f"Updating model to: {model}")
self._settings["model"] = model
update.model = model
if enable_automatic_punctuation is not None:
logger.debug(f"Updating automatic punctuation to: {enable_automatic_punctuation}")
self._settings["enable_automatic_punctuation"] = enable_automatic_punctuation
update.enable_automatic_punctuation = enable_automatic_punctuation
if enable_spoken_punctuation is not None:
logger.debug(f"Updating spoken punctuation to: {enable_spoken_punctuation}")
self._settings["enable_spoken_punctuation"] = enable_spoken_punctuation
update.enable_spoken_punctuation = enable_spoken_punctuation
if enable_spoken_emojis is not None:
logger.debug(f"Updating spoken emojis to: {enable_spoken_emojis}")
self._settings["enable_spoken_emojis"] = enable_spoken_emojis
update.enable_spoken_emojis = enable_spoken_emojis
if profanity_filter is not None:
logger.debug(f"Updating profanity filter to: {profanity_filter}")
self._settings["profanity_filter"] = profanity_filter
update.profanity_filter = profanity_filter
if enable_word_time_offsets is not None:
logger.debug(f"Updating word time offsets to: {enable_word_time_offsets}")
self._settings["enable_word_time_offsets"] = enable_word_time_offsets
update.enable_word_time_offsets = enable_word_time_offsets
if enable_word_confidence is not None:
logger.debug(f"Updating word confidence to: {enable_word_confidence}")
self._settings["enable_word_confidence"] = enable_word_confidence
update.enable_word_confidence = enable_word_confidence
if enable_interim_results is not None:
logger.debug(f"Updating interim results to: {enable_interim_results}")
self._settings["enable_interim_results"] = enable_interim_results
update.enable_interim_results = enable_interim_results
if enable_voice_activity_events is not None:
logger.debug(f"Updating voice activity events to: {enable_voice_activity_events}")
self._settings["enable_voice_activity_events"] = enable_voice_activity_events
update.enable_voice_activity_events = enable_voice_activity_events
if location is not None:
logger.debug(f"Updating location to: {location}")
self._location = location
# Reconnect the stream for updates
await self._reconnect_if_needed()
await self._update_settings(update)
async def _connect(self):
"""Initialize streaming recognition config and stream."""
@@ -714,20 +789,20 @@ class GoogleSTTService(STTService):
sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
language_codes=self._settings["language_codes"],
model=self._settings["model"],
language_codes=self._get_language_codes(),
model=self._settings.model,
features=cloud_speech.RecognitionFeatures(
enable_automatic_punctuation=self._settings["enable_automatic_punctuation"],
enable_spoken_punctuation=self._settings["enable_spoken_punctuation"],
enable_spoken_emojis=self._settings["enable_spoken_emojis"],
profanity_filter=self._settings["profanity_filter"],
enable_word_time_offsets=self._settings["enable_word_time_offsets"],
enable_word_confidence=self._settings["enable_word_confidence"],
enable_automatic_punctuation=self._settings.enable_automatic_punctuation,
enable_spoken_punctuation=self._settings.enable_spoken_punctuation,
enable_spoken_emojis=self._settings.enable_spoken_emojis,
profanity_filter=self._settings.profanity_filter,
enable_word_time_offsets=self._settings.enable_word_time_offsets,
enable_word_confidence=self._settings.enable_word_confidence,
),
),
streaming_features=cloud_speech.StreamingRecognitionFeatures(
enable_voice_activity_events=self._settings["enable_voice_activity_events"],
interim_results=self._settings["enable_interim_results"],
enable_voice_activity_events=self._settings.enable_voice_activity_events,
interim_results=self._settings.enable_interim_results,
),
)
@@ -857,7 +932,7 @@ class GoogleSTTService(STTService):
if not transcript:
continue
primary_language = self._settings["language_codes"][0]
primary_language = self._get_language_codes()[0]
if result.is_final:
self._last_transcript_was_final = True

243
src/pipecat/services/google/tts.py
View File

@@ -23,7 +23,8 @@ from pipecat.utils.tracing.service_decorators import traced_tts
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional
from loguru import logger
from pydantic import BaseModel
@@ -36,6 +37,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
@@ -474,6 +476,71 @@ def language_to_gemini_tts_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class GoogleHttpTTSSettings(TTSSettings):
"""Settings for Google HTTP TTS service.
Parameters:
pitch: Voice pitch adjustment (e.g., "+2st", "-50%").
rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). Used for
SSML prosody tags (non-Chirp voices).
speaking_rate: Speaking rate for AudioConfig (Chirp/Journey voices).
Range [0.25, 2.0].
volume: Volume adjustment (e.g., "loud", "soft", "+6dB").
emphasis: Emphasis level for the text.
language: Language for synthesis. Defaults to English.
gender: Voice gender preference.
google_style: Google-specific voice style.
"""
pitch: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emphasis: Literal["strong", "moderate", "reduced", "none"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
gender: Literal["male", "female", "neutral"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
google_style: (
Literal["apologetic", "calm", "empathetic", "firm", "lively"] | None | _NotGiven
) = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class GoogleStreamTTSSettings(TTSSettings):
"""Settings for Google streaming TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
speaking_rate: The speaking rate, in the range [0.25, 2.0].
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class GeminiTTSSettings(TTSSettings):
"""Settings for Gemini TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
prompt: Optional style instructions for how to synthesize the content.
multi_speaker: Whether to enable multi-speaker support.
speaker_configs: List of speaker configurations for multi-speaker mode.
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
multi_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_configs: list[dict[str, Any]] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class GoogleHttpTTSService(TTSService):
"""Google Cloud Text-to-Speech HTTP service with SSML support.
@@ -488,6 +555,8 @@ class GoogleHttpTTSService(TTSService):
Chirp and Journey voices don't support SSML and will use plain text input.
"""
_settings: GoogleHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for Google HTTP TTS voice customization.
@@ -538,19 +607,19 @@ class GoogleHttpTTSService(TTSService):
params = params or GoogleHttpTTSService.InputParams()
self._location = location
self._settings = {
"pitch": params.pitch,
"rate": params.rate,
"speaking_rate": params.speaking_rate,
"volume": params.volume,
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
self._settings = GoogleHttpTTSSettings(
pitch=params.pitch,
rate=params.rate,
speaking_rate=params.speaking_rate,
volume=params.volume,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
"gender": params.gender,
"google_style": params.google_style,
}
self.set_voice(voice_id)
gender=params.gender,
google_style=params.google_style,
voice=voice_id,
)
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
@@ -619,61 +688,60 @@ class GoogleHttpTTSService(TTSService):
"""
return language_to_google_tts_language(language)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle speaking_rate updates for Chirp/Journey voices.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Override to handle speaking_rate validation.
Args:
settings: Dictionary of settings to update. Can include 'speaking_rate' (float)
update: Settings delta. Can include 'speaking_rate' (float).
"""
if "speaking_rate" in settings:
rate_value = float(settings["speaking_rate"])
if 0.25 <= rate_value <= 2.0:
self._settings["speaking_rate"] = rate_value
else:
if isinstance(update, GoogleHttpTTSSettings) and is_given(update.speaking_rate):
rate_value = float(update.speaking_rate)
if not (0.25 <= rate_value <= 2.0):
logger.warning(
f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0"
)
await super()._update_settings(settings)
update.speaking_rate = NOT_GIVEN
return await super()._update_settings(update)
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
# Voice tag
voice_attrs = [f"name='{self._voice_id}'"]
voice_attrs = [f"name='{self._settings.voice}'"]
language = self._settings["language"]
language = self._settings.language
voice_attrs.append(f"language='{language}'")
if self._settings["gender"]:
voice_attrs.append(f"gender='{self._settings['gender']}'")
if self._settings.gender:
voice_attrs.append(f"gender='{self._settings.gender}'")
ssml += f"<voice {' '.join(voice_attrs)}>"
# Prosody tag
prosody_attrs = []
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
# Emphasis tag
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
if self._settings.emphasis:
ssml += f"<emphasis level='{self._settings.emphasis}'>"
# Google style tag
if self._settings["google_style"]:
ssml += f"<google:style name='{self._settings['google_style']}'>"
if self._settings.google_style:
ssml += f"<google:style name='{self._settings.google_style}'>"
ssml += text
# Close tags
if self._settings["google_style"]:
if self._settings.google_style:
ssml += "</google:style>"
if self._settings["emphasis"]:
if self._settings.emphasis:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
@@ -698,8 +766,8 @@ class GoogleHttpTTSService(TTSService):
await self.start_ttfb_metrics()
# Check if the voice is a Chirp voice (including Chirp 3) or Journey voice
is_chirp_voice = "chirp" in self._voice_id.lower()
is_journey_voice = "journey" in self._voice_id.lower()
is_chirp_voice = "chirp" in self._settings.voice.lower()
is_journey_voice = "journey" in self._settings.voice.lower()
# Create synthesis input based on voice_id
if is_chirp_voice or is_journey_voice:
@@ -710,7 +778,7 @@ class GoogleHttpTTSService(TTSService):
synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], name=self._voice_id
language_code=self._settings.language, name=self._settings.voice
)
# Build audio config with conditional speaking_rate
audio_config_params = {
@@ -719,8 +787,8 @@ class GoogleHttpTTSService(TTSService):
}
# For Chirp and Journey voices, include speaking_rate in AudioConfig
if (is_chirp_voice or is_journey_voice) and self._settings["speaking_rate"] is not None:
audio_config_params["speaking_rate"] = self._settings["speaking_rate"]
if (is_chirp_voice or is_journey_voice) and self._settings.speaking_rate is not None:
audio_config_params["speaking_rate"] = self._settings.speaking_rate
audio_config = texttospeech_v1.AudioConfig(**audio_config_params)
@@ -910,6 +978,8 @@ class GoogleTTSService(GoogleBaseTTSService):
)
"""
_settings: GoogleStreamTTSSettings
class InputParams(BaseModel):
"""Input parameters for Google streaming TTS configuration.
@@ -950,33 +1020,32 @@ class GoogleTTSService(GoogleBaseTTSService):
params = params or GoogleTTSService.InputParams()
self._location = location
self._settings = {
"language": self.language_to_service_language(params.language)
self._settings = GoogleStreamTTSSettings(
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
"speaking_rate": params.speaking_rate,
}
self.set_voice(voice_id)
speaking_rate=params.speaking_rate,
voice=voice_id,
)
self._voice_cloning_key = voice_cloning_key
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle speaking_rate updates for streaming API.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Override to handle speaking_rate validation.
Args:
settings: Dictionary of settings to update. Can include 'speaking_rate' (float)
update: Settings delta. Can include 'speaking_rate' (float).
"""
if "speaking_rate" in settings:
rate_value = float(settings["speaking_rate"])
if 0.25 <= rate_value <= 2.0:
self._settings["speaking_rate"] = rate_value
else:
if isinstance(update, GoogleStreamTTSSettings) and is_given(update.speaking_rate):
rate_value = float(update.speaking_rate)
if not (0.25 <= rate_value <= 2.0):
logger.warning(
f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0"
)
await super()._update_settings(settings)
update.speaking_rate = NOT_GIVEN
return await super()._update_settings(update)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -1000,11 +1069,11 @@ class GoogleTTSService(GoogleBaseTTSService):
voice_cloning_key=self._voice_cloning_key
)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], voice_clone=voice_clone_params
language_code=self._settings.language, voice_clone=voice_clone_params
)
else:
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], name=self._voice_id
language_code=self._settings.language, name=self._settings.voice
)
# Create streaming config
@@ -1013,7 +1082,7 @@ class GoogleTTSService(GoogleBaseTTSService):
streaming_audio_config=texttospeech_v1.StreamingAudioConfig(
audio_encoding=texttospeech_v1.AudioEncoding.PCM,
sample_rate_hertz=self.sample_rate,
speaking_rate=self._settings["speaking_rate"],
speaking_rate=self._settings.speaking_rate,
),
)
@@ -1052,6 +1121,8 @@ class GeminiTTSService(GoogleBaseTTSService):
)
"""
_settings: GeminiTTSSettings
GOOGLE_SAMPLE_RATE = 24000 # Google TTS always outputs at 24kHz
# List of available Gemini TTS voices
@@ -1158,15 +1229,15 @@ class GeminiTTSService(GoogleBaseTTSService):
self._location = location
self._model = model
self._voice_id = voice_id
self._settings = {
"language": self.language_to_service_language(params.language)
self._settings = GeminiTTSSettings(
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
"prompt": params.prompt,
"multi_speaker": params.multi_speaker,
"speaker_configs": params.speaker_configs,
}
prompt=params.prompt,
multi_speaker=params.multi_speaker,
speaker_configs=params.speaker_configs,
voice=voice_id,
)
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
@@ -1183,16 +1254,6 @@ class GeminiTTSService(GoogleBaseTTSService):
"""
return language_to_gemini_tts_language(language)
def set_voice(self, voice_id: str):
"""Set the voice for TTS generation.
Args:
voice_id: Name of the voice to use from AVAILABLE_VOICES.
"""
if voice_id not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
self._voice_id = voice_id
async def start(self, frame: StartFrame):
"""Start the Gemini TTS service.
@@ -1206,15 +1267,19 @@ class GeminiTTSService(GoogleBaseTTSService):
f"Current rate of {self.sample_rate}Hz may cause issues."
)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle prompt updates.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update with voice validation.
Args:
settings: Dictionary of settings to update. Can include 'prompt' (str)
update: Settings delta. Can include 'voice', 'prompt', etc.
Returns:
Dict mapping changed field names to their previous values.
"""
if "prompt" in settings:
self._settings["prompt"] = settings["prompt"]
await super()._update_settings(settings)
if is_given(update.voice) and update.voice not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{update.voice}' not in known voices list. Using anyway.")
return await super()._update_settings(update)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -1234,14 +1299,14 @@ class GeminiTTSService(GoogleBaseTTSService):
await self.start_ttfb_metrics()
# Build voice selection params
if self._settings["multi_speaker"] and self._settings["speaker_configs"]:
if self._settings.multi_speaker and self._settings.speaker_configs:
# Multi-speaker mode
speaker_voice_configs = []
for speaker_config in self._settings["speaker_configs"]:
for speaker_config in self._settings.speaker_configs:
speaker_voice_configs.append(
texttospeech_v1.MultispeakerPrebuiltVoice(
speaker_alias=speaker_config["speaker_alias"],
speaker_id=speaker_config.get("speaker_id", self._voice_id),
speaker_id=speaker_config.get("speaker_id", self._settings.voice),
)
)
@@ -1250,15 +1315,15 @@ class GeminiTTSService(GoogleBaseTTSService):
)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"],
language_code=self._settings.language,
model_name=self._model,
multi_speaker_voice_config=multi_speaker_voice_config,
)
else:
# Single speaker mode
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"],
name=self._voice_id,
language_code=self._settings.language,
name=self._settings.voice,
model_name=self._model,
)
@@ -1273,7 +1338,7 @@ class GeminiTTSService(GoogleBaseTTSService):
# Use base class streaming logic with prompt support
async for frame in self._stream_tts(
streaming_config, text, context_id, self._settings["prompt"]
streaming_config, text, context_id, self._settings.prompt
):
yield frame

50
src/pipecat/services/gradium/stt.py
View File

@@ -12,7 +12,8 @@ WebSocket API for streaming audio transcription.
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import GRADIUM_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -64,6 +66,18 @@ def language_to_gradium_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class GradiumSTTSettings(STTSettings):
"""Settings for the Gradium STT service.
Parameters:
delay_in_frames: Delay in audio frames (80ms each) before text is
generated. Higher delays allow more context but increase latency.
"""
delay_in_frames: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GradiumSTTService(WebsocketSTTService):
"""Gradium real-time speech-to-text service.
@@ -72,6 +86,8 @@ class GradiumSTTService(WebsocketSTTService):
for audio processing and connection management.
"""
_settings: GradiumSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Gradium STT API.
@@ -127,9 +143,15 @@ class GradiumSTTService(WebsocketSTTService):
self._api_key = api_key
self._api_endpoint_base_url = api_endpoint_base_url
self._websocket = None
self._params = params or GradiumSTTService.InputParams()
self._json_config = json_config
params = params or GradiumSTTService.InputParams()
self._settings = GradiumSTTSettings(
language=params.language,
delay_in_frames=params.delay_in_frames if params.delay_in_frames else NOT_GIVEN,
)
self._receive_task = None
self._audio_buffer = bytearray()
@@ -149,16 +171,22 @@ class GradiumSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update, sync params, and reconnect.
Args:
language: The language to use for speech recognition.
update: A :class:`STTSettings` (or ``GradiumSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.info(f"Switching STT language to: [{language}]")
self._params.language = language
changed = await super()._update_settings(update)
if not changed:
return changed
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
@@ -298,12 +326,12 @@ class GradiumSTTService(WebsocketSTTService):
json_config = {}
if self._json_config:
json_config = json.loads(self._json_config)
if self._params.language:
gradium_language = language_to_gradium_language(self._params.language)
if is_given(self._settings.language) and self._settings.language:
gradium_language = language_to_gradium_language(self._settings.language)
if gradium_language:
json_config["language"] = gradium_language
if self._params.delay_in_frames:
json_config["delay_in_frames"] = self._params.delay_in_frames
if is_given(self._settings.delay_in_frames) and self._settings.delay_in_frames:
json_config["delay_in_frames"] = self._settings.delay_in_frames
if json_config:
setup_msg["json_config"] = json_config
await self._websocket.send(json.dumps(setup_msg))

55
src/pipecat/services/gradium/tts.py
View File

@@ -6,7 +6,8 @@
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextWordTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -38,9 +40,22 @@ except ModuleNotFoundError as e:
SAMPLE_RATE = 48000
@dataclass
class GradiumTTSSettings(TTSSettings):
"""Settings for the Gradium TTS service.
Parameters:
output_format: Audio output format.
"""
output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GradiumTTSService(AudioContextWordTTSService):
"""Text-to-Speech service using Gradium's websocket API."""
_settings: GradiumTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Gradium TTS service.
@@ -85,14 +100,12 @@ class GradiumTTSService(AudioContextWordTTSService):
# Store service configuration
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._json_config = json_config
self._model = model
self._settings = {
"voice_id": voice_id,
"model_name": model,
"output_format": "pcm",
}
self._settings = GradiumTTSSettings(
model=model,
voice=voice_id,
output_format="pcm",
)
# State tracking
self._receive_task = None
@@ -105,24 +118,22 @@ class GradiumTTSService(AudioContextWordTTSService):
"""
return True
async def set_model(self, model: str):
"""Update the TTS model.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and reconnect if voice changed.
Args:
model: The model name to use for synthesis.
"""
self._model = model
await super().set_model(model)
update: A :class:`TTSSettings` (or ``GradiumTTSSettings``) delta.
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
self._settings["voice_id"] = self._voice_id
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if "voice" in changed:
await self._disconnect()
await self._connect()
else:
self._warn_unhandled_updated_settings(changed)
return changed
def _build_msg(self, text: str = "") -> dict:
"""Build JSON message for Gradium API."""
@@ -200,7 +211,7 @@ class GradiumTTSService(AudioContextWordTTSService):
setup_msg = {
"type": "setup",
"output_format": "pcm",
"voice_id": self._voice_id,
"voice_id": self._settings.voice,
"close_ws_on_eos": False,
}
if self._json_config is not None:

118
src/pipecat/services/grok/realtime/llm.py
View File

@@ -13,8 +13,8 @@ https://docs.x.ai/docs/guides/voice/agent
import base64
import json
import time
from dataclasses import dataclass
from typing import Optional
from dataclasses import dataclass, field
from typing import Any, Optional
from loguru import logger
@@ -56,6 +56,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
from . import events
@@ -85,6 +86,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class GrokRealtimeLLMSettings(LLMSettings):
"""Settings for Grok Realtime LLM services.
Parameters:
session_properties: Grok Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class GrokRealtimeLLMService(LLMService):
"""Grok Realtime Voice Agent LLM service providing real-time audio and text communication.
@@ -101,6 +115,8 @@ class GrokRealtimeLLMService(LLMService):
- Server-side VAD (Voice Activity Detection)
"""
_settings: GrokRealtimeLLMSettings
# Use the Grok-specific adapter
adapter_class = GrokRealtimeLLMAdapter
@@ -134,9 +150,8 @@ class GrokRealtimeLLMService(LLMService):
self.api_key = api_key
self.base_url = base_url
# Initialize session_properties
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
self._settings = GrokRealtimeLLMSettings(
session_properties=session_properties or events.SessionProperties(),
)
self._audio_input_paused = start_audio_paused
@@ -186,13 +201,13 @@ class GrokRealtimeLLMService(LLMService):
Configured sample rate or None if not manually configured.
For PCMU/PCMA formats, returns 8000 Hz (G.711 standard).
"""
if not self._session_properties.audio:
if not self._settings.session_properties.audio:
return None
audio_config = (
self._session_properties.audio.input
self._settings.session_properties.audio.input
if direction == "input"
else self._session_properties.audio.output
else self._settings.session_properties.audio.output
)
if audio_config and audio_config.format:
@@ -222,8 +237,8 @@ class GrokRealtimeLLMService(LLMService):
def _is_turn_detection_enabled(self) -> bool:
"""Check if server-side VAD is enabled."""
if self._session_properties.turn_detection:
return self._session_properties.turn_detection.type == "server_vad"
if self._settings.session_properties.turn_detection:
return self._settings.session_properties.turn_detection.type == "server_vad"
return False
async def _handle_interruption(self):
@@ -281,6 +296,27 @@ class GrokRealtimeLLMService(LLMService):
# Standard AIService frame handling
#
def _ensure_audio_config(self, input_sample_rate: int, output_sample_rate: int):
"""Ensure session_properties.audio has input and output configs.
Fills in any missing audio configuration using the given sample rates.
Args:
input_sample_rate: Sample rate for audio input (Hz).
output_sample_rate: Sample rate for audio output (Hz).
"""
props = self._settings.session_properties
if not props.audio:
props.audio = events.AudioConfiguration()
if not props.audio.input:
props.audio.input = events.AudioInput(
format=events.PCMAudioFormat(rate=input_sample_rate)
)
if not props.audio.output:
props.audio.output = events.AudioOutput(
format=events.PCMAudioFormat(rate=output_sample_rate)
)
async def start(self, frame: StartFrame):
"""Start the service and establish WebSocket connection.
@@ -288,23 +324,7 @@ class GrokRealtimeLLMService(LLMService):
frame: The start frame triggering service initialization.
"""
await super().start(frame)
# Ensure audio configuration exists with both input and output
if not self._session_properties.audio:
self._session_properties.audio = events.AudioConfiguration()
# Fill in missing input configuration
if not self._session_properties.audio.input:
self._session_properties.audio.input = events.AudioInput(
format=events.PCMAudioFormat(rate=frame.audio_in_sample_rate)
)
# Fill in missing output configuration
if not self._session_properties.audio.output:
self._session_properties.audio.output = events.AudioOutput(
format=events.PCMAudioFormat(rate=frame.audio_out_sample_rate)
)
self._ensure_audio_config(frame.audio_in_sample_rate, frame.audio_out_sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -336,6 +356,16 @@ class GrokRealtimeLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.update path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -355,11 +385,8 @@ class GrokRealtimeLLMService(LLMService):
await self._handle_bot_stopped_speaking()
elif isinstance(frame, LLMMessagesAppendFrame):
await self._handle_messages_append(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
await self.push_frame(frame, direction)
@@ -436,9 +463,30 @@ class GrokRealtimeLLMService(LLMService):
return
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
async def _update_settings(self, update):
"""Apply a settings update, sending a session update if needed."""
# Capture current sample rates before the update replaces them.
input_rate = self._get_configured_sample_rate("input")
output_rate = self._get_configured_sample_rate("output")
changed = await super()._update_settings(update)
if "session_properties" in changed:
if input_rate and output_rate:
self._ensure_audio_config(input_rate, output_rate)
else:
logger.warning(
"Attempting to apply session properties update without configured sample rates. "
"Audio configuration may be incomplete."
)
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"})
return changed
async def _send_session_update(self):
"""Update session settings on the server."""
settings = self._session_properties
settings = self._settings.session_properties
adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
@@ -516,7 +564,7 @@ class GrokRealtimeLLMService(LLMService):
async def _handle_evt_conversation_created(self, evt):
"""Handle conversation.created event - first event after connecting."""
await self._update_settings()
await self._send_session_update()
async def _handle_evt_response_created(self, evt):
"""Handle response.created event - response generation started."""
@@ -719,7 +767,7 @@ class GrokRealtimeLLMService(LLMService):
self._messages_added_manually[evt.item.id] = True
await self.send_client_event(evt)
await self._update_settings()
await self._send_session_update()
self._llm_needs_conversation_setup = False
logger.debug("Creating Grok response")

2
src/pipecat/services/groq/stt.py
View File

@@ -62,7 +62,7 @@ class GroqSTTService(BaseWhisperSTTService):
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"model": self._settings.model,
# Use verbose_json to get probability metrics
"response_format": "verbose_json" if self._include_prob_metrics else "json",
"language": self._language,

46
src/pipecat/services/groq/tts.py
View File

@@ -8,7 +8,8 @@
import io
import wave
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, ClassVar, Dict, Optional
from loguru import logger
from pydantic import BaseModel
@@ -20,6 +21,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -32,6 +34,23 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class GroqTTSSettings(TTSSettings):
"""Settings for the Groq TTS service.
Parameters:
output_format: Audio output format.
speed: Speech speed multiplier. Defaults to 1.0.
groq_sample_rate: Audio sample rate.
"""
output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
groq_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "groq_sample_rate"}
class GroqTTSService(TTSService):
"""Groq text-to-speech service implementation.
@@ -40,6 +59,8 @@ class GroqTTSService(TTSService):
and output formats.
"""
_settings: GroqTTSSettings
class InputParams(BaseModel):
"""Input parameters for Groq TTS configuration.
@@ -87,19 +108,18 @@ class GroqTTSService(TTSService):
params = params or GroqTTSService.InputParams()
self._api_key = api_key
self._model_name = model_name
self._output_format = output_format
self._voice_id = voice_id
self._params = params
self._settings = {
"model": model_name,
"voice_id": voice_id,
"output_format": output_format,
"language": str(params.language) if params.language else "en",
"speed": params.speed,
"sample_rate": sample_rate,
}
self._settings = GroqTTSSettings(
model=model_name,
voice=voice_id,
language=str(params.language) if params.language else "en",
output_format=output_format,
speed=params.speed,
groq_sample_rate=sample_rate,
)
self._sync_model_name_to_metrics()
self._client = AsyncGroq(api_key=self._api_key)
@@ -129,8 +149,8 @@ class GroqTTSService(TTSService):
try:
response = await self._client.audio.speech.create(
model=self._model_name,
voice=self._voice_id,
model=self._settings.model,
voice=self._settings.voice,
response_format=self._output_format,
input=text,
)

40
src/pipecat/services/hathora/stt.py
View File

@@ -8,6 +8,7 @@
import base64
import os
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional
import aiohttp
@@ -18,6 +19,7 @@ from pipecat.frames.frames import (
Frame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import HATHORA_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
@@ -27,12 +29,27 @@ from pipecat.utils.tracing.service_decorators import traced_stt
from .utils import ConfigOption
@dataclass
class HathoraSTTSettings(STTSettings):
"""Settings for the Hathora STT service.
Parameters:
config: Some models support additional config, refer to
`docs <https://models.hathora.dev>`_ for each model to see
what is supported.
"""
config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HathoraSTTService(SegmentedSTTService):
"""This service supports several different speech-to-text models hosted by Hathora.
[Documentation](https://models.hathora.dev)
"""
_settings: HathoraSTTSettings
class InputParams(BaseModel):
"""Optional input parameters for Hathora STT configuration.
@@ -83,12 +100,12 @@ class HathoraSTTService(SegmentedSTTService):
params = params or HathoraSTTService.InputParams()
self._settings = {
"language": params.language,
"config": params.config,
}
self.set_model_name(model)
self._settings = HathoraSTTSettings(
model=model,
language=params.language,
config=params.config,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -123,12 +140,11 @@ class HathoraSTTService(SegmentedSTTService):
"model": self._model,
}
if self._settings["language"] is not None:
payload["language"] = self._settings["language"]
if self._settings["config"] is not None:
if self._settings.language is not None:
payload["language"] = self._settings.language
if self._settings.config is not None:
payload["model_config"] = [
{"name": option.name, "value": option.value}
for option in self._settings["config"]
{"name": option.name, "value": option.value} for option in self._settings.config
]
base64_audio = base64.b64encode(audio).decode("utf-8")
@@ -147,7 +163,7 @@ class HathoraSTTService(SegmentedSTTService):
if text: # Only yield non-empty text
# Hathora's API currently doesn't return language info
# so we default to the requested language or "en"
response_language = self._settings["language"] or "en"
response_language = self._settings.language or "en"
await self._handle_transcription(text, True, response_language)
yield TranscriptionFrame(
text,

46
src/pipecat/services/hathora/tts.py
View File

@@ -9,6 +9,7 @@
import io
import os
import wave
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional, Tuple
import aiohttp
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -45,12 +47,29 @@ def _decode_audio_payload(
return audio_bytes, fallback_sample_rate, fallback_channels
@dataclass
class HathoraTTSSettings(TTSSettings):
"""Settings for Hathora TTS service.
Parameters:
speed: Speech speed multiplier (if supported by model).
config: Some models support additional config, refer to
[docs](https://models.hathora.dev) for each model to see
what is supported.
"""
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HathoraTTSService(TTSService):
"""This service supports several different text-to-speech models hosted by Hathora.
[Documentation](https://models.hathora.dev)
"""
_settings: HathoraTTSSettings
class InputParams(BaseModel):
"""Optional input parameters for Hathora TTS configuration.
@@ -98,13 +117,13 @@ class HathoraTTSService(TTSService):
params = params or HathoraTTSService.InputParams()
self._settings = {
"speed": params.speed,
"config": params.config,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._settings = HathoraTTSSettings(
model=model,
voice=voice_id,
speed=params.speed,
config=params.config,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -133,14 +152,13 @@ class HathoraTTSService(TTSService):
payload = {"model": self._model, "text": text}
if self._voice_id is not None:
payload["voice"] = self._voice_id
if self._settings["speed"] is not None:
payload["speed"] = self._settings["speed"]
if self._settings["config"] is not None:
if self._settings.voice is not None:
payload["voice"] = self._settings.voice
if self._settings.speed is not None:
payload["speed"] = self._settings.speed
if self._settings.config is not None:
payload["model_config"] = [
{"name": option.name, "value": option.value}
for option in self._settings["config"]
{"name": option.name, "value": option.value} for option in self._settings.config
]
yield TTSStartedFrame(context_id=context_id)

86
src/pipecat/services/hume/tts.py
View File

@@ -6,6 +6,8 @@
import base64
import os
import warnings
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import httpx
@@ -24,6 +26,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import WordTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -46,6 +49,21 @@ DEFAULT_HEADERS = {
}
@dataclass
class HumeTTSSettings(TTSSettings):
"""Settings for Hume TTS service.
Parameters:
description: Natural-language acting directions (up to 100 characters).
speed: Speaking-rate multiplier (0.5-2.0).
trailing_silence: Seconds of silence to append at the end (0-5).
"""
description: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
trailing_silence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HumeTTSService(WordTTSService):
"""Hume Octave Text-to-Speech service.
@@ -61,6 +79,8 @@ class HumeTTSService(WordTTSService):
- Provides metrics for Time To First Byte (TTFB) and TTS usage.
"""
_settings: HumeTTSSettings
class InputParams(BaseModel):
"""Optional synthesis parameters for Hume TTS.
@@ -114,10 +134,14 @@ class HumeTTSService(WordTTSService):
self._http_client = httpx.AsyncClient(headers=DEFAULT_HEADERS)
self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client)
self._params = params or HumeTTSService.InputParams()
# Store voice in the base class (mirrors other services)
self.set_voice(voice_id)
params = params or HumeTTSService.InputParams()
self._settings = HumeTTSSettings(
voice=voice_id,
description=params.description,
speed=params.speed,
trailing_silence=params.trailing_silence,
)
self._audio_bytes = b""
@@ -183,7 +207,10 @@ class HumeTTSService(WordTTSService):
await self.add_word_timestamps([("Reset", 0)])
async def update_setting(self, key: str, value: Any) -> None:
"""Runtime updates via `TTSUpdateSettingsFrame`.
"""Runtime updates via key/value pair.
.. deprecated:: 0.0.103
Use ``TTSUpdateSettingsFrame(update=HumeTTSSettings(...))`` instead.
Args:
key: The name of the setting to update. Recognized keys are:
@@ -193,20 +220,29 @@ class HumeTTSService(WordTTSService):
- "trailing_silence"
value: The new value for the setting.
"""
key_l = (key or "").lower()
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'update_setting' is deprecated, use "
"'TTSUpdateSettingsFrame(update=HumeTTSSettings(...))' instead.",
DeprecationWarning,
stacklevel=2,
)
if key_l == "voice_id":
self.set_voice(str(value))
logger.debug(f"HumeTTSService voice_id set to: {self.voice}")
elif key_l == "description":
self._params.description = None if value is None else str(value)
elif key_l == "speed":
self._params.speed = None if value is None else float(value)
elif key_l == "trailing_silence":
self._params.trailing_silence = None if value is None else float(value)
else:
# Defer unknown keys to the base class
await super().update_setting(key, value)
key_l = (key or "").lower()
known_keys = {"voice_id", "voice", "description", "speed", "trailing_silence"}
if key_l in known_keys:
kwargs: dict[str, Any] = {}
if key_l in ("voice_id", "voice"):
kwargs["voice"] = str(value)
elif key_l == "description":
kwargs["description"] = None if value is None else str(value)
elif key_l == "speed":
kwargs["speed"] = None if value is None else float(value)
elif key_l == "trailing_silence":
kwargs["trailing_silence"] = None if value is None else float(value)
await self._update_settings(HumeTTSSettings(**kwargs))
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -226,14 +262,14 @@ class HumeTTSService(WordTTSService):
# Build the request payload
utterance_kwargs: dict[str, Any] = {
"text": text,
"voice": PostedUtteranceVoiceWithId(id=self._voice_id),
"voice": PostedUtteranceVoiceWithId(id=self._settings.voice),
}
if self._params.description is not None:
utterance_kwargs["description"] = self._params.description
if self._params.speed is not None:
utterance_kwargs["speed"] = self._params.speed
if self._params.trailing_silence is not None:
utterance_kwargs["trailing_silence"] = self._params.trailing_silence
if self._settings.description is not None:
utterance_kwargs["description"] = self._settings.description
if self._settings.speed is not None:
utterance_kwargs["speed"] = self._settings.speed
if self._settings.trailing_silence is not None:
utterance_kwargs["trailing_silence"] = self._settings.trailing_silence
utterance = PostedUtterance(**utterance_kwargs)
@@ -257,7 +293,7 @@ class HumeTTSService(WordTTSService):
# Use version "2" by default if no description is provided
# Version "1" is needed when description is used
version = "1" if self._params.description is not None else "2"
version = "1" if self._settings.description is not None else "2"
# Track the duration of this utterance based on the last timestamp
utterance_duration = 0.0

181
src/pipecat/services/inworld/tts.py
View File

@@ -17,7 +17,8 @@ import asyncio
import base64
import json
import uuid
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Tuple
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, List, Literal, Mapping, Optional, Tuple
import aiohttp
import websockets
@@ -28,6 +29,8 @@ from pipecat import version as pipecat_version
USER_AGENT = f"pipecat/{pipecat_version()}"
from pydantic import BaseModel
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
try:
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
@@ -52,6 +55,53 @@ from pipecat.services.tts_service import AudioContextWordTTSService, WordTTSServ
from pipecat.utils.tracing.service_decorators import traced_tts
@dataclass
class InworldTTSSettings(TTSSettings):
"""Settings for Inworld TTS services.
Parameters:
audio_encoding: Audio encoding format (e.g. LINEAR16).
audio_sample_rate: Audio sample rate in Hz.
speaking_rate: Speaking rate for speech synthesis.
temperature: Temperature for speech synthesis.
auto_mode: Whether to use auto mode. Recommended when texts are sent
in full sentences/phrases. When enabled, the server controls
flushing of buffered text to achieve minimal latency while
maintaining high quality audio output. If None (default),
automatically set based on aggregate_sentences.
apply_text_normalization: Whether to apply text normalization.
timestamp_transport_strategy: Strategy for timestamp transport ("ASYNC" or "SYNC").
"""
audio_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
auto_mode: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
timestamp_transport_strategy: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {
"voice_id": "voice",
"voiceId": "voice",
"modelId": "model",
"applyTextNormalization": "apply_text_normalization",
"autoMode": "auto_mode",
"timestampTransportStrategy": "timestamp_transport_strategy",
}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "InworldTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``audioConfig``."""
flat = dict(settings)
nested = flat.pop("audioConfig", None)
if isinstance(nested, dict):
flat.setdefault("audio_encoding", nested.get("audioEncoding"))
flat.setdefault("audio_sample_rate", nested.get("sampleRateHertz"))
flat.setdefault("speaking_rate", nested.get("speakingRate"))
return super().from_mapping(flat)
class InworldHttpTTSService(WordTTSService):
"""Inworld AI HTTP-based TTS service.
@@ -59,6 +109,8 @@ class InworldHttpTTSService(WordTTSService):
Outputs LINEAR16 audio at configurable sample rates with word-level timestamps.
"""
_settings: InworldTTSSettings
class InputParams(BaseModel):
"""Input parameters for Inworld TTS configuration.
@@ -117,26 +169,23 @@ class InworldHttpTTSService(WordTTSService):
else:
self._base_url = "https://api.inworld.ai/tts/v1/voice"
self._settings = {
"voiceId": voice_id,
"modelId": model,
"audioConfig": {
"audioEncoding": encoding,
"sampleRateHertz": 0,
},
}
self._settings = InworldTTSSettings(
model=model,
voice=voice_id,
audio_encoding=encoding,
audio_sample_rate=0,
)
if params.temperature is not None:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
if params.speaking_rate is not None:
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
self._settings.speaking_rate = params.speaking_rate
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
self._settings.timestamp_transport_strategy = params.timestamp_transport_strategy
self._cumulative_time = 0.0
self.set_voice(voice_id)
self.set_model_name(model)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -153,7 +202,7 @@ class InworldHttpTTSService(WordTTSService):
frame: The start frame.
"""
await super().start(frame)
self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
async def stop(self, frame: EndFrame):
"""Stop the Inworld TTS service.
@@ -232,20 +281,27 @@ class InworldHttpTTSService(WordTTSService):
"""
logger.debug(f"{self}: Generating TTS [{text}] (streaming={self._streaming})")
audio_config = {
"audioEncoding": self._settings.audio_encoding,
"sampleRateHertz": self._settings.audio_sample_rate,
}
if is_given(self._settings.speaking_rate):
audio_config["speakingRate"] = self._settings.speaking_rate
payload = {
"text": text,
"voiceId": self._settings["voiceId"],
"modelId": self._settings["modelId"],
"audioConfig": self._settings["audioConfig"],
"voiceId": self._settings.voice,
"modelId": self._settings.model,
"audioConfig": audio_config,
}
if "temperature" in self._settings:
payload["temperature"] = self._settings["temperature"]
if is_given(self._settings.temperature):
payload["temperature"] = self._settings.temperature
# Use WORD timestamps for simplicity and correct spacing/capitalization
payload["timestampType"] = self._timestamp_type
if "timestampTransportStrategy" in self._settings:
payload["timestampTransportStrategy"] = self._settings["timestampTransportStrategy"]
if is_given(self._settings.timestamp_transport_strategy):
payload["timestampTransportStrategy"] = self._settings.timestamp_transport_strategy
request_id = str(uuid.uuid4())
headers = {
@@ -419,6 +475,8 @@ class InworldTTSService(AudioContextWordTTSService):
with word-level timestamps.
"""
_settings: InworldTTSSettings
class InputParams(BaseModel):
"""Input parameters for Inworld WebSocket TTS configuration.
@@ -486,29 +544,27 @@ class InworldTTSService(AudioContextWordTTSService):
self._api_key = api_key
self._url = url
self._settings: Dict[str, Any] = {
"voiceId": voice_id,
"modelId": model,
"audioConfig": {
"audioEncoding": encoding,
"sampleRateHertz": 0,
},
}
self._settings = InworldTTSSettings(
model=model,
voice=voice_id,
audio_encoding=encoding,
audio_sample_rate=0,
)
self._timestamp_type = "WORD"
if params.temperature is not None:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
if params.speaking_rate is not None:
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
self._settings.speaking_rate = params.speaking_rate
if params.apply_text_normalization is not None:
self._settings["applyTextNormalization"] = params.apply_text_normalization
self._settings.apply_text_normalization = params.apply_text_normalization
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
self._settings.timestamp_transport_strategy = params.timestamp_transport_strategy
if params.auto_mode is not None:
self._settings["autoMode"] = params.auto_mode
self._settings.auto_mode = params.auto_mode
else:
self._settings["autoMode"] = aggregate_sentences
self._settings.auto_mode = aggregate_sentences
self._buffer_settings = {
"maxBufferDelayMs": params.max_buffer_delay_ms,
@@ -526,8 +582,7 @@ class InworldTTSService(AudioContextWordTTSService):
# Track the end time of the last word in the current generation
self._generation_end_time = 0.0
self.set_voice(voice_id)
self.set_model_name(model)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -544,7 +599,7 @@ class InworldTTSService(AudioContextWordTTSService):
frame: The start frame.
"""
await super().start(frame)
self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -700,6 +755,21 @@ class InworldTTSService(AudioContextWordTTSService):
await self._disconnect_websocket()
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to the Inworld WebSocket TTS service.
@@ -883,22 +953,29 @@ class InworldTTSService(AudioContextWordTTSService):
Args:
context_id: The context ID.
"""
audio_config = {
"audioEncoding": self._settings.audio_encoding,
"sampleRateHertz": self._settings.audio_sample_rate,
}
if is_given(self._settings.speaking_rate):
audio_config["speakingRate"] = self._settings.speaking_rate
create_config: Dict[str, Any] = {
"voiceId": self._settings["voiceId"],
"modelId": self._settings["modelId"],
"audioConfig": self._settings["audioConfig"],
"voiceId": self._settings.voice,
"modelId": self._settings.model,
"audioConfig": audio_config,
}
if "temperature" in self._settings:
create_config["temperature"] = self._settings["temperature"]
if "applyTextNormalization" in self._settings:
create_config["applyTextNormalization"] = self._settings["applyTextNormalization"]
if "autoMode" in self._settings:
create_config["autoMode"] = self._settings["autoMode"]
if "timestampTransportStrategy" in self._settings:
create_config["timestampTransportStrategy"] = self._settings[
"timestampTransportStrategy"
]
if is_given(self._settings.temperature):
create_config["temperature"] = self._settings.temperature
if is_given(self._settings.apply_text_normalization):
create_config["applyTextNormalization"] = self._settings.apply_text_normalization
if is_given(self._settings.auto_mode):
create_config["autoMode"] = self._settings.auto_mode
if is_given(self._settings.timestamp_transport_strategy):
create_config["timestampTransportStrategy"] = (
self._settings.timestamp_transport_strategy
)
# Set buffer settings for timely audio generation.
# Use provided values or defaults that work well for streaming LLM output.

24
src/pipecat/services/kokoro/tts.py
View File

@@ -7,6 +7,7 @@
"""Kokoro TTS service implementation using kokoro-onnx."""
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import AsyncGenerator, Optional
@@ -22,6 +23,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -87,6 +89,17 @@ def language_to_kokoro_language(language: Language) -> str:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class KokoroTTSSettings(TTSSettings):
"""Settings for the Kokoro TTS service.
Parameters:
lang_code: Kokoro language code for synthesis.
"""
lang_code: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class KokoroTTSService(TTSService):
"""Kokoro TTS service implementation.
@@ -94,6 +107,8 @@ class KokoroTTSService(TTSService):
Automatically downloads model files on first use.
"""
_settings: KokoroTTSSettings
class InputParams(BaseModel):
"""Input parameters for Kokoro TTS configuration.
@@ -126,9 +141,14 @@ class KokoroTTSService(TTSService):
params = params or KokoroTTSService.InputParams()
self._voice_id = voice_id
self._lang_code = language_to_kokoro_language(params.language)
self._settings = KokoroTTSSettings(
voice=voice_id,
language=language_to_kokoro_language(params.language),
lang_code=language_to_kokoro_language(params.language),
)
model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx"
voices = Path(voices_path) if voices_path else KOKORO_CACHE_DIR / "voices-v1.0.bin"
@@ -161,7 +181,7 @@ class KokoroTTSService(TTSService):
yield TTSStartedFrame(context_id=context_id)
stream = self._kokoro.create_stream(
text, voice=self._voice_id, lang=self._lang_code, speed=1.0
text, voice=self._settings.voice, lang=self._lang_code, speed=1.0
)
async for samples, sample_rate in stream:

56
src/pipecat/services/llm_service.py
View File

@@ -44,6 +44,7 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
UserImageRequestFrame,
)
@@ -58,6 +59,7 @@ from pipecat.processors.aggregators.llm_response import (
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import LLMSettings, is_given
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
from pipecat.utils.context.llm_context_summarization import (
LLMContextSummarizationUtil,
@@ -172,6 +174,8 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
logger.info(f"Starting {len(function_calls)} function calls")
"""
_settings: LLMSettings
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
# However, subclasses should override this with a more specific adapter when necessary.
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
@@ -200,6 +204,7 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
self._sequential_runner_task: Optional[asyncio.Task] = None
self._skip_tts: Optional[bool] = None
self._summary_task: Optional[asyncio.Task] = None
self._settings = LLMSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._register_event_handler("on_function_calls_started")
self._register_event_handler("on_completion_timeout")
@@ -307,34 +312,28 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
await self._cancel_sequential_runner_task()
await self._cancel_summary_task()
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update LLM service settings.
Handles turn completion settings specially since they are not model
parameters and should not be passed to the underlying LLM API.
async def _update_settings(self, update: LLMSettings) -> dict[str, Any]:
"""Apply a settings update, handling turn-completion fields.
Args:
settings: Dictionary of settings to update.
"""
# Turn completion settings to extract (not model parameters)
turn_completion_keys = {"filter_incomplete_user_turns", "user_turn_completion_config"}
update: An LLM settings delta.
# Handle turn completion settings
if "filter_incomplete_user_turns" in settings:
self._filter_incomplete_user_turns = settings["filter_incomplete_user_turns"]
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if "filter_incomplete_user_turns" in changed:
self._filter_incomplete_user_turns = self._settings.filter_incomplete_user_turns
logger.info(
f"{self}: Incomplete turn filtering {'enabled' if self._filter_incomplete_user_turns else 'disabled'}"
f"{self}: Incomplete turn filtering "
f"{'enabled' if self._filter_incomplete_user_turns else 'disabled'}"
)
# Configure the mixin with config object
if self._filter_incomplete_user_turns and "user_turn_completion_config" in settings:
self.set_user_turn_completion_config(settings["user_turn_completion_config"])
if "user_turn_completion_config" in changed and self._filter_incomplete_user_turns:
self.set_user_turn_completion_config(self._settings.user_turn_completion_config)
# Remove turn completion settings before passing to parent
settings = {k: v for k, v in settings.items() if k not in turn_completion_keys}
# Let the parent handle remaining model parameters
await super()._update_settings(settings)
return changed
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame.
@@ -349,6 +348,21 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
await self._handle_interruptions(frame)
elif isinstance(frame, LLMConfigureOutputFrame):
self._skip_tts = frame.skip_tts
elif isinstance(frame, LLMUpdateSettingsFrame):
if frame.update is not None:
await self._update_settings(frame.update)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via LLMUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.103, use LLMUpdateSettingsFrame(update=LLMSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
update = type(self._settings).from_mapping(frame.settings)
await self._update_settings(update)
elif isinstance(frame, LLMContextSummaryRequestFrame):
await self._handle_summary_request(frame)

55
src/pipecat/services/lmnt/tts.py
View File

@@ -7,7 +7,8 @@
"""LMNT text-to-speech service implementation."""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -71,6 +73,17 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class LmntTTSSettings(TTSSettings):
"""Settings for LMNT TTS service.
Parameters:
format: Audio output format. Defaults to "raw".
"""
format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class LmntTTSService(InterruptibleTTSService):
"""LMNT real-time text-to-speech service.
@@ -79,6 +92,8 @@ class LmntTTSService(InterruptibleTTSService):
language settings.
"""
_settings: LmntTTSSettings
def __init__(
self,
*,
@@ -107,12 +122,13 @@ class LmntTTSService(InterruptibleTTSService):
)
self._api_key = api_key
self.set_voice(voice_id)
self.set_model_name(model)
self._settings = {
"language": self.language_to_service_language(language),
"format": "raw", # Use raw format for direct PCM data
}
self._settings = LmntTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(language),
format="raw",
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._context_id: Optional[str] = None
@@ -190,6 +206,23 @@ class LmntTTSService(InterruptibleTTSService):
await self._disconnect_websocket()
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update.
Args:
update: A :class:`TTSSettings` (or ``LmntTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if changed:
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to LMNT websocket."""
try:
@@ -201,11 +234,11 @@ class LmntTTSService(InterruptibleTTSService):
# Build initial connection message
init_msg = {
"X-API-Key": self._api_key,
"voice": self._voice_id,
"format": self._settings["format"],
"voice": self._settings.voice,
"format": self._settings.format,
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"model": self.model_name,
"language": self._settings.language,
"model": self._settings.model,
}
# Connect to LMNT's websocket directly

164
src/pipecat/services/minimax/tts.py
View File

@@ -11,7 +11,8 @@ for streaming text-to-speech synthesis.
"""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Mapping, Optional
import aiohttp
from loguru import logger
@@ -25,6 +26,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -85,6 +87,69 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class MiniMaxTTSSettings(TTSSettings):
"""Settings for MiniMax TTS service.
Parameters:
stream: Whether to use streaming mode.
speed: Speech speed (range: 0.5 to 2.0).
volume: Speech volume (range: 0 to 10).
pitch: Pitch adjustment (range: -12 to 12).
emotion: Emotional tone (options: "happy", "sad", "angry", "fearful",
"disgusted", "surprised", "calm", "fluent").
text_normalization: Enable text normalization (Chinese/English).
latex_read: Enable LaTeX formula reading.
audio_bitrate: Audio bitrate in bps.
audio_format: Audio output format.
audio_channel: Number of audio channels.
audio_sample_rate: Audio sample rate in Hz.
language_boost: Language boost string for multilingual support.
"""
stream: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emotion: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
text_normalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
latex_read: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_bitrate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_channel: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_boost: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "MiniMaxTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested dicts.
Handles ``voice_setting`` (with ``vol`` → ``volume`` rename) and
``audio_setting`` (with prefixed field mapping).
"""
flat = dict(settings)
voice = flat.pop("voice_setting", None)
if isinstance(voice, dict):
flat.setdefault("speed", voice.get("speed"))
flat.setdefault("volume", voice.get("vol"))
flat.setdefault("pitch", voice.get("pitch"))
flat.setdefault("emotion", voice.get("emotion"))
flat.setdefault("text_normalization", voice.get("text_normalization"))
flat.setdefault("latex_read", voice.get("latex_read"))
audio = flat.pop("audio_setting", None)
if isinstance(audio, dict):
flat.setdefault("audio_bitrate", audio.get("bitrate"))
flat.setdefault("audio_format", audio.get("format"))
flat.setdefault("audio_channel", audio.get("channel"))
flat.setdefault("audio_sample_rate", audio.get("sample_rate"))
return super().from_mapping(flat)
class MiniMaxHttpTTSService(TTSService):
"""Text-to-speech service using MiniMax's T2A (Text-to-Audio) API.
@@ -96,6 +161,8 @@ class MiniMaxHttpTTSService(TTSService):
https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643
"""
_settings: MiniMaxTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for MiniMax TTS.
@@ -168,33 +235,26 @@ class MiniMaxHttpTTSService(TTSService):
self._group_id = group_id
self._base_url = f"{base_url}?GroupId={group_id}"
self._session = aiohttp_session
self._model_name = model
self._voice_id = voice_id
# Create voice settings
self._settings = {
"stream": True,
"voice_setting": {
"speed": params.speed,
"vol": params.volume,
"pitch": params.pitch,
},
"audio_setting": {
"bitrate": 128000,
"format": "pcm",
"channel": 1,
},
}
# Set voice and model
self.set_voice(voice_id)
self.set_model_name(model)
self._settings = MiniMaxTTSSettings(
model=model,
voice=voice_id,
stream=True,
speed=params.speed,
volume=params.volume,
pitch=params.pitch,
audio_bitrate=128000,
audio_format="pcm",
audio_channel=1,
)
self._sync_model_name_to_metrics()
# Add language boost if provided
if params.language:
service_lang = self.language_to_service_language(params.language)
if service_lang:
self._settings["language_boost"] = service_lang
self._settings.language_boost = service_lang
# Add optional emotion if provided
if params.emotion:
@@ -210,7 +270,7 @@ class MiniMaxHttpTTSService(TTSService):
"fluent",
]
if params.emotion in supported_emotions:
self._settings["voice_setting"]["emotion"] = params.emotion
self._settings.emotion = params.emotion
else:
logger.warning(
f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}"
@@ -226,15 +286,15 @@ class MiniMaxHttpTTSService(TTSService):
"Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.",
DeprecationWarning,
)
self._settings["voice_setting"]["text_normalization"] = params.english_normalization
self._settings.text_normalization = params.english_normalization
# Add text_normalization if provided (corrected parameter name)
if params.text_normalization is not None:
self._settings["voice_setting"]["text_normalization"] = params.text_normalization
self._settings.text_normalization = params.text_normalization
# Add latex_read if provided
if params.latex_read is not None:
self._settings["voice_setting"]["latex_read"] = params.latex_read
self._settings.latex_read = params.latex_read
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -255,24 +315,6 @@ class MiniMaxHttpTTSService(TTSService):
"""
return language_to_minimax_language(language)
def set_model_name(self, model: str):
"""Set the TTS model to use.
Args:
model: The model name to use for synthesis.
"""
self._model_name = model
def set_voice(self, voice: str):
"""Set the voice to use.
Args:
voice: The voice identifier to use for synthesis.
"""
self._voice_id = voice
if "voice_setting" in self._settings:
self._settings["voice_setting"]["voice_id"] = voice
async def start(self, frame: StartFrame):
"""Start the MiniMax TTS service.
@@ -280,7 +322,7 @@ class MiniMaxHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["audio_setting"]["sample_rate"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}")
@traced_tts
@@ -302,10 +344,38 @@ class MiniMaxHttpTTSService(TTSService):
"Authorization": f"Bearer {self._api_key}",
}
# Build voice_setting dict for API
voice_setting = {
"voice_id": self._settings.voice,
"speed": self._settings.speed,
"vol": self._settings.volume,
"pitch": self._settings.pitch,
}
if is_given(self._settings.emotion):
voice_setting["emotion"] = self._settings.emotion
if is_given(self._settings.text_normalization):
voice_setting["text_normalization"] = self._settings.text_normalization
if is_given(self._settings.latex_read):
voice_setting["latex_read"] = self._settings.latex_read
# Build audio_setting dict for API
audio_setting = {
"bitrate": self._settings.audio_bitrate,
"format": self._settings.audio_format,
"channel": self._settings.audio_channel,
"sample_rate": self._settings.audio_sample_rate,
}
# Create payload from settings
payload = self._settings.copy()
payload["model"] = self._model_name
payload["text"] = text
payload = {
"stream": self._settings.stream,
"voice_setting": voice_setting,
"audio_setting": audio_setting,
"model": self._settings.model,
"text": text,
}
if is_given(self._settings.language_boost):
payload["language_boost"] = self._settings.language_boost
try:
await self.start_ttfb_metrics()

18
src/pipecat/services/mistral/llm.py
View File

@@ -180,24 +180,24 @@ class MistralLLMService(OpenAILLMService):
fixed_messages = self._apply_mistral_fixups(params_from_context["messages"])
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"messages": fixed_messages,
"tools": params_from_context["tools"],
"tool_choice": params_from_context["tool_choice"],
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Handle Mistral-specific parameter mapping
# Mistral uses "random_seed" instead of "seed"
if self._settings["seed"]:
params["random_seed"] = self._settings["seed"]
if self._settings.seed:
params["random_seed"] = self._settings.seed
# Add any extra parameters
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

3
src/pipecat/services/moondream/vision.py
View File

@@ -81,7 +81,8 @@ class MoondreamService(VisionService):
"""
super().__init__(**kwargs)
self.set_model_name(model)
self._settings.model = model
self._sync_model_name_to_metrics()
if not use_cpu:
device, dtype = detect_device()

86
src/pipecat/services/neuphonic/tts.py
View File

@@ -13,7 +13,8 @@ text-to-speech API for real-time audio synthesis.
import asyncio
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -34,6 +35,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -72,6 +74,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class NeuphonicTTSSettings(TTSSettings):
"""Settings for Neuphonic TTS service.
Parameters:
speed: Speech speed multiplier. Defaults to 1.0.
encoding: Audio encoding format.
sampling_rate: Audio sample rate.
"""
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sampling_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NeuphonicTTSService(InterruptibleTTSService):
"""Neuphonic real-time text-to-speech service using WebSocket streaming.
@@ -80,6 +97,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
parameters for high-quality speech generation.
"""
_settings: NeuphonicTTSSettings
class InputParams(BaseModel):
"""Input parameters for Neuphonic TTS configuration.
@@ -127,13 +146,13 @@ class NeuphonicTTSService(InterruptibleTTSService):
self._api_key = api_key
self._url = url
self._settings = {
"lang_code": self.language_to_service_language(params.language),
"speed": params.speed,
"encoding": encoding,
"sampling_rate": sample_rate,
}
self.set_voice(voice_id)
self._settings = NeuphonicTTSSettings(
language=self.language_to_service_language(params.language),
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
voice=voice_id,
)
self._cumulative_time = 0
@@ -160,15 +179,14 @@ class NeuphonicTTSService(InterruptibleTTSService):
"""
return language_to_neuphonic_lang_code(language)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect with new configuration."""
if "voice_id" in settings:
self.set_voice(settings["voice_id"])
await super()._update_settings(settings)
await self._disconnect()
await self._connect()
logger.info(f"Switching TTS to settings: [{self._settings}]")
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and reconnect with new configuration."""
changed = await super()._update_settings(update)
if changed:
await self._disconnect()
await self._connect()
logger.info(f"Switching TTS to settings: [{self._settings}]")
return changed
async def start(self, frame: StartFrame):
"""Start the Neuphonic TTS service.
@@ -266,8 +284,11 @@ class NeuphonicTTSService(InterruptibleTTSService):
logger.debug("Connecting to Neuphonic")
tts_config = {
**self._settings,
"voice_id": self._voice_id,
"lang_code": self._settings.language,
"speed": self._settings.speed,
"encoding": self._settings.encoding,
"sampling_rate": self._settings.sampling_rate,
"voice_id": self._settings.voice,
}
query_params = []
@@ -275,7 +296,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
if value is not None:
query_params.append(f"{key}={value}")
url = f"{self._url}/speak/{self._settings['lang_code']}"
url = f"{self._url}/speak/{self._settings.language}"
if query_params:
url += f"?{'&'.join(query_params)}"
@@ -384,6 +405,8 @@ class NeuphonicHttpTTSService(TTSService):
HTTP-based communication over WebSocket connections.
"""
_settings: NeuphonicTTSSettings
class InputParams(BaseModel):
"""Input parameters for Neuphonic HTTP TTS configuration.
@@ -426,10 +449,13 @@ class NeuphonicHttpTTSService(TTSService):
self._api_key = api_key
self._session = aiohttp_session
self._base_url = url.rstrip("/")
self._lang_code = self.language_to_service_language(params.language) or "en"
self._speed = params.speed
self._encoding = encoding
self.set_voice(voice_id)
self._settings = NeuphonicTTSSettings(
voice=voice_id,
language=self.language_to_service_language(params.language) or "en",
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -513,7 +539,7 @@ class NeuphonicHttpTTSService(TTSService):
"""
logger.debug(f"Generating TTS: [{text}]")
url = f"{self._base_url}/sse/speak/{self._lang_code}"
url = f"{self._base_url}/sse/speak/{self._settings.language}"
headers = {
"X-API-KEY": self._api_key,
@@ -522,14 +548,14 @@ class NeuphonicHttpTTSService(TTSService):
payload = {
"text": text,
"lang_code": self._lang_code,
"encoding": self._encoding,
"lang_code": self._settings.language,
"encoding": self._settings.encoding,
"sampling_rate": self.sample_rate,
"speed": self._speed,
"speed": self._settings.speed,
}
if self._voice_id:
payload["voice_id"] = self._voice_id
if self._settings.voice:
payload["voice_id"] = self._settings.voice
try:
await self.start_ttfb_metrics()

195
src/pipecat/services/nvidia/stt.py
View File

@@ -8,7 +8,8 @@
import asyncio
from concurrent.futures import CancelledError as FuturesCancelledError
from typing import AsyncGenerator, List, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, List, Mapping, Optional
from loguru import logger
from pydantic import BaseModel
@@ -22,6 +23,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import NVIDIA_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService, STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -89,6 +91,32 @@ def language_to_nvidia_riva_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class NvidiaSTTSettings(STTSettings):
"""Settings for the NVIDIA Riva streaming STT service."""
pass
@dataclass
class NvidiaSegmentedSTTSettings(STTSettings):
"""Settings for the NVIDIA Riva segmented STT service.
Parameters:
profanity_filter: Whether to filter profanity from results.
automatic_punctuation: Whether to add automatic punctuation.
verbatim_transcripts: Whether to return verbatim transcripts.
boosted_lm_words: List of words to boost in language model.
boosted_lm_score: Score boost for specified words.
"""
profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
verbatim_transcripts: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
boosted_lm_words: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
boosted_lm_score: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NvidiaSTTService(STTService):
"""Real-time speech-to-text service using NVIDIA Riva streaming ASR.
@@ -97,6 +125,8 @@ class NvidiaSTTService(STTService):
processing for low-latency applications.
"""
_settings: NvidiaSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for NVIDIA Riva STT service.
@@ -141,12 +171,6 @@ class NvidiaSTTService(STTService):
self._server = server
self._api_key = api_key
self._use_ssl = use_ssl
self._profanity_filter = False
self._automatic_punctuation = True
self._no_verbatim_transcripts = False
self._language_code = params.language
self._boosted_lm_words = None
self._boosted_lm_score = 4.0
self._start_history = -1
self._start_threshold = -1.0
self._stop_history = -1
@@ -156,16 +180,11 @@ class NvidiaSTTService(STTService):
self._custom_configuration = ""
self._function_id = model_function_map.get("function_id")
self._settings = {
"language": str(params.language),
"profanity_filter": self._profanity_filter,
"automatic_punctuation": self._automatic_punctuation,
"verbatim_transcripts": not self._no_verbatim_transcripts,
"boosted_lm_words": self._boosted_lm_words,
"boosted_lm_score": self._boosted_lm_score,
}
self.set_model_name(model_function_map.get("model_name"))
self._settings = NvidiaSTTSettings(
model=model_function_map.get("model_name"),
language=params.language,
)
self._sync_model_name_to_metrics()
self._asr_service = None
self._queue = None
@@ -186,22 +205,18 @@ class NvidiaSTTService(STTService):
config = riva.client.StreamingRecognitionConfig(
config=riva.client.RecognitionConfig(
encoding=riva.client.AudioEncoding.LINEAR_PCM,
language_code=self._language_code,
language_code=self._settings.language,
model="",
max_alternatives=1,
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=not self._no_verbatim_transcripts,
profanity_filter=False,
enable_automatic_punctuation=True,
verbatim_transcripts=True,
sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
interim_results=True,
)
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
)
riva.client.add_endpoint_parameters_to_config(
config,
self._start_history,
@@ -226,18 +241,31 @@ class NvidiaSTTService(STTService):
async def set_model(self, model: str):
"""Set the ASR model for transcription.
.. deprecated:: 0.0.103
Model cannot be changed after initialization for NVIDIA Riva streaming STT.
Set model and function id in the constructor instead, e.g.::
NvidiaSTTService(
api_key=...,
model_function_map={"function_id": "<UUID>", "model_name": "<model_name>"},
)
Args:
model: Model name to set.
Note:
Model cannot be changed after initialization. Use model_function_map
parameter in constructor instead.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated. Model cannot be changed after initialization"
" for NVIDIA Riva streaming STT. Set model and function id in the"
" constructor instead, e.g.:"
" NvidiaSTTService(api_key=..., model_function_map="
"{'function_id': '<UUID>', 'model_name': '<model_name>'})",
DeprecationWarning,
stacklevel=2,
)
async def start(self, frame: StartFrame):
"""Start the NVIDIA Riva STT service and initialize streaming configuration.
@@ -254,7 +282,7 @@ class NvidiaSTTService(STTService):
if not self._thread_task:
self._thread_task = self.create_task(self._thread_task_handler())
logger.debug(f"Initialized NvidiaSTTService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaSTTService with model: {self._settings.model}")
async def stop(self, frame: EndFrame):
"""Stop the NVIDIA Riva STT service and clean up resources.
@@ -318,14 +346,14 @@ class NvidiaSTTService(STTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language_code,
self._settings.language,
result=result,
)
)
await self._handle_transcription(
transcript=transcript,
is_final=result.is_final,
language=self._language_code,
language=self._settings.language,
)
else:
await self.push_frame(
@@ -333,7 +361,7 @@ class NvidiaSTTService(STTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language_code,
self._settings.language,
result=result,
)
)
@@ -386,6 +414,8 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
audio buffering and speech detection.
"""
_settings: NvidiaSegmentedSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for NVIDIA Riva segmented STT service.
@@ -437,26 +467,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
params = params or NvidiaSegmentedSTTService.InputParams()
# Set model name
self.set_model_name(model_function_map.get("model_name"))
# Initialize NVIDIA Riva settings
self._api_key = api_key
self._server = server
self._use_ssl = use_ssl
self._function_id = model_function_map.get("function_id")
self._model_name = model_function_map.get("model_name")
# Store the language as a Language enum and as a string
self._language_enum = params.language or Language.EN_US
self._language = self.language_to_service_language(self._language_enum) or "en-US"
# Configure transcription parameters
self._profanity_filter = params.profanity_filter
self._automatic_punctuation = params.automatic_punctuation
self._verbatim_transcripts = params.verbatim_transcripts
self._boosted_lm_words = params.boosted_lm_words
self._boosted_lm_score = params.boosted_lm_score
# Voice activity detection thresholds (use NVIDIA Riva defaults)
self._start_history = -1
@@ -467,10 +482,19 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
self._stop_threshold_eou = -1.0
self._custom_configuration = ""
# Create NVIDIA Riva client
self._config = None
self._asr_service = None
self._settings = {"language": self._language_enum}
self._settings = NvidiaSegmentedSTTSettings(
model=model_function_map.get("model_name"),
language=self.language_to_service_language(params.language or Language.EN_US)
or "en-US",
profanity_filter=params.profanity_filter,
automatic_punctuation=params.automatic_punctuation,
verbatim_transcripts=params.verbatim_transcripts,
boosted_lm_words=params.boosted_lm_words,
boosted_lm_score=params.boosted_lm_score,
)
self._sync_model_name_to_metrics()
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to NVIDIA Riva's language code.
@@ -498,21 +522,25 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
auth = riva.client.Auth(None, self._use_ssl, self._server, metadata)
self._asr_service = riva.client.ASRService(auth)
def _get_language_code(self) -> str:
"""Get the current NVIDIA Riva language code string."""
return self._settings.language or "en-US"
def _create_recognition_config(self):
"""Create the NVIDIA Riva ASR recognition configuration."""
# Create base configuration
config = riva.client.RecognitionConfig(
language_code=self._language, # Now using the string, not a tuple
language_code=self._get_language_code(),
max_alternatives=1,
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=self._verbatim_transcripts,
profanity_filter=self._settings.profanity_filter,
enable_automatic_punctuation=self._settings.automatic_punctuation,
verbatim_transcripts=self._settings.verbatim_transcripts,
)
# Add word boosting if specified
if self._boosted_lm_words:
if self._settings.boosted_lm_words:
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
config, self._settings.boosted_lm_words, self._settings.boosted_lm_score
)
# Add voice activity detection parameters
@@ -540,22 +568,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
"""
return True
async def set_model(self, model: str):
"""Set the ASR model for transcription.
Args:
model: Model name to set.
Note:
Model cannot be changed after initialization. Use model_function_map
parameter in constructor instead.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
async def start(self, frame: StartFrame):
"""Initialize the service when the pipeline starts.
@@ -565,22 +577,23 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
await super().start(frame)
self._initialize_client()
self._config = self._create_recognition_config()
logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self._settings.model}")
async def set_language(self, language: Language):
"""Set the language for the STT service.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update and sync internal state.
Args:
language: Target language for transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._language_enum = language
self._language = self.language_to_service_language(language) or "en-US"
self._settings["language"] = language
update: A :class:`STTSettings` (or ``NvidiaSegmentedSTTSettings``) delta.
# Update configuration with new language
if self._config:
self._config.language_code = self._language
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if changed:
self._config = self._create_recognition_config()
return changed
@traced_stt
async def _handle_transcription(
@@ -633,11 +646,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
text,
self._user_id,
time_now_iso8601(),
self._language_enum,
self._settings.language,
)
transcription_found = True
await self._handle_transcription(text, True, self._language_enum)
await self._handle_transcription(text, True, self._settings.language)
if not transcription_found:
logger.debug(f"{self}: No transcription results found in NVIDIA Riva response")

79
src/pipecat/services/nvidia/tts.py
View File

@@ -12,7 +12,8 @@ gRPC API for high-quality speech synthesis.
import asyncio
import os
from typing import AsyncGenerator, AsyncIterator, Generator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, AsyncIterator, Generator, Mapping, Optional
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -30,6 +31,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
@@ -42,6 +44,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class NvidiaTTSSettings(TTSSettings):
"""Settings for NVIDIA Riva TTS service.
Parameters:
quality: Audio quality setting (0-100).
"""
quality: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NvidiaTTSService(TTSService):
"""NVIDIA Riva text-to-speech service.
@@ -50,6 +63,8 @@ class NvidiaTTSService(TTSService):
configurable quality settings.
"""
_settings: NvidiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Riva TTS configuration.
@@ -94,30 +109,58 @@ class NvidiaTTSService(TTSService):
self._server = server
self._api_key = api_key
self._voice_id = voice_id
self._language_code = params.language
self._quality = params.quality
self._function_id = model_function_map.get("function_id")
self._use_ssl = use_ssl
self.set_model_name(model_function_map.get("model_name"))
self.set_voice(voice_id)
self._settings = NvidiaTTSSettings(
model=model_function_map.get("model_name"),
voice=voice_id,
language=params.language,
quality=params.quality,
)
self._sync_model_name_to_metrics()
self._service = None
self._config = None
async def set_model(self, model: str):
"""Attempt to set the TTS model.
"""Set the TTS model.
Note: Model cannot be changed after initialization for Riva service.
.. deprecated:: 0.0.103
Model cannot be changed after initialization for NVIDIA Riva TTS.
Set model and function id in the constructor instead, e.g.::
NvidiaTTSService(
api_key=...,
model_function_map={"function_id": "<UUID>", "model_name": "<model_name>"},
)
Args:
model: The model name to set (operation not supported).
model: The model name to set.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated. Model cannot be changed after initialization"
" for NVIDIA Riva TTS. Set model and function id in the constructor"
" instead, e.g.: NvidiaTTSService(api_key=..., model_function_map="
"{'function_id': '<UUID>', 'model_name': '<model_name>'})",
DeprecationWarning,
stacklevel=2,
)
async def _update_settings(self, update: NvidiaTTSSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: reconnect gRPC client to apply changed settings.
self._warn_unhandled_updated_settings(changed)
return changed
def _initialize_client(self):
if self._service is not None:
@@ -150,7 +193,7 @@ class NvidiaTTSService(TTSService):
await super().start(frame)
self._initialize_client()
self._config = self._create_synthesis_config()
logger.debug(f"Initialized NvidiaTTSService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaTTSService with model: {self._settings.model}")
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -167,11 +210,11 @@ class NvidiaTTSService(TTSService):
def read_audio_responses() -> Generator[rtts.SynthesizeSpeechResponse, None, None]:
responses = self._service.synthesize_online(
text,
self._voice_id,
self._language_code,
self._settings.voice,
self._settings.language,
sample_rate_hz=self.sample_rate,
zero_shot_audio_prompt_file=None,
zero_shot_quality=self._quality,
zero_shot_quality=self._settings.quality,
custom_dictionary={},
)
return responses

68
src/pipecat/services/openai/base_llm.py
View File

@@ -10,7 +10,8 @@ import asyncio
import base64
import json
from contextlib import asynccontextmanager
from typing import Any, Dict, List, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Mapping, Optional
import httpx
from loguru import logger
@@ -32,7 +33,6 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -42,9 +42,24 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
from pipecat.services.settings import LLMSettings, _NotGiven
from pipecat.utils.tracing.service_decorators import traced_llm
@dataclass
class OpenAILLMSettings(LLMSettings):
"""Settings for OpenAI-compatible LLM services.
Parameters:
max_completion_tokens: Maximum completion tokens to generate.
service_tier: Service tier to use (e.g., "auto", "flex", "priority").
"""
max_completion_tokens: int | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
service_tier: str | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
class BaseOpenAILLMService(LLMService):
"""Base class for all services that use the AsyncOpenAI client.
@@ -55,6 +70,8 @@ class BaseOpenAILLMService(LLMService):
configurations.
"""
_settings: OpenAILLMSettings
class InputParams(BaseModel):
"""Input parameters for OpenAI model configuration.
@@ -120,20 +137,21 @@ class BaseOpenAILLMService(LLMService):
params = params or BaseOpenAILLMService.InputParams()
self._settings = {
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,
"seed": params.seed,
"temperature": params.temperature,
"top_p": params.top_p,
"max_tokens": params.max_tokens,
"max_completion_tokens": params.max_completion_tokens,
"service_tier": params.service_tier,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self._settings = OpenAILLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
presence_penalty=params.presence_penalty,
seed=params.seed,
temperature=params.temperature,
top_p=params.top_p,
max_tokens=params.max_tokens,
max_completion_tokens=params.max_completion_tokens,
service_tier=params.service_tier,
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self.set_model_name(model)
self._sync_model_name_to_metrics()
self._full_model_name: str = ""
self._client = self.create_client(
api_key=api_key,
@@ -247,23 +265,23 @@ class BaseOpenAILLMService(LLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"service_tier": self._settings["service_tier"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"seed": self._settings.seed,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
"max_completion_tokens": self._settings.max_completion_tokens,
"service_tier": self._settings.service_tier,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params
async def run_inference(
@@ -517,8 +535,6 @@ class BaseOpenAILLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = OpenAILLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

5
src/pipecat/services/openai/image.py
View File

@@ -53,7 +53,8 @@ class OpenAIImageGenService(ImageGenService):
model: DALL-E model to use for generation. Defaults to "dall-e-3".
"""
super().__init__()
self.set_model_name(model)
self._settings.model = model
self._sync_model_name_to_metrics()
self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
self._aiohttp_session = aiohttp_session
@@ -70,7 +71,7 @@ class OpenAIImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
image = await self._client.images.generate(
prompt=prompt, model=self.model_name, n=1, size=self._image_size
prompt=prompt, model=self._settings.model, n=1, size=self._image_size
)
image_url = image.data[0].url

75
src/pipecat/services/openai/realtime/llm.py
View File

@@ -10,8 +10,8 @@ import base64
import io
import json
import time
from dataclasses import dataclass
from typing import Optional
from dataclasses import dataclass, field
from typing import Any, Optional
from loguru import logger
from PIL import Image
@@ -59,6 +59,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt
@@ -90,6 +91,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class OpenAIRealtimeLLMSettings(LLMSettings):
"""Settings for OpenAI Realtime LLM services.
Parameters:
session_properties: OpenAI Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class OpenAIRealtimeLLMService(LLMService):
"""OpenAI Realtime LLM service providing real-time audio and text communication.
@@ -98,6 +112,8 @@ class OpenAIRealtimeLLMService(LLMService):
management, and real-time transcription.
"""
_settings: OpenAIRealtimeLLMSettings
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
adapter_class = OpenAIRealtimeLLMAdapter
@@ -159,12 +175,12 @@ class OpenAIRealtimeLLMService(LLMService):
self.api_key = api_key
self.base_url = full_url
self.set_model_name(model)
# Initialize session_properties
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
self._settings = OpenAIRealtimeLLMSettings(
model=model,
session_properties=session_properties or events.SessionProperties(),
)
self._sync_model_name_to_metrics()
self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused
self._video_frame_detail = video_frame_detail
@@ -227,12 +243,12 @@ class OpenAIRealtimeLLMService(LLMService):
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
modalities = self._session_properties.output_modalities or ["audio", "text"]
modalities = self._settings.session_properties.output_modalities or ["audio", "text"]
return modality in modalities
def _get_enabled_modalities(self) -> list[str]:
"""Get the list of enabled modalities."""
modalities = self._session_properties.output_modalities or ["audio", "text"]
modalities = self._settings.session_properties.output_modalities or ["audio", "text"]
# API only supports single modality responses: either ["text"] or ["audio"]
if "audio" in modalities:
return ["audio"]
@@ -305,9 +321,9 @@ class OpenAIRealtimeLLMService(LLMService):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
turn_detection_disabled = (
self._session_properties.audio
and self._session_properties.audio.input
and self._session_properties.audio.input.turn_detection is False
self._settings.session_properties.audio
and self._settings.session_properties.audio.input
and self._settings.session_properties.audio.input.turn_detection is False
)
if turn_detection_disabled:
await self.send_client_event(events.InputAudioBufferClearEvent())
@@ -327,9 +343,9 @@ class OpenAIRealtimeLLMService(LLMService):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
turn_detection_disabled = (
self._session_properties.audio
and self._session_properties.audio.input
and self._session_properties.audio.input.turn_detection is False
self._settings.session_properties.audio
and self._settings.session_properties.audio.input
and self._settings.session_properties.audio.input.turn_detection is False
)
if turn_detection_disabled:
await self.send_client_event(events.InputAudioBufferCommitEvent())
@@ -397,6 +413,16 @@ class OpenAIRealtimeLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.update path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -424,11 +450,8 @@ class OpenAIRealtimeLLMService(LLMService):
await self._handle_bot_stopped_speaking()
elif isinstance(frame, LLMMessagesAppendFrame):
await self._handle_messages_append(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
await self.push_frame(frame, direction)
@@ -513,8 +536,16 @@ class OpenAIRealtimeLLMService(LLMService):
# treat a send-side error as fatal.
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
settings = self._session_properties
async def _update_settings(self, update):
"""Apply a settings update, sending a session update if needed."""
changed = await super()._update_settings(update)
if "session_properties" in changed:
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"})
return changed
async def _send_session_update(self):
settings = self._settings.session_properties
adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
@@ -585,7 +616,7 @@ class OpenAIRealtimeLLMService(LLMService):
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
# to configure the session properties.
await self._update_settings()
await self._send_session_update()
async def _handle_evt_session_updated(self, evt):
# If this is our first context frame, run the LLM
@@ -868,7 +899,7 @@ class OpenAIRealtimeLLMService(LLMService):
await self.send_client_event(evt)
# Send new settings if needed
await self._update_settings()
await self._send_session_update()
# We're done configuring the LLM for this session
self._llm_needs_conversation_setup = False

73
src/pipecat/services/openai/stt.py
View File

@@ -16,7 +16,8 @@ Provides two STT services:
import base64
import json
from typing import AsyncGenerator, Literal, Optional, Union
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Literal, Optional, Union
from loguru import logger
@@ -34,6 +35,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription
@@ -98,24 +100,24 @@ class OpenAISTTService(BaseWhisperSTTService):
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"language": self._language,
"model": self._settings.model,
"language": self._settings.language,
}
if self._include_prob_metrics:
# GPT-4o-transcribe models only support logprobs (not verbose_json)
if self.model_name in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"):
if self._settings.model in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"):
kwargs["response_format"] = "json"
kwargs["include"] = ["logprobs"]
else:
# Whisper models support verbose_json
kwargs["response_format"] = "verbose_json"
if self._prompt is not None:
kwargs["prompt"] = self._prompt
if self._settings.prompt is not None:
kwargs["prompt"] = self._settings.prompt
if self._temperature is not None:
kwargs["temperature"] = self._temperature
if self._settings.temperature is not None:
kwargs["temperature"] = self._settings.temperature
return await self._client.audio.transcriptions.create(**kwargs)
@@ -123,6 +125,17 @@ class OpenAISTTService(BaseWhisperSTTService):
_OPENAI_SAMPLE_RATE = 24000
@dataclass
class OpenAIRealtimeSTTSettings(STTSettings):
"""Settings for the OpenAI Realtime STT service.
Parameters:
prompt: Optional prompt text to guide transcription style.
"""
prompt: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class OpenAIRealtimeSTTService(WebsocketSTTService):
"""OpenAI Realtime Speech-to-Text service using WebSocket transcription sessions.
@@ -156,6 +169,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
)
"""
_settings: OpenAIRealtimeSTTSettings
def __init__(
self,
*,
@@ -211,14 +226,19 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
self._api_key = api_key
self._base_url = base_url
self.set_model_name(model)
self._language_code = self._language_to_code(language) if language else None
self._prompt = prompt
self._turn_detection = turn_detection
self._noise_reduction = noise_reduction
self._should_interrupt = should_interrupt
self._settings = OpenAIRealtimeSTTSettings(
model=model,
language=language,
prompt=prompt,
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._session_ready = False
self._resampler = create_stream_resampler()
@@ -248,19 +268,31 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the language for speech recognition.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update and send session update if needed.
If the session is already active, sends an updated configuration
to the server.
Keeps ``_language_code`` and ``_prompt`` in sync with settings
and sends a ``session.update`` to the server when the session is active.
Args:
language: The language to use for speech recognition.
update: A :class:`STTSettings` (or ``OpenAIRealtimeSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
self._language_code = self._language_to_code(language)
changed = await super()._update_settings(update)
if not changed:
return changed
if "prompt" in changed and isinstance(self._settings, OpenAIRealtimeSTTSettings):
self._prompt = self._settings.prompt
if self._session_ready:
await self._send_session_update()
return changed
async def start(self, frame: StartFrame):
"""Start the service and establish WebSocket connection.
@@ -405,10 +437,13 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
async def _send_session_update(self):
"""Send ``session.update`` to configure the transcription session."""
transcription: dict = {"model": self.model_name}
transcription: dict = {"model": self._settings.model}
if self._language_code:
transcription["language"] = self._language_code
language_code = (
self._language_to_code(self._settings.language) if self._settings.language else None
)
if language_code:
transcription["language"] = language_code
if self._prompt:
transcription["prompt"] = self._prompt

51
src/pipecat/services/openai/tts.py
View File

@@ -10,6 +10,7 @@ This module provides integration with OpenAI's text-to-speech API for
generating high-quality synthetic speech from text input.
"""
from dataclasses import dataclass, field
from typing import AsyncGenerator, Dict, Literal, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -60,6 +62,19 @@ VALID_VOICES: Dict[str, ValidVoice] = {
}
@dataclass
class OpenAITTSSettings(TTSSettings):
"""Settings for OpenAI TTS service.
Parameters:
instructions: Instructions to guide voice synthesis behavior.
speed: Voice speed control (0.25 to 4.0, default 1.0).
"""
instructions: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class OpenAITTSService(TTSService):
"""OpenAI Text-to-Speech service that generates audio from text.
@@ -68,6 +83,8 @@ class OpenAITTSService(TTSService):
speech synthesis with streaming audio output.
"""
_settings: OpenAITTSSettings
OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz
class InputParams(BaseModel):
@@ -117,8 +134,6 @@ class OpenAITTSService(TTSService):
)
super().__init__(sample_rate=sample_rate, **kwargs)
self.set_model_name(model)
self.set_voice(voice)
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
if instructions or speed:
@@ -132,10 +147,13 @@ class OpenAITTSService(TTSService):
stacklevel=2,
)
self._settings = {
"instructions": params.instructions if params else instructions,
"speed": params.speed if params else speed,
}
self._settings = OpenAITTSSettings(
model=model,
voice=voice,
instructions=params.instructions if params else instructions,
speed=params.speed if params else speed,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -145,15 +163,6 @@ class OpenAITTSService(TTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model to use.
Args:
model: The model name to use for text-to-speech synthesis.
"""
logger.info(f"Switching TTS model to: [{model}]")
self.set_model_name(model)
async def start(self, frame: StartFrame):
"""Start the OpenAI TTS service.
@@ -185,16 +194,16 @@ class OpenAITTSService(TTSService):
# Setup API parameters
create_params = {
"input": text,
"model": self.model_name,
"voice": VALID_VOICES[self._voice_id],
"model": self._settings.model,
"voice": VALID_VOICES[self._settings.voice],
"response_format": "pcm",
}
if self._settings["instructions"]:
create_params["instructions"] = self._settings["instructions"]
if self._settings.instructions:
create_params["instructions"] = self._settings.instructions
if self._settings["speed"]:
create_params["speed"] = self._settings["speed"]
if self._settings.speed:
create_params["speed"] = self._settings.speed
async with self._client.audio.speech.with_streaming_response.create(
**create_params

63
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -10,7 +10,7 @@ import base64
import json
import time
import warnings
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Optional
from loguru import logger
@@ -54,6 +54,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai.llm import OpenAIContextAggregatorPair
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt
@@ -91,6 +92,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class OpenAIRealtimeBetaLLMSettings(LLMSettings):
"""Settings for OpenAI Realtime Beta LLM services.
Parameters:
session_properties: OpenAI Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class OpenAIRealtimeBetaLLMService(LLMService):
"""OpenAI Realtime Beta LLM service providing real-time audio and text communication.
@@ -103,6 +117,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
management, and real-time transcription.
"""
_settings: OpenAIRealtimeBetaLLMSettings
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
adapter_class = OpenAIRealtimeLLMAdapter
@@ -144,11 +160,12 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self.api_key = api_key
self.base_url = full_url
self.set_model_name(model)
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
self._settings = OpenAIRealtimeBetaLLMSettings(
model=model,
session_properties=session_properties or events.SessionProperties(),
)
self._sync_model_name_to_metrics()
self._audio_input_paused = start_audio_paused
self._send_transcription_frames = send_transcription_frames
self._websocket = None
@@ -187,12 +204,12 @@ class OpenAIRealtimeBetaLLMService(LLMService):
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
modalities = self._session_properties.modalities or ["audio", "text"]
modalities = self._settings.session_properties.modalities or ["audio", "text"]
return modality in modalities
def _get_enabled_modalities(self) -> list[str]:
"""Get the list of enabled modalities."""
return self._session_properties.modalities or ["audio", "text"]
return self._settings.session_properties.modalities or ["audio", "text"]
async def retrieve_conversation_item(self, item_id: str):
"""Retrieve a conversation item by ID from the server.
@@ -259,7 +276,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_interruption(self):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
if self._session_properties.turn_detection is False:
if self._settings.session_properties.turn_detection is False:
await self.send_client_event(events.InputAudioBufferClearEvent())
await self.send_client_event(events.ResponseCancelEvent())
await self._truncate_current_audio_response()
@@ -276,7 +293,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_user_stopped_speaking(self, frame):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
if self._session_properties.turn_detection is False:
if self._settings.session_properties.turn_detection is False:
await self.send_client_event(events.InputAudioBufferCommitEvent())
await self.send_client_event(events.ResponseCreateEvent())
@@ -342,6 +359,16 @@ class OpenAIRealtimeBetaLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.update path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.update is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -377,11 +404,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._handle_messages_append(frame)
elif isinstance(frame, RealtimeMessagesUpdateFrame):
self._context = frame.context
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
elif isinstance(frame, RealtimeFunctionCallResultFrame):
await self._handle_function_call_result(frame.result_frame)
@@ -456,8 +480,15 @@ class OpenAIRealtimeBetaLLMService(LLMService):
# treat a send-side error as fatal.
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
settings = self._session_properties
async def _update_settings(self, update):
"""Apply a settings update, sending a session update if needed."""
changed = await super()._update_settings(update)
if "session_properties" in changed:
await self._send_session_update()
return changed
async def _send_session_update(self):
settings = self._settings.session_properties
# tools given in the context override the tools in the session properties
if self._context and self._context.tools:
settings.tools = self._context.tools
@@ -511,7 +542,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
# to configure the session properties.
await self._update_settings()
await self._send_session_update()
async def _handle_evt_session_updated(self, evt):
# If this is our first context frame, run the LLM
@@ -750,7 +781,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self._context.llm_needs_initial_messages = False
if self._context.llm_needs_settings_update:
await self._update_settings()
await self._send_session_update()
self._context.llm_needs_settings_update = False
logger.debug(f"Creating response: {self._context.get_messages_for_logging()}")

3
src/pipecat/services/openrouter/llm.py
View File

@@ -72,8 +72,7 @@ class OpenRouterLLMService(OpenAILLMService):
Transformed parameters ready for the API call.
"""
params = super().build_chat_completion_params(params_from_context)
model = getattr(self, "model_name", getattr(self, "model", "")).lower()
if "gemini" in model:
if "gemini" in self._settings.model.lower():
messages = params.get("messages", [])
if not messages:
return params

22
src/pipecat/services/perplexity/llm.py
View File

@@ -66,22 +66,22 @@ class PerplexityLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"messages": params_from_context["messages"],
}
# Add OpenAI-compatible parameters if they're set
if self._settings["frequency_penalty"] is not NOT_GIVEN:
params["frequency_penalty"] = self._settings["frequency_penalty"]
if self._settings["presence_penalty"] is not NOT_GIVEN:
params["presence_penalty"] = self._settings["presence_penalty"]
if self._settings["temperature"] is not NOT_GIVEN:
params["temperature"] = self._settings["temperature"]
if self._settings["top_p"] is not NOT_GIVEN:
params["top_p"] = self._settings["top_p"]
if self._settings["max_tokens"] is not NOT_GIVEN:
params["max_tokens"] = self._settings["max_tokens"]
if self._settings.frequency_penalty is not NOT_GIVEN:
params["frequency_penalty"] = self._settings.frequency_penalty
if self._settings.presence_penalty is not NOT_GIVEN:
params["presence_penalty"] = self._settings.presence_penalty
if self._settings.temperature is not NOT_GIVEN:
params["temperature"] = self._settings.temperature
if self._settings.top_p is not NOT_GIVEN:
params["top_p"] = self._settings.top_p
if self._settings.max_tokens is not NOT_GIVEN:
params["max_tokens"] = self._settings.max_tokens
return params

40
src/pipecat/services/piper/tts.py
View File

@@ -7,8 +7,9 @@
"""Piper TTS service implementation."""
import asyncio
from dataclasses import dataclass
from pathlib import Path
from typing import AsyncGenerator, AsyncIterator, Optional
from typing import Any, AsyncGenerator, AsyncIterator, Optional
import aiohttp
from loguru import logger
@@ -19,6 +20,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import TTSSettings
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -31,6 +33,13 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class PiperTTSSettings(TTSSettings):
"""Settings for Piper TTS service."""
pass
class PiperTTSService(TTSService):
"""Piper TTS service implementation.
@@ -39,6 +48,8 @@ class PiperTTSService(TTSService):
match the configured sample rate.
"""
_settings: PiperTTSSettings
def __init__(
self,
*,
@@ -60,7 +71,7 @@ class PiperTTSService(TTSService):
"""
super().__init__(**kwargs)
self._voice_id = voice_id
self._settings = PiperTTSSettings(voice=voice_id)
download_dir = download_dir or Path.cwd()
@@ -85,6 +96,18 @@ class PiperTTSService(TTSService):
"""
return True
async def _update_settings(self, update: PiperTTSSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: voice changes would require re-downloading and loading the model.
self._warn_unhandled_updated_settings(changed)
return changed
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Piper.
@@ -143,6 +166,13 @@ class PiperTTSService(TTSService):
# $ uv pip install "piper-tts[http]"
# $ uv run python -m piper.http_server -m en_US-ryan-high
#
@dataclass
class PiperHttpTTSSettings(TTSSettings):
"""Settings for Piper HTTP TTS service."""
pass
class PiperHttpTTSService(TTSService):
"""Piper HTTP TTS service implementation.
@@ -151,6 +181,8 @@ class PiperHttpTTSService(TTSService):
rates and automatic WAV header removal.
"""
_settings: PiperHttpTTSSettings
def __init__(
self,
*,
@@ -175,7 +207,7 @@ class PiperHttpTTSService(TTSService):
self._base_url = base_url
self._session = aiohttp_session
self._model_id = voice_id
self._settings = PiperHttpTTSSettings(voice=voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -205,7 +237,7 @@ class PiperHttpTTSService(TTSService):
data = {
"text": text,
"voice": self._model_id,
"voice": self._settings.voice,
}
async with self._session.post(self._base_url, json=data, headers=headers) as response:

120
src/pipecat/services/playht/tts.py
View File

@@ -15,7 +15,8 @@ import json
import struct
import uuid
import warnings
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -33,6 +34,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -98,6 +100,25 @@ def language_to_playht_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class PlayHTTTSSettings(TTSSettings):
"""Settings for PlayHT TTS services.
Parameters:
output_format: Audio output format.
voice_engine: Voice engine to use.
speed: Speech speed multiplier. Defaults to 1.0.
seed: Random seed for voice consistency.
playht_sample_rate: Audio sample rate sent to the API.
"""
output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
voice_engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
seed: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
playht_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class PlayHTTTSService(InterruptibleTTSService):
"""PlayHT WebSocket-based text-to-speech service.
@@ -111,6 +132,8 @@ class PlayHTTTSService(InterruptibleTTSService):
language settings.
"""
_settings: PlayHTTTSSettings
class InputParams(BaseModel):
"""Input parameters for PlayHT TTS configuration.
@@ -171,17 +194,18 @@ class PlayHTTTSService(InterruptibleTTSService):
self._receive_task = None
self._context_id = None
self._settings = {
"language": self.language_to_service_language(params.language)
self._settings = PlayHTTTSSettings(
model=voice_engine,
voice=voice_url,
language=self.language_to_service_language(params.language)
if params.language
else "english",
"output_format": output_format,
"voice_engine": voice_engine,
"speed": params.speed,
"seed": params.seed,
}
self.set_model_name(voice_engine)
self.set_voice(voice_url)
output_format=output_format,
voice_engine=voice_engine,
speed=params.speed,
seed=params.seed,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -191,6 +215,25 @@ class PlayHTTTSService(InterruptibleTTSService):
"""
return True
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT service language format.
@@ -305,13 +348,13 @@ class PlayHTTTSService(InterruptibleTTSService):
# Handle the new response format with multiple URLs
if "websocket_urls" in data:
# Select URL based on voice_engine
if self._settings["voice_engine"] in data["websocket_urls"]:
if self._settings.voice_engine in data["websocket_urls"]:
self._websocket_url = data["websocket_urls"][
self._settings["voice_engine"]
self._settings.voice_engine
]
else:
raise ValueError(
f"Unsupported voice engine: {self._settings['voice_engine']}"
f"Unsupported voice engine: {self._settings.voice_engine}"
)
else:
raise ValueError("Invalid response: missing websocket_urls")
@@ -396,13 +439,13 @@ class PlayHTTTSService(InterruptibleTTSService):
tts_command = {
"text": text,
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
"voice": self._settings.voice,
"voice_engine": self._settings.voice_engine,
"output_format": self._settings.output_format,
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"speed": self._settings["speed"],
"seed": self._settings["seed"],
"language": self._settings.language,
"speed": self._settings.speed,
"seed": self._settings.seed,
"request_id": self._context_id,
}
@@ -436,6 +479,8 @@ class PlayHTHttpTTSService(TTSService):
required and simpler integration is preferred.
"""
_settings: PlayHTTTSSettings
class InputParams(BaseModel):
"""Input parameters for PlayHT HTTP TTS configuration.
@@ -514,17 +559,18 @@ class PlayHTHttpTTSService(TTSService):
# Extract the base engine name
voice_engine = voice_engine.replace("-ws", "")
self._settings = {
"language": self.language_to_service_language(params.language)
self._settings = PlayHTTTSSettings(
model=voice_engine,
voice=voice_url,
language=self.language_to_service_language(params.language)
if params.language
else "english",
"output_format": output_format,
"voice_engine": voice_engine,
"speed": params.speed,
"seed": params.seed,
}
self.set_model_name(voice_engine)
self.set_voice(voice_url)
output_format=output_format,
voice_engine=voice_engine,
speed=params.speed,
seed=params.seed,
)
self._sync_model_name_to_metrics()
async def start(self, frame: StartFrame):
"""Start the PlayHT HTTP TTS service.
@@ -533,7 +579,7 @@ class PlayHTHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.playht_sample_rate = self.sample_rate
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -573,18 +619,18 @@ class PlayHTHttpTTSService(TTSService):
# Prepare the request payload
payload = {
"text": text,
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
"voice": self._settings.voice,
"voice_engine": self._settings.voice_engine,
"output_format": self._settings.output_format,
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"language": self._settings.language,
}
# Add optional parameters if they exist
if self._settings["speed"] is not None:
payload["speed"] = self._settings["speed"]
if self._settings["seed"] is not None:
payload["seed"] = self._settings["seed"]
if self._settings.speed is not None:
payload["speed"] = self._settings.speed
if self._settings.seed is not None:
payload["seed"] = self._settings.seed
headers = {
"Authorization": f"Bearer {self._api_key}",

50
src/pipecat/services/resembleai/tts.py
View File

@@ -8,7 +8,8 @@
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, ClassVar, Dict, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextWordTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -36,6 +38,26 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class ResembleAITTSSettings(TTSSettings):
"""Settings for Resemble AI TTS service.
Parameters:
precision: PCM bit depth (PCM_32, PCM_24, PCM_16, or MULAW).
output_format: Audio format (wav or mp3).
resemble_sample_rate: Audio sample rate sent to the API.
"""
precision: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_format: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
resemble_sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {
"voice_id": "voice",
"sample_rate": "resemble_sample_rate",
}
class ResembleAITTSService(AudioContextWordTTSService):
"""Resemble AI TTS service with WebSocket streaming and word timestamps.
@@ -44,6 +66,8 @@ class ResembleAITTSService(AudioContextWordTTSService):
multiple simultaneous synthesis requests with proper interruption support.
"""
_settings: ResembleAITTSSettings
def __init__(
self,
*,
@@ -73,13 +97,13 @@ class ResembleAITTSService(AudioContextWordTTSService):
)
self._api_key = api_key
self._voice_id = voice_id
self._url = url
self._settings = {
"precision": precision,
"output_format": output_format,
"sample_rate": sample_rate,
}
self._settings = ResembleAITTSSettings(
voice=voice_id,
precision=precision,
output_format=output_format,
resemble_sample_rate=sample_rate,
)
self._websocket = None
self._request_id_counter = 0
@@ -100,8 +124,6 @@ class ResembleAITTSService(AudioContextWordTTSService):
self._jitter_buffer_bytes = 44100 # ~1000ms at 22050Hz to handle 400ms+ network gaps
self._playback_started: dict[str, bool] = {} # Track if we've started playback per request
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -120,13 +142,13 @@ class ResembleAITTSService(AudioContextWordTTSService):
JSON string containing the request payload.
"""
msg = {
"voice_uuid": self._voice_id,
"voice_uuid": self._settings.voice,
"data": text,
"binary_response": False, # Use JSON frames to get timestamps
"request_id": self._request_id_counter, # ResembleAI only accepts number
"output_format": self._settings["output_format"],
"sample_rate": self._settings["sample_rate"],
"precision": self._settings["precision"],
"output_format": self._settings.output_format,
"sample_rate": self._settings.resemble_sample_rate,
"precision": self._settings.precision,
"no_audio_header": True,
}
@@ -140,7 +162,7 @@ class ResembleAITTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.resemble_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):

416
src/pipecat/services/rime/tts.py
View File

@@ -12,7 +12,8 @@ using Rime's API for streaming and batch audio synthesis.
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Optional
import aiohttp
from loguru import logger
@@ -30,6 +31,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import (
AudioContextWordTTSService,
InterruptibleTTSService,
@@ -68,6 +70,66 @@ def language_to_rime_language(language: Language) -> str:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class RimeTTSSettings(TTSSettings):
"""Settings for Rime WS JSON and HTTP TTS services.
Parameters:
audioFormat: Audio output format.
samplingRate: Audio sample rate.
segment: Text segmentation mode ("immediate", "bySentence", "never").
speedAlpha: Speech speed multiplier (mistv2 only).
reduceLatency: Whether to reduce latency at potential quality cost (mistv2 only).
pauseBetweenBrackets: Whether to add pauses between bracketed content (mistv2 only).
phonemizeBetweenBrackets: Whether to phonemize bracketed content (mistv2 only).
noTextNormalization: Whether to disable text normalization (mistv2 only).
saveOovs: Whether to save out-of-vocabulary words (mistv2 only).
inlineSpeedAlpha: Inline speed control markup.
repetition_penalty: Token repetition penalty (arcana only, 1.0-2.0).
temperature: Sampling temperature (arcana only, 0.0-1.0).
top_p: Cumulative probability threshold (arcana only, 0.0-1.0).
"""
audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speedAlpha: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
reduceLatency: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pauseBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
phonemizeBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
noTextNormalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
saveOovs: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
inlineSpeedAlpha: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"}
@dataclass
class RimeNonJsonTTSSettings(TTSSettings):
"""Settings for Rime non-JSON WS TTS service.
Parameters:
audioFormat: Audio output format.
samplingRate: Audio sample rate.
segment: Text segmentation mode ("immediate", "bySentence", "never").
repetition_penalty: Token repetition penalty (1.0-2.0).
temperature: Sampling temperature (0.0-1.0).
top_p: Cumulative probability threshold (0.0-1.0).
"""
audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"}
class RimeTTSService(AudioContextWordTTSService):
"""Text-to-Speech service using Rime's websocket API.
@@ -76,6 +138,8 @@ class RimeTTSService(AudioContextWordTTSService):
within a turn.
"""
_settings: RimeTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime TTS service.
@@ -156,14 +220,41 @@ class RimeTTSService(AudioContextWordTTSService):
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("spell(", ")")])
self._params = params or RimeTTSService.InputParams()
params = params or RimeTTSService.InputParams()
# Store service configuration
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._model = model
self._settings = self._build_settings()
self._settings = RimeTTSSettings(
model=model,
voice=voice_id,
audioFormat="pcm",
samplingRate=0, # updated in start()
language=self.language_to_service_language(params.language)
if params.language
else NOT_GIVEN,
segment=params.segment if params.segment is not None else NOT_GIVEN,
# Arcana params
repetition_penalty=params.repetition_penalty
if params.repetition_penalty is not None
else NOT_GIVEN,
temperature=params.temperature if params.temperature is not None else NOT_GIVEN,
top_p=params.top_p if params.top_p is not None else NOT_GIVEN,
# Mistv2 params
speedAlpha=params.speed_alpha if params.speed_alpha is not None else NOT_GIVEN,
reduceLatency=params.reduce_latency if params.reduce_latency is not None else NOT_GIVEN,
pauseBetweenBrackets=params.pause_between_brackets
if params.pause_between_brackets is not None
else NOT_GIVEN,
phonemizeBetweenBrackets=params.phonemize_between_brackets
if params.phonemize_between_brackets is not None
else NOT_GIVEN,
noTextNormalization=params.no_text_normalization
if params.no_text_normalization is not None
else NOT_GIVEN,
saveOovs=params.save_oovs if params.save_oovs is not None else NOT_GIVEN,
)
self._sync_model_name_to_metrics()
# State tracking
self._receive_task = None
@@ -189,60 +280,49 @@ class RimeTTSService(AudioContextWordTTSService):
"""
return language_to_rime_language(language)
def _build_settings(self) -> dict:
"""Build query params for the WebSocket URL based on the current model and params.
def _build_ws_params(self) -> dict[str, Any]:
"""Build query params for the WebSocket URL from current settings.
Returns:
Dictionary of query parameters. Only explicitly-set values are included.
Dictionary of query parameters for the WebSocket URL.
Only explicitly-set values are included. Boolean mistv2 params
are serialized with ``json.dumps()`` for the wire format.
"""
settings = {
"speaker": self._voice_id,
"modelId": self._model,
"audioFormat": "pcm",
"samplingRate": self.sample_rate or 0,
params: dict[str, Any] = {
"speaker": self._settings.voice,
"modelId": self._settings.model,
"audioFormat": self._settings.audioFormat,
"samplingRate": self._settings.samplingRate,
}
if self._params.language:
settings["lang"] = self.language_to_service_language(self._params.language) or "eng"
if self._params.segment is not None:
settings["segment"] = self._params.segment
if is_given(self._settings.language):
params["lang"] = self._settings.language
if is_given(self._settings.segment):
params["segment"] = self._settings.segment
if self._model == "arcana":
if self._params.repetition_penalty is not None:
settings["repetition_penalty"] = self._params.repetition_penalty
if self._params.temperature is not None:
settings["temperature"] = self._params.temperature
if self._params.top_p is not None:
settings["top_p"] = self._params.top_p
if self._settings.model == "arcana":
if is_given(self._settings.repetition_penalty):
params["repetition_penalty"] = self._settings.repetition_penalty
if is_given(self._settings.temperature):
params["temperature"] = self._settings.temperature
if is_given(self._settings.top_p):
params["top_p"] = self._settings.top_p
else: # mistv2/mist
if self._params.speed_alpha is not None:
settings["speedAlpha"] = self._params.speed_alpha
if self._params.reduce_latency is not None:
settings["reduceLatency"] = self._params.reduce_latency
if self._params.pause_between_brackets is not None:
settings["pauseBetweenBrackets"] = json.dumps(self._params.pause_between_brackets)
if self._params.phonemize_between_brackets is not None:
settings["phonemizeBetweenBrackets"] = json.dumps(
self._params.phonemize_between_brackets
if is_given(self._settings.speedAlpha):
params["speedAlpha"] = self._settings.speedAlpha
if is_given(self._settings.reduceLatency):
params["reduceLatency"] = self._settings.reduceLatency
if is_given(self._settings.pauseBetweenBrackets):
params["pauseBetweenBrackets"] = json.dumps(self._settings.pauseBetweenBrackets)
if is_given(self._settings.phonemizeBetweenBrackets):
params["phonemizeBetweenBrackets"] = json.dumps(
self._settings.phonemizeBetweenBrackets
)
if self._params.no_text_normalization is not None:
settings["noTextNormalization"] = json.dumps(self._params.no_text_normalization)
if self._params.save_oovs is not None:
settings["saveOovs"] = json.dumps(self._params.save_oovs)
if is_given(self._settings.noTextNormalization):
params["noTextNormalization"] = json.dumps(self._settings.noTextNormalization)
if is_given(self._settings.saveOovs):
params["saveOovs"] = json.dumps(self._settings.saveOovs)
return settings
async def set_model(self, model: str):
"""Update the TTS model and reconnect.
Args:
model: The model name to use for synthesis.
"""
self._model = model
self._settings = self._build_settings()
await super().set_model(model)
if self._websocket:
await self._disconnect()
await self._connect()
return params
# A set of Rime-specific helpers for text transformations
def SPELL(text: str) -> str:
@@ -269,72 +349,20 @@ class RimeTTSService(AudioContextWordTTSService):
self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)])
return f"[{text}]"
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if necessary.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and reconnect if necessary.
Since all settings are WebSocket URL query parameters,
any setting change requires reconnecting to apply the new values.
"""
prev_settings = self._settings.copy()
await super()._update_settings(settings)
changed = await super()._update_settings(update)
needs_reconnect = False
if "voice" in settings or "voice_id" in settings:
self._settings["speaker"] = self._voice_id
if prev_settings.get("speaker") != self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
needs_reconnect = True
if "model" in settings:
self._settings = self._build_settings()
needs_reconnect = True
if "language" in settings:
new_lang = self.language_to_service_language(settings["language"])
if new_lang and new_lang != prev_settings.get("lang"):
logger.info(f"Updating language to: [{new_lang}]")
self._settings["lang"] = new_lang
needs_reconnect = True
# Arcana params
for key, settings_key in [
("repetition_penalty", "repetition_penalty"),
("temperature", "temperature"),
("top_p", "top_p"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 params
for key, settings_key in [
("speed_alpha", "speedAlpha"),
("reduce_latency", "reduceLatency"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 boolean params (need json.dumps)
for key, settings_key in [
("pause_between_brackets", "pauseBetweenBrackets"),
("phonemize_between_brackets", "phonemizeBetweenBrackets"),
("no_text_normalization", "noTextNormalization"),
("save_oovs", "saveOovs"),
]:
if key in settings and json.dumps(settings[key]) != prev_settings.get(settings_key):
self._settings[settings_key] = json.dumps(settings[key])
needs_reconnect = True
if "segment" in settings and settings["segment"] != prev_settings.get("segment"):
self._settings["segment"] = settings["segment"]
needs_reconnect = True
if needs_reconnect and self._websocket:
if changed and self._websocket:
await self._disconnect()
await self._connect()
return changed
def _build_msg(self, text: str = "") -> dict:
"""Build JSON message for Rime API."""
msg = {"text": text, "contextId": self.get_active_audio_context_id()}
@@ -358,7 +386,7 @@ class RimeTTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings = self._build_settings()
self._settings.samplingRate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -404,7 +432,8 @@ class RimeTTSService(AudioContextWordTTSService):
if self._websocket and self._websocket.state is State.OPEN:
return
params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None)
ws_params = self._build_ws_params()
params = "&".join(f"{k}={v}" for k, v in ws_params.items() if v is not None)
url = f"{self._url}?{params}"
headers = {"Authorization": f"Bearer {self._api_key}"}
self._websocket = await websocket_connect(url, additional_headers=headers)
@@ -580,6 +609,8 @@ class RimeHttpTTSService(TTSService):
Suitable for use cases where streaming is not required.
"""
_settings: RimeTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime HTTP TTS service.
@@ -628,20 +659,19 @@ class RimeHttpTTSService(TTSService):
self._api_key = api_key
self._session = aiohttp_session
self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = {
"lang": self.language_to_service_language(params.language)
self._settings = RimeTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": params.pause_between_brackets,
"phonemizeBetweenBrackets": params.phonemize_between_brackets,
}
self.set_voice(voice_id)
self.set_model_name(model)
if params.inline_speed_alpha:
self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else NOT_GIVEN,
voice=voice_id,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -681,10 +711,18 @@ class RimeHttpTTSService(TTSService):
"Content-Type": "application/json",
}
payload = self._settings.copy()
payload = {
"lang": self._settings.language,
"speedAlpha": self._settings.speedAlpha,
"reduceLatency": self._settings.reduceLatency,
"pauseBetweenBrackets": self._settings.pauseBetweenBrackets,
"phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets,
}
if is_given(self._settings.inlineSpeedAlpha):
payload["inlineSpeedAlpha"] = self._settings.inlineSpeedAlpha
payload["text"] = text
payload["speaker"] = self._voice_id
payload["modelId"] = self._model_name
payload["speaker"] = self._settings.voice
payload["modelId"] = self._settings.model
payload["samplingRate"] = self.sample_rate
# Arcana does not support PCM audio
@@ -743,6 +781,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
accepts and returns non-JSON messages.
"""
_settings: RimeNonJsonTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime Non-JSON WebSocket TTS service.
@@ -798,28 +838,25 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
params = params or RimeNonJsonTTSService.InputParams()
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._model = model
self._settings = {
"speaker": voice_id,
"modelId": model,
"audioFormat": audio_format,
"samplingRate": sample_rate,
}
if params.language:
self._settings["lang"] = self.language_to_service_language(params.language)
if params.segment is not None:
self._settings["segment"] = params.segment
if params.repetition_penalty is not None:
self._settings["repetition_penalty"] = params.repetition_penalty
if params.temperature is not None:
self._settings["temperature"] = params.temperature
if params.top_p is not None:
self._settings["top_p"] = params.top_p
self._settings = RimeNonJsonTTSSettings(
voice=voice_id,
model=model,
audioFormat=audio_format,
samplingRate=sample_rate,
language=self.language_to_service_language(params.language)
if params.language
else NOT_GIVEN,
segment=params.segment if params.segment is not None else NOT_GIVEN,
repetition_penalty=params.repetition_penalty
if params.repetition_penalty is not None
else NOT_GIVEN,
temperature=params.temperature if params.temperature is not None else NOT_GIVEN,
top_p=params.top_p if params.top_p is not None else NOT_GIVEN,
)
self._sync_model_name_to_metrics()
# Add any extra parameters for future compatibility
if params.extra:
self._settings.update(params.extra)
self._settings.extra.update(params.extra)
self._receive_task = None
self._context_id: Optional[str] = None
@@ -851,7 +888,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["samplingRate"] = self.sample_rate
self._settings.samplingRate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -895,8 +932,26 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
try:
if self._websocket and self._websocket.state is State.OPEN:
return
# Build URL with query parameters (only non-None values)
params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None)
# Build URL with query parameters (only given, non-None values)
settings_dict = {
"speaker": self._settings.voice,
"modelId": self._settings.model,
"audioFormat": self._settings.audioFormat,
"samplingRate": self._settings.samplingRate,
}
if is_given(self._settings.language):
settings_dict["lang"] = self._settings.language
if is_given(self._settings.segment):
settings_dict["segment"] = self._settings.segment
if is_given(self._settings.repetition_penalty):
settings_dict["repetition_penalty"] = self._settings.repetition_penalty
if is_given(self._settings.temperature):
settings_dict["temperature"] = self._settings.temperature
if is_given(self._settings.top_p):
settings_dict["top_p"] = self._settings.top_p
# Include extras
settings_dict.update(self._settings.extra)
params = "&".join(f"{k}={v}" for k, v in settings_dict.items() if v is not None)
url = f"{self._url}?{params}"
headers = {"Authorization": f"Bearer {self._api_key}"}
self._websocket = await websocket_connect(
@@ -990,68 +1045,17 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if necessary.
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and reconnect if necessary.
Since all settings are WebSocket URL query parameters,
any setting change requires reconnecting to apply the new values.
"""
needs_reconnect = False
changed = await super()._update_settings(update)
# Track previous values from self._settings only
prev_settings = self._settings.copy()
# Let parent class handle standard settings (voice, model, language)
await super()._update_settings(settings)
# Check if voice changed and update settings dict
if "voice" in settings or "voice_id" in settings:
self._settings["speaker"] = self._voice_id
if prev_settings.get("speaker") != self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
needs_reconnect = True
# Check if model changed and update settings dict
if "model" in settings:
self._settings["modelId"] = self._model
if prev_settings.get("modelId") != self._model:
logger.info(f"Switching TTS model to: [{self._model}]")
needs_reconnect = True
# Handle language explicitly
if "language" in settings:
new_lang = self.language_to_service_language(settings["language"])
if new_lang and new_lang != prev_settings.get("lang"):
logger.info(f"Updating language to: [{new_lang}]")
self._settings["lang"] = new_lang
needs_reconnect = True
# Check other parameters
for key in ["segment", "repetition_penalty", "temperature", "top_p"]:
if key in settings and settings[key] != prev_settings.get(key):
logger.info(f"Updating {key} to: [{settings[key]}]")
self._settings[key] = settings[key]
needs_reconnect = True
# Handle extra parameters
for key, value in settings.items():
if key not in [
"voice",
"voice_id",
"model",
"language",
"segment",
"repetition_penalty",
"temperature",
"top_p",
]:
if value != prev_settings.get(key):
logger.info(f"Updating extra parameter {key} to: [{value}]")
self._settings[key] = value
needs_reconnect = True
# Reconnect if any setting changed
if needs_reconnect:
if changed:
logger.debug("Settings changed, reconnecting WebSocket with new parameters")
await self._disconnect()
await self._connect()
return changed

12
src/pipecat/services/sambanova/llm.py
View File

@@ -84,19 +84,19 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
"max_completion_tokens": self._settings.max_completion_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params
@traced_llm # type: ignore

2
src/pipecat/services/sambanova/stt.py
View File

@@ -72,7 +72,7 @@ class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"model": self._settings.model,
"response_format": "json",
"language": self._language,
}

182
src/pipecat/services/sarvam/stt.py
View File

@@ -12,8 +12,8 @@ can handle multiple audio formats for Indian language speech recognition.
"""
import base64
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Literal, Optional
from loguru import logger
from pydantic import BaseModel
@@ -32,6 +32,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.sarvam._sdk import sdk_headers
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import SARVAM_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -130,6 +131,23 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = {
}
@dataclass
class SarvamSTTSettings(STTSettings):
"""Settings for the Sarvam STT service.
Parameters:
prompt: Optional prompt to guide transcription/translation style.
mode: Mode of operation (transcribe, translate, verbatim, etc.).
vad_signals: Enable VAD signals in response.
high_vad_sensitivity: Enable high VAD sensitivity.
"""
prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
mode: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SarvamSTTService(STTService):
"""Sarvam speech-to-text service.
@@ -148,6 +166,8 @@ class SarvamSTTService(STTService):
...
"""
_settings: SarvamSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Sarvam STT service.
@@ -228,24 +248,9 @@ class SarvamSTTService(STTService):
**kwargs,
)
self.set_model_name(model)
self._api_key = api_key
self._language_code: Optional[Language] = params.language
# Set language string: use provided language or model's default
if params.language:
self._language_string = language_to_sarvam_language(params.language)
else:
self._language_string = self._config.default_language
self._prompt = params.prompt
# Set mode: use provided mode or model's default
self._mode = params.mode if params.mode is not None else self._config.default_mode
# Store connection parameters
self._vad_signals = params.vad_signals
self._high_vad_sensitivity = params.high_vad_sensitivity
self._input_audio_codec = input_audio_codec
# Initialize Sarvam SDK client
@@ -263,7 +268,20 @@ class SarvamSTTService(STTService):
self._socket_client = None
self._receive_task = None
if self._vad_signals:
# Resolve mode default from model config
mode = params.mode if params.mode is not None else self._config.default_mode
self._settings = SarvamSTTSettings(
model=model,
language=params.language,
prompt=params.prompt if params.prompt is not None else NOT_GIVEN,
mode=mode if mode is not None else NOT_GIVEN,
vad_signals=params.vad_signals,
high_vad_sensitivity=params.high_vad_sensitivity,
)
self._sync_model_name_to_metrics()
if params.vad_signals:
self._register_event_handler("on_speech_started")
self._register_event_handler("on_speech_stopped")
self._register_event_handler("on_utterance_end")
@@ -281,6 +299,12 @@ class SarvamSTTService(STTService):
"""
return language_to_sarvam_language(language)
def _get_language_string(self) -> Optional[str]:
"""Resolve the current language setting to a Sarvam language code string."""
if self._settings.language:
return language_to_sarvam_language(self._settings.language)
return self._config.default_language
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -298,50 +322,91 @@ class SarvamSTTService(STTService):
await super().process_frame(frame, direction)
# Only handle VAD frames when not using Sarvam's VAD signals
if not self._vad_signals:
if not self._settings.vad_signals:
if isinstance(frame, VADUserStartedSpeakingFrame):
await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
if self._socket_client:
await self._socket_client.flush()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update, validate, sync state, and reconnect.
Args:
language: The language to use for speech recognition.
update: A :class:`STTSettings` (or ``SarvamSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
Raises:
ValueError: If called on a model that auto-detects language.
ValueError: If a setting is not supported by the current model.
"""
if not self._config.supports_language:
raise ValueError(
f"Model '{self.model_name}' does not support language parameter "
"(auto-detects language)."
)
# Validate against model capabilities before applying
if is_given(update.language) and update.language is not None:
if not self._config.supports_language:
raise ValueError(
f"Model '{self._settings.model}' does not support language parameter "
"(auto-detects language)."
)
logger.info(f"Switching STT language to: [{language}]")
self._language_code = language
self._language_string = language_to_sarvam_language(language)
await self._disconnect()
await self._connect()
if isinstance(update, SarvamSTTSettings):
if is_given(update.prompt) and update.prompt is not None:
if not self._config.supports_prompt:
raise ValueError(
f"Model '{self._settings.model}' does not support prompt parameter."
)
if is_given(update.mode) and update.mode is not None:
if not self._config.supports_mode:
raise ValueError(
f"Model '{self._settings.model}' does not support mode parameter."
)
changed = await super()._update_settings(update)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if not changed:
# return changed
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def set_prompt(self, prompt: Optional[str]):
"""Set the transcription/translation prompt and reconnect.
.. deprecated::
Use ``STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...))`` instead.
Args:
prompt: Prompt text to guide transcription/translation style/context.
Pass None to clear/disable prompt.
Only applicable to models that support prompts.
"""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
f"{self.__class__.__name__}.set_prompt() is deprecated. "
"Use STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...)) instead.",
DeprecationWarning,
stacklevel=2,
)
if not self._config.supports_prompt:
if prompt is not None:
raise ValueError(f"Model '{self.model_name}' does not support prompt parameter.")
raise ValueError(
f"Model '{self._settings.model}' does not support prompt parameter."
)
# If prompt is None and model doesn't support prompts, silently return (no-op)
return
logger.info(f"Updating {self.model_name} prompt.")
self._prompt = prompt
logger.info(f"Updating {self._settings.model} prompt.")
self._settings.prompt = prompt
await self._disconnect()
await self._connect()
@@ -422,35 +487,40 @@ class SarvamSTTService(STTService):
try:
# Build common connection parameters
connect_kwargs = {
"model": self.model_name,
"model": self._settings.model,
"sample_rate": str(self.sample_rate),
}
# Enable flush signal when using Pipecat's VAD (not Sarvam's) so that
# the flush() call on user-stopped-speaking is honored by the server.
if not self._vad_signals:
if not self._settings.vad_signals:
connect_kwargs["flush_signal"] = "true"
# Only send vad parameters when explicitly set (avoid overriding server defaults)
if self._vad_signals is not None:
connect_kwargs["vad_signals"] = "true" if self._vad_signals else "false"
if self._high_vad_sensitivity is not None:
if self._settings.vad_signals is not None:
connect_kwargs["vad_signals"] = "true" if self._settings.vad_signals else "false"
if self._settings.high_vad_sensitivity is not None:
connect_kwargs["high_vad_sensitivity"] = (
"true" if self._high_vad_sensitivity else "false"
"true" if self._settings.high_vad_sensitivity else "false"
)
# Add language_code for models that support it
if self._language_string is not None:
connect_kwargs["language_code"] = self._language_string
language_string = self._get_language_string()
if language_string is not None:
connect_kwargs["language_code"] = language_string
# Add mode for models that support it
if self._config.supports_mode and self._mode is not None:
connect_kwargs["mode"] = self._mode
if self._config.supports_mode and is_given(self._settings.mode):
connect_kwargs["mode"] = self._settings.mode
# Prompt support differs across sarvamai versions. Prefer connect-time prompt
# when available and gracefully degrade if the SDK doesn't accept it.
if self._prompt is not None and self._config.supports_prompt:
connect_kwargs["prompt"] = self._prompt
if (
is_given(self._settings.prompt)
and self._settings.prompt is not None
and self._config.supports_prompt
):
connect_kwargs["prompt"] = self._settings.prompt
def _connect_with_sdk_headers(connect_fn, **kwargs):
# Different SDK versions may use different kwarg names.
@@ -491,10 +561,14 @@ class SarvamSTTService(STTService):
self._socket_client = await self._websocket_context.__aenter__()
# Fallback for SDKs that support runtime prompt updates.
if self._prompt is not None and self._config.supports_prompt:
if (
is_given(self._settings.prompt)
and self._settings.prompt is not None
and self._config.supports_prompt
):
prompt_setter = getattr(self._socket_client, "set_prompt", None)
if callable(prompt_setter):
await prompt_setter(self._prompt)
await prompt_setter(self._settings.prompt)
# Register event handler for incoming messages
def _message_handler(message):
@@ -592,10 +666,12 @@ class SarvamSTTService(STTService):
# Prefer language from message (auto-detected for translate models). Fallback to configured.
if language_code:
language = self._map_language_code_to_enum(language_code)
elif self._language_string:
language = self._map_language_code_to_enum(self._language_string)
else:
language = Language.HI_IN
language_string = self._get_language_string()
if language_string:
language = self._map_language_code_to_enum(language_string)
else:
language = Language.HI_IN
# Emit utterance end event
await self._call_event_handler("on_utterance_end")

210
src/pipecat/services/sarvam/tts.py
View File

@@ -40,9 +40,9 @@ See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for full API
import asyncio
import base64
import json
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
import aiohttp
from loguru import logger
@@ -62,6 +62,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.sarvam._sdk import sdk_headers
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -244,6 +245,78 @@ def language_to_sarvam_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class SarvamHttpTTSSettings(TTSSettings):
"""Settings for Sarvam HTTP TTS service.
Parameters:
language: Sarvam language code.
enable_preprocessing: Whether to enable text preprocessing. Defaults to False.
**Note:** Always enabled for bulbul:v3-beta (cannot be disabled).
pace: Speech pace multiplier. Defaults to 1.0.
- bulbul:v2: Range 0.3 to 3.0
- bulbul:v3-beta: Range 0.5 to 2.0
pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0).
Lower values = more deterministic, higher = more random. Defaults to 0.6.
**Note:** Only supported for bulbul:v3-beta. Ignored for v2.
sample_rate: Audio sample rate.
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sarvam_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class SarvamTTSSettings(TTSSettings):
"""Settings for Sarvam WebSocket TTS service.
Parameters:
target_language_code: Sarvam language code.
speech_sample_rate: Audio sample rate as string.
enable_preprocessing: Enable text preprocessing. Defaults to False.
**Note:** Always enabled for bulbul:v3-beta.
min_buffer_size: Minimum characters to buffer before generating audio.
Lower values reduce latency but may affect quality. Defaults to 50.
max_chunk_length: Maximum characters processed in a single chunk.
Controls memory usage and processing efficiency. Defaults to 150.
output_audio_codec: Audio codec format. Options: linear16, mulaw, alaw,
opus, flac, aac, wav, mp3. Defaults to "linear16".
output_audio_bitrate: Audio bitrate (32k, 64k, 96k, 128k, 192k).
Defaults to "128k".
pace: Speech pace multiplier. Defaults to 1.0.
- bulbul:v2: Range 0.3 to 3.0
- bulbul:v3-beta: Range 0.5 to 2.0
pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0).
Lower = more deterministic, higher = more random. Defaults to 0.6.
**Note:** Only supported for bulbul:v3-beta. Ignored for v2.
"""
target_language_code: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speech_sample_rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_buffer_size: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_chunk_length: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_audio_codec: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_audio_bitrate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SarvamHttpTTSService(TTSService):
"""Text-to-Speech service using Sarvam AI's API.
@@ -296,6 +369,8 @@ class SarvamHttpTTSService(TTSService):
)
"""
_settings: SarvamHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for Sarvam TTS configuration.
@@ -383,14 +458,14 @@ class SarvamHttpTTSService(TTSService):
if sample_rate is None:
sample_rate = self._config.default_sample_rate
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or SarvamHttpTTSService.InputParams()
# Set default voice based on model if not specified
if voice_id is None:
voice_id = self._config.default_speaker
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
@@ -403,36 +478,35 @@ class SarvamHttpTTSService(TTSService):
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = {
"language": (
self._settings = SarvamHttpTTSSettings(
language=(
self.language_to_service_language(params.language) if params.language else "en-IN"
),
"enable_preprocessing": (
enable_preprocessing=(
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
"pace": pace,
"model": model,
}
pace=pace,
model=model,
voice=voice_id,
)
self._sync_model_name_to_metrics()
# Add parameters based on model support
if self._config.supports_pitch:
self._settings["pitch"] = params.pitch
self._settings.pitch = params.pitch
elif params.pitch != 0.0:
logger.warning(f"pitch parameter is ignored for {model}")
if self._config.supports_loudness:
self._settings["loudness"] = params.loudness
self._settings.loudness = params.loudness
elif params.loudness != 1.0:
logger.warning(f"loudness parameter is ignored for {model}")
if self._config.supports_temperature:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
elif params.temperature != 0.6:
logger.warning(f"temperature parameter is ignored for {model}")
self.set_model_name(model)
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -459,7 +533,7 @@ class SarvamHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.sarvam_sample_rate = self.sample_rate
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -480,21 +554,25 @@ class SarvamHttpTTSService(TTSService):
# Build payload with common parameters
payload = {
"text": text,
"target_language_code": self._settings["language"],
"speaker": self._voice_id,
"target_language_code": self._settings.language,
"speaker": self._settings.voice,
"sample_rate": self.sample_rate,
"enable_preprocessing": self._settings["enable_preprocessing"],
"model": self._model_name,
"pace": self._settings.get("pace", 1.0),
"enable_preprocessing": self._settings.enable_preprocessing,
"model": self._settings.model,
"pace": self._settings.pace if is_given(self._settings.pace) else 1.0,
}
# Add model-specific parameters based on config
if self._config.supports_pitch:
payload["pitch"] = self._settings.get("pitch", 0.0)
payload["pitch"] = self._settings.pitch if is_given(self._settings.pitch) else 0.0
if self._config.supports_loudness:
payload["loudness"] = self._settings.get("loudness", 1.0)
payload["loudness"] = (
self._settings.loudness if is_given(self._settings.loudness) else 1.0
)
if self._config.supports_temperature:
payload["temperature"] = self._settings.get("temperature", 0.6)
payload["temperature"] = (
self._settings.temperature if is_given(self._settings.temperature) else 0.6
)
headers = {
"api-subscription-key": self._api_key,
@@ -605,6 +683,8 @@ class SarvamTTSService(InterruptibleTTSService):
See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for API details.
"""
_settings: SarvamTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Sarvam TTS WebSocket service.
@@ -729,6 +809,10 @@ class SarvamTTSService(InterruptibleTTSService):
if sample_rate is None:
sample_rate = self._config.default_sample_rate
# Set default voice based on model if not specified
if voice_id is None:
voice_id = self._config.default_speaker
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
@@ -740,15 +824,9 @@ class SarvamTTSService(InterruptibleTTSService):
)
params = params or SarvamTTSService.InputParams()
# Set default voice based on model if not specified
if voice_id is None:
voice_id = self._config.default_speaker
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
self.set_model_name(model)
self.set_voice(voice_id)
# Validate and clamp pace to model's valid range
pace = params.pace
@@ -758,36 +836,37 @@ class SarvamTTSService(InterruptibleTTSService):
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = {
"target_language_code": (
self._settings = SarvamTTSSettings(
target_language_code=(
self.language_to_service_language(params.language) if params.language else "en-IN"
),
"speaker": voice_id,
"speech_sample_rate": str(sample_rate),
"enable_preprocessing": (
speech_sample_rate=str(sample_rate),
enable_preprocessing=(
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
"min_buffer_size": params.min_buffer_size,
"max_chunk_length": params.max_chunk_length,
"output_audio_codec": params.output_audio_codec,
"output_audio_bitrate": params.output_audio_bitrate,
"pace": pace,
"model": model,
}
min_buffer_size=params.min_buffer_size,
max_chunk_length=params.max_chunk_length,
output_audio_codec=params.output_audio_codec,
output_audio_bitrate=params.output_audio_bitrate,
pace=pace,
model=model,
voice=voice_id,
)
self._sync_model_name_to_metrics()
# Add parameters based on model support
if self._config.supports_pitch:
self._settings["pitch"] = params.pitch
self._settings.pitch = params.pitch
elif params.pitch != 0.0:
logger.warning(f"pitch parameter is ignored for {model}")
if self._config.supports_loudness:
self._settings["loudness"] = params.loudness
self._settings.loudness = params.loudness
elif params.loudness != 1.0:
logger.warning(f"loudness parameter is ignored for {model}")
if self._config.supports_temperature:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
elif params.temperature != 0.6:
logger.warning(f"temperature parameter is ignored for {model}")
@@ -823,7 +902,7 @@ class SarvamTTSService(InterruptibleTTSService):
await super().start(frame)
# WebSocket API expects sample rate as string
self._settings["speech_sample_rate"] = str(self.sample_rate)
self._settings.speech_sample_rate = str(self.sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -870,14 +949,15 @@ class SarvamTTSService(InterruptibleTTSService):
if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
await self.flush_audio()
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a settings update and resend config if voice changed."""
changed = await super()._update_settings(update)
if changed:
await self._send_config()
return changed
async def _connect(self):
"""Connect to Sarvam WebSocket and start background tasks."""
await super()._connect()
@@ -934,9 +1014,27 @@ class SarvamTTSService(InterruptibleTTSService):
"""Send initial configuration message."""
if not self._websocket:
raise Exception("WebSocket not connected")
self._settings["speaker"] = self._voice_id
logger.debug(f"Config being sent is {self._settings}")
config_message = {"type": "config", "data": self._settings}
# Build config dict for the API
config_data = {
"target_language_code": self._settings.target_language_code,
"speaker": self._settings.voice,
"speech_sample_rate": self._settings.speech_sample_rate,
"enable_preprocessing": self._settings.enable_preprocessing,
"min_buffer_size": self._settings.min_buffer_size,
"max_chunk_length": self._settings.max_chunk_length,
"output_audio_codec": self._settings.output_audio_codec,
"output_audio_bitrate": self._settings.output_audio_bitrate,
"pace": self._settings.pace,
"model": self._settings.model,
}
if is_given(self._settings.pitch):
config_data["pitch"] = self._settings.pitch
if is_given(self._settings.loudness):
config_data["loudness"] = self._settings.loudness
if is_given(self._settings.temperature):
config_data["temperature"] = self._settings.temperature
logger.debug(f"Config being sent is {config_data}")
config_message = {"type": "config", "data": config_data}
try:
await self._websocket.send(json.dumps(config_message))

333
src/pipecat/services/settings.py Normal file
View File

@@ -0,0 +1,333 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Settings infrastructure for Pipecat AI services.
This module provides dataclass-based settings objects for service configuration.
Each service type has a corresponding settings class (e.g. ``TTSSettings``,
``LLMSettings``) whose fields use the ``NOT_GIVEN`` sentinel to distinguish
"leave unchanged" from an explicit ``None``.
Key concepts:
- **NOT_GIVEN sentinel**: A value meaning "this field was not provided in the
update". Distinct from ``None`` (which may be a valid value for a setting).
- **Settings as both state and delta**: The same class is used for the
service's current settings *and* for update objects. Fields set to
``NOT_GIVEN`` are simply skipped when applying an update.
- **apply_update**: Applies a delta onto a target settings object and returns
a dict mapping each changed field name to its previous value.
- **from_mapping**: Constructs a settings object from a plain dict,
supporting field aliases (e.g. ``"voice_id"`` → ``"voice"``).
- **Extras**: Unknown keys land in the ``extra`` dict so services that have
non-standard settings don't lose data.
"""
from __future__ import annotations
import copy
from dataclasses import dataclass, field, fields
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Mapping, Optional, Type, TypeVar
from loguru import logger
from pipecat.transcriptions.language import Language
if TYPE_CHECKING:
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionConfig
# ---------------------------------------------------------------------------
# NOT_GIVEN sentinel
# ---------------------------------------------------------------------------
class _NotGiven:
"""Sentinel indicating a settings field was not provided.
``NOT_GIVEN`` means "the caller did not supply this value" — distinct from
``None``, which may be a legitimate setting value. It is used as the
default for every settings field so that ``apply_update`` can tell which
fields the caller actually wants to change.
"""
_instance: Optional[_NotGiven] = None
def __new__(cls) -> _NotGiven:
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __repr__(self) -> str:
return "NOT_GIVEN"
def __bool__(self) -> bool:
return False
NOT_GIVEN: _NotGiven = _NotGiven()
"""Singleton sentinel meaning "this field was not included in the update"."""
def is_given(value: Any) -> bool:
"""Check whether a value was explicitly provided (i.e. is not ``NOT_GIVEN``).
Args:
value: The value to check.
Returns:
``True`` if *value* is anything other than ``NOT_GIVEN``.
"""
return not isinstance(value, _NotGiven)
# ---------------------------------------------------------------------------
# Base ServiceSettings
# ---------------------------------------------------------------------------
_S = TypeVar("_S", bound="ServiceSettings")
@dataclass
class ServiceSettings:
"""Base class for runtime-updatable service settings.
These settings represent the subset of a service's configuration that can
be changed **while the pipeline is running** (e.g. switching the model or
changing the voice). They are *not* meant to capture every constructor
parameter — only those that support live updates via
``*UpdateSettingsFrame``.
Every AI service type (LLM, TTS, STT) extends this with its own fields.
Fields default to ``NOT_GIVEN`` so that an instance can represent either
the full current state **or** a sparse update delta. Note that in the full
current state, **all fields will be given** (i.e. ``NOT_GIVEN`` is reserved
for update deltas).
Parameters:
model: The model identifier used by the service.
extra: Overflow dict for service-specific keys that don't map to a
declared field.
"""
# -- common fields -------------------------------------------------------
model: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
"""AI model identifier (e.g. ``"gpt-4o"``, ``"eleven_turbo_v2_5"``)."""
extra: Dict[str, Any] = field(default_factory=dict)
"""Catch-all for service-specific keys that have no declared field."""
# -- class-level configuration -------------------------------------------
_aliases: ClassVar[Dict[str, str]] = {}
"""Map of alternative key names to canonical field names.
For example ``{"voice_id": "voice"}`` lets callers use either spelling.
Subclasses should override this as needed.
"""
# -- public API ----------------------------------------------------------
def given_fields(self) -> Dict[str, Any]:
"""Return a dict of only the fields that were explicitly provided.
Skips ``NOT_GIVEN`` values and the ``extra`` field itself. Entries
from ``extra`` are included at the top level.
Returns:
Dictionary mapping field names to their provided values.
"""
result: Dict[str, Any] = {}
for f in fields(self):
if f.name == "extra":
continue
val = getattr(self, f.name)
if is_given(val):
result[f.name] = val
result.update(self.extra)
return result
def apply_update(self: _S, update: _S) -> Dict[str, Any]:
"""Apply *update* onto this settings object, returning changed fields.
Only fields in *update* that are **given** (i.e. not ``NOT_GIVEN``)
are considered. A field is "changed" if its new value differs from
the current value.
The ``extra`` dicts are merged: keys present in the update overwrite
keys in the target.
Args:
update: A settings object of the same type containing the delta.
Returns:
A dict mapping each changed field name to its **pre-update** value.
Use ``changed.keys()`` for the set of names, or index with
``changed["field"]`` to inspect the old value.
Examples::
current = TTSSettings(voice="alice", language="en")
delta = TTSSettings(voice="bob")
changed = current.apply_update(delta)
# changed == {"voice": "alice"}
# current.voice == "bob", current.language == "en"
"""
changed: Dict[str, Any] = {}
for f in fields(self):
if f.name == "extra":
continue
new_val = getattr(update, f.name)
if not is_given(new_val):
continue
old_val = getattr(self, f.name)
if old_val != new_val:
setattr(self, f.name, new_val)
changed[f.name] = old_val
# Merge extra
for key, new_val in update.extra.items():
old_val = self.extra.get(key, NOT_GIVEN)
if old_val != new_val:
self.extra[key] = new_val
changed[key] = old_val
return changed
@classmethod
def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S:
"""Construct a settings object from a plain dictionary.
This exists for backward compatibility with code that passes plain
dicts via ``*UpdateSettingsFrame(settings={...})``.
Keys are matched to dataclass fields by name. Keys listed in
``_aliases`` are translated to their canonical name first. Any
remaining unrecognized keys are placed into ``extra``.
Args:
settings: A dictionary of setting names to values.
Returns:
A new settings instance with the corresponding fields populated.
Examples::
update = TTSSettings.from_mapping({"voice_id": "alice", "speed": 1.2})
# update.voice == "alice" (via alias)
# update.extra == {"speed": 1.2}
"""
field_names = {f.name for f in fields(cls)} - {"extra"}
kwargs: Dict[str, Any] = {}
extra: Dict[str, Any] = {}
for key, value in settings.items():
# Resolve aliases first
canonical = cls._aliases.get(key, key)
if canonical in field_names:
kwargs[canonical] = value
else:
extra[key] = value
instance = cls(**kwargs)
instance.extra = extra
return instance
def copy(self: _S) -> _S:
"""Return a deep copy of this settings instance.
Returns:
A new settings object with the same field values.
"""
return copy.deepcopy(self)
# ---------------------------------------------------------------------------
# Service-specific settings
# ---------------------------------------------------------------------------
@dataclass
class LLMSettings(ServiceSettings):
"""Runtime-updatable settings for LLM services.
See ``ServiceSettings`` for the general concept.
Parameters:
model: LLM model identifier.
temperature: Sampling temperature.
max_tokens: Maximum tokens to generate.
top_p: Nucleus sampling probability.
top_k: Top-k sampling parameter.
frequency_penalty: Frequency penalty.
presence_penalty: Presence penalty.
seed: Random seed for reproducibility.
filter_incomplete_user_turns: Enable LLM-based turn completion detection
to suppress bot responses when the user was cut off mid-thought.
See ``examples/foundational/22-filter-incomplete-turns.py`` and
``UserTurnCompletionLLMServiceMixin``.
user_turn_completion_config: Configuration for turn completion behavior
when ``filter_incomplete_user_turns`` is enabled. Controls timeouts
and prompts for incomplete turns.
"""
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_tokens: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_k: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
frequency_penalty: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
presence_penalty: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
seed: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
filter_incomplete_user_turns: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
user_turn_completion_config: UserTurnCompletionConfig | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
@dataclass
class TTSSettings(ServiceSettings):
"""Runtime-updatable settings for TTS services.
See ``ServiceSettings`` for the general concept.
Parameters:
model: TTS model identifier.
voice: Voice identifier or name.
language: Language for speech synthesis. The union type reflects the
*input* side: callers may pass a ``Language`` enum or a raw string.
However, the **stored** value is always a service-specific string
— ``TTSService._update_settings`` converts ``Language`` enums via
``language_to_service_language()`` before writing, and ``__init__``
methods do the same at construction time. Code that reads
``self._settings.language`` after initialisation can treat it as
``str``.
"""
voice: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@dataclass
class STTSettings(ServiceSettings):
"""Runtime-updatable settings for STT services.
See ``ServiceSettings`` for the general concept.
Parameters:
model: STT model identifier.
language: Language for speech recognition. The union type reflects the
*input* side: callers may pass a ``Language`` enum or a raw string.
However, the **stored** value is always a service-specific string
— ``STTService._update_settings`` converts ``Language`` enums via
``language_to_service_language()`` before writing, and ``__init__``
methods do the same at construction time. Code that reads
``self._settings.language`` after initialisation can treat it as
``str``.
"""
language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)

86
src/pipecat/services/soniox/stt.py
View File

@@ -8,7 +8,8 @@
import json
import time
from typing import AsyncGenerator, List, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, List, Optional
from loguru import logger
from pydantic import BaseModel
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import SONIOX_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -134,6 +136,17 @@ def _prepare_language_hints(
return list(set(prepared_languages))
@dataclass
class SonioxSTTSettings(STTSettings):
"""Settings for Soniox STT service.
Parameters:
input_params: Soniox ``SonioxInputParams`` for detailed configuration.
"""
input_params: SonioxInputParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SonioxSTTService(WebsocketSTTService):
"""Speech-to-Text service using Soniox's WebSocket API.
@@ -144,6 +157,8 @@ class SonioxSTTService(WebsocketSTTService):
For complete API documentation, see: https://soniox.com/docs/speech-to-text/api-reference/websocket-api
"""
_settings: SonioxSTTSettings
def __init__(
self,
*,
@@ -180,10 +195,14 @@ class SonioxSTTService(WebsocketSTTService):
self._api_key = api_key
self._url = url
self.set_model_name(params.model)
self._params = params
self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._settings = SonioxSTTSettings(
model=params.model,
input_params=params,
)
self._sync_model_name_to_metrics()
self._final_transcription_buffer = []
self._last_tokens_received: Optional[float] = None
@@ -198,6 +217,47 @@ class SonioxSTTService(WebsocketSTTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, update: SonioxSTTSettings) -> dict[str, Any]:
"""Apply a settings update, keeping ``input_params`` in sync.
Top-level ``model`` is the source of truth. When it is given in
*update* its value is propagated into ``input_params``. When only
``input_params`` is given, its ``model`` is propagated *up* to the
top-level field.
Settings are stored but not applied to the active connection.
Args:
update: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
model_given = is_given(getattr(update, "model", NOT_GIVEN))
changed = await super()._update_settings(update)
if not changed:
return changed
# --- Sync model --------------------------------------------------
if model_given:
# Top-level model wins → push into input_params.
self._settings.input_params.model = self._settings.model
elif "input_params" in changed and self._settings.input_params.model is not None:
# Only input_params was given → pull model up.
self._settings.model = self._settings.input_params.model
self._sync_model_name_to_metrics()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def stop(self, frame: EndFrame):
"""Stop the Soniox STT websocket connection.
@@ -311,24 +371,26 @@ class SonioxSTTService(WebsocketSTTService):
# Either one or the other is required.
enable_endpoint_detection = not self._vad_force_turn_endpoint
context = self._params.context
params = self._settings.input_params
context = params.context
if isinstance(context, SonioxContextObject):
context = context.model_dump()
# Send the initial configuration message.
config = {
"api_key": self._api_key,
"model": self._model_name,
"audio_format": self._params.audio_format,
"num_channels": self._params.num_channels or 1,
"model": self._settings.model,
"audio_format": params.audio_format,
"num_channels": params.num_channels or 1,
"enable_endpoint_detection": enable_endpoint_detection,
"sample_rate": self.sample_rate,
"language_hints": _prepare_language_hints(self._params.language_hints),
"language_hints_strict": self._params.language_hints_strict,
"language_hints": _prepare_language_hints(params.language_hints),
"language_hints_strict": params.language_hints_strict,
"context": context,
"enable_speaker_diarization": self._params.enable_speaker_diarization,
"enable_language_identification": self._params.enable_language_identification,
"client_reference_id": self._params.client_reference_id,
"enable_speaker_diarization": params.enable_speaker_diarization,
"enable_language_identification": params.enable_language_identification,
"client_reference_id": params.client_reference_id,
}
# Send the configuration message.

259
src/pipecat/services/speechmatics/stt.py
View File

@@ -8,8 +8,10 @@
import asyncio
import os
import warnings
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator
from typing import Any, AsyncGenerator, ClassVar
from dotenv import load_dotenv
from loguru import logger
@@ -31,6 +33,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import SPEECHMATICS_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -80,6 +83,83 @@ class TurnDetectionMode(str, Enum):
SMART_TURN = "smart_turn"
@dataclass
class SpeechmaticsSTTSettings(STTSettings):
"""Settings for Speechmatics STT service.
See ``SpeechmaticsSTTService.InputParams`` for detailed descriptions of each field.
Parameters:
model: The operating point / model name.
domain: Domain for Speechmatics API.
turn_detection_mode: Endpoint handling mode.
speaker_active_format: Formatter for active speaker ID.
speaker_passive_format: Formatter for passive speaker ID.
focus_speakers: List of speaker IDs to focus on.
ignore_speakers: List of speaker IDs to ignore.
focus_mode: Speaker focus mode for diarization.
known_speakers: List of known speaker labels and identifiers.
additional_vocab: List of additional vocabulary entries.
audio_encoding: Audio encoding format.
operating_point: Operating point for accuracy vs. latency.
max_delay: Maximum delay in seconds for transcription.
end_of_utterance_silence_trigger: Maximum delay for end of utterance trigger.
end_of_utterance_max_delay: Maximum delay for end of utterance.
punctuation_overrides: Punctuation overrides.
include_partials: Include partial segment fragments.
split_sentences: Emit finalized sentences mid-turn.
enable_diarization: Enable speaker diarization.
speaker_sensitivity: Diarization sensitivity.
max_speakers: Maximum number of speakers to detect.
prefer_current_speaker: Prefer current speaker ID.
extra_params: Extra parameters for the STT engine.
"""
domain: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
turn_detection_mode: TurnDetectionMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_active_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_passive_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
focus_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
ignore_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
focus_mode: SpeakerFocusMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
known_speakers: list[SpeakerIdentifier] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
additional_vocab: list[AdditionalVocabEntry] | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
audio_encoding: AudioEncoding | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
operating_point: OperatingPoint | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
end_of_utterance_silence_trigger: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
end_of_utterance_max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
punctuation_overrides: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_partials: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
split_sentences: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_diarization: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_sensitivity: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_speakers: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prefer_current_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
extra_params: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
#: Fields that can be updated on a live connection via the Speechmatics
#: diarization-config API — no reconnect needed.
HOT_FIELDS: ClassVar[frozenset[str]] = frozenset(
{
"focus_speakers",
"ignore_speakers",
"focus_mode",
}
)
#: Fields that are purely local (formatting templates) — no reconnect
#: and no API call needed.
LOCAL_FIELDS: ClassVar[frozenset[str]] = frozenset(
{
"speaker_active_format",
"speaker_passive_format",
}
)
class SpeechmaticsSTTService(STTService):
"""Speechmatics STT service implementation.
@@ -98,6 +178,8 @@ class SpeechmaticsSTTService(STTService):
...
"""
_settings: SpeechmaticsSTTSettings
# Export related classes as class attributes
TurnDetectionMode = TurnDetectionMode
AudioEncoding = AudioEncoding
@@ -337,31 +419,57 @@ class SpeechmaticsSTTService(STTService):
# Deprecation check
self._check_deprecated_args(kwargs, params)
# Voice agent
# Output formatting defaults
speaker_active_format = params.speaker_active_format
if speaker_active_format is None:
speaker_active_format = (
"@{speaker_id}: {text}" if params.enable_diarization else "{text}"
)
speaker_passive_format = params.speaker_passive_format or speaker_active_format
# Settings — seeded from InputParams
self._settings = SpeechmaticsSTTSettings(
language=params.language,
domain=params.domain,
turn_detection_mode=params.turn_detection_mode,
speaker_active_format=speaker_active_format,
speaker_passive_format=speaker_passive_format,
focus_speakers=params.focus_speakers,
ignore_speakers=params.ignore_speakers,
focus_mode=params.focus_mode,
known_speakers=params.known_speakers,
additional_vocab=params.additional_vocab,
audio_encoding=params.audio_encoding,
operating_point=params.operating_point,
max_delay=params.max_delay,
end_of_utterance_silence_trigger=params.end_of_utterance_silence_trigger,
end_of_utterance_max_delay=params.end_of_utterance_max_delay,
punctuation_overrides=params.punctuation_overrides,
include_partials=params.include_partials,
split_sentences=params.split_sentences,
enable_diarization=params.enable_diarization,
speaker_sensitivity=params.speaker_sensitivity,
max_speakers=params.max_speakers,
prefer_current_speaker=params.prefer_current_speaker,
extra_params=params.extra_params,
)
# Build SDK config from settings
self._client: VoiceAgentClient | None = None
self._config: VoiceAgentConfig = self._prepare_config(params)
self._config: VoiceAgentConfig = self._build_config()
# Outbound frame queue
self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue()
# Output formatting
if params.speaker_active_format is None:
params.speaker_active_format = (
"@{speaker_id}: {text}" if params.enable_diarization else "{text}"
)
# Framework options
self._enable_vad: bool = self._config.end_of_utterance_mode not in [
EndOfUtteranceMode.FIXED,
EndOfUtteranceMode.EXTERNAL,
]
self._speaker_active_format: str = params.speaker_active_format
self._speaker_passive_format: str = (
params.speaker_passive_format or params.speaker_active_format
)
# Model + metrics
self.set_model_name(self._config.operating_point.value)
# Model + metrics (operating_point comes from the SDK config/preset)
self._settings.model = self._config.operating_point.value
self._sync_model_name_to_metrics()
# Message queue
self._stt_msg_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
@@ -384,6 +492,64 @@ class SpeechmaticsSTTService(STTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, update: SpeechmaticsSTTSettings) -> dict[str, Any]:
"""Apply settings update, reconnecting only when necessary.
Fields are classified into three categories (see
``SpeechmaticsSTTSettings``):
* **HOT_FIELDS** – diarization speaker settings that can be pushed
to a live Speechmatics connection without reconnecting.
* **LOCAL_FIELDS** – formatting templates evaluated locally; no
reconnect or API call needed.
* Everything else – baked into ``VoiceAgentConfig`` at connection
time and therefore require a full disconnect / reconnect.
Args:
update: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(update)
if not changed:
return changed
no_reconnect = SpeechmaticsSTTSettings.HOT_FIELDS | SpeechmaticsSTTSettings.LOCAL_FIELDS
needs_reconnect = bool(changed.keys() - no_reconnect)
if needs_reconnect:
logger.debug(f"{self} settings update requires reconnect: {changed.keys()}")
# Connection-level fields changed — rebuild the SDK config
# from the now-updated self._settings, then reconnect.
self._config = self._build_config()
await self._disconnect()
await self._connect()
elif changed.keys() & SpeechmaticsSTTSettings.HOT_FIELDS:
logger.debug(f"{self} applying hot settings update: {changed.keys()}")
if self._config.enable_diarization:
# Only hot-updatable fields changed — push to the live session.
self._config.speaker_config.focus_speakers = self._settings.focus_speakers
self._config.speaker_config.ignore_speakers = self._settings.ignore_speakers
self._config.speaker_config.focus_mode = self._settings.focus_mode
if self._client:
self._client.update_diarization_config(self._config.speaker_config)
else:
logger.debug(
f"{self} hot settings updated but diarization not enabled: {changed.keys()}. ignoring."
)
# Diarization not enabled — the new settings will take effect
# if/when diarization is enabled, which does require a reconnect.
elif changed.keys() & SpeechmaticsSTTSettings.LOCAL_FIELDS:
logger.debug(
f"{self} local settings update, no special action required: {changed.keys()}"
)
# Only local fields changed — no need to push to the STT engine,
# the new settings will take effect immediately.
return changed
async def stop(self, frame: EndFrame):
"""Called when the session ends."""
await super().stop(frame)
@@ -494,28 +660,35 @@ class SpeechmaticsSTTService(STTService):
# CONFIGURATION
# ============================================================================
def _prepare_config(self, params: InputParams) -> VoiceAgentConfig:
"""Parse the InputParams into VoiceAgentConfig."""
# Preset
config = VoiceAgentConfigPreset.load(params.turn_detection_mode.value)
def _build_config(self) -> VoiceAgentConfig:
"""Build a ``VoiceAgentConfig`` from the current ``self._settings``.
Used both at init time and before reconnecting so the connection
always reflects the latest settings.
"""
s = self._settings
# Preset from turn detection mode
config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value)
# Language + domain
config.language = self._language_to_speechmatics_language(params.language)
config.domain = params.domain
config.output_locale = self._locale_to_speechmatics_locale(config.language, params.language)
language = s.language
config.language = self._language_to_speechmatics_language(language)
config.domain = s.domain if is_given(s.domain) else None
config.output_locale = self._locale_to_speechmatics_locale(config.language, language)
# Speaker config
config.speaker_config = SpeakerFocusConfig(
focus_speakers=params.focus_speakers,
ignore_speakers=params.ignore_speakers,
focus_mode=params.focus_mode,
focus_speakers=s.focus_speakers if is_given(s.focus_speakers) else [],
ignore_speakers=s.ignore_speakers if is_given(s.ignore_speakers) else [],
focus_mode=s.focus_mode if is_given(s.focus_mode) else SpeakerFocusMode.RETAIN,
)
config.known_speakers = params.known_speakers
config.known_speakers = s.known_speakers if is_given(s.known_speakers) else []
# Custom dictionary
config.additional_vocab = params.additional_vocab
config.additional_vocab = s.additional_vocab if is_given(s.additional_vocab) else []
# Advanced parameters
# Advanced parameters — only set if given (not NOT_GIVEN or None)
for param in [
"operating_point",
"max_delay",
@@ -529,21 +702,20 @@ class SpeechmaticsSTTService(STTService):
"max_speakers",
"prefer_current_speaker",
]:
if getattr(params, param) is not None:
setattr(config, param, getattr(params, param))
val = getattr(s, param)
if is_given(val) and val is not None:
setattr(config, param, val)
# Extra parameters
if isinstance(params.extra_params, dict):
for key, value in params.extra_params.items():
if is_given(s.extra_params) and isinstance(s.extra_params, dict):
for key, value in s.extra_params.items():
if hasattr(config, key):
setattr(config, key, value)
# Enable sentences
config.speech_segment_config = SpeechSegmentConfig(
emit_sentences=params.split_sentences or False
)
split = s.split_sentences if is_given(s.split_sentences) else False
config.speech_segment_config = SpeechSegmentConfig(emit_sentences=split or False)
# Return the complete config
return config
def update_params(
@@ -552,12 +724,23 @@ class SpeechmaticsSTTService(STTService):
) -> None:
"""Updates the speaker configuration.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with
``SpeechmaticsSTTSettings(...)`` instead.
This can update the speakers to listen to or ignore during an in-flight
transcription. Only available if diarization is enabled.
Args:
params: Update parameters for the service.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"update_params() is deprecated. Use STTUpdateSettingsFrame with "
"SpeechmaticsSTTSettings(...) instead.",
DeprecationWarning,
)
# Check possible
if not self._config.enable_diarization:
raise ValueError("Diarization is not enabled")
@@ -727,9 +910,9 @@ class SpeechmaticsSTTService(STTService):
def attr_from_segment(segment: dict[str, Any]) -> dict[str, Any]:
# Formats the output text based on the speaker and defined formats from the config.
text = (
self._speaker_active_format
self._settings.speaker_active_format
if segment.get("is_active", True)
else self._speaker_passive_format
else self._settings.speaker_passive_format
).format(
**{
"speaker_id": segment.get("speaker_id", "UU"),

31
src/pipecat/services/speechmatics/tts.py
View File

@@ -7,7 +7,8 @@
"""Speechmatics TTS service integration."""
import asyncio
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from urllib.parse import urlencode
import aiohttp
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.network import exponential_backoff_time
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -35,6 +37,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class SpeechmaticsTTSSettings(TTSSettings):
"""Settings for Speechmatics TTS service.
Parameters:
max_retries: Maximum number of retries for HTTP requests.
"""
max_retries: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SpeechmaticsTTSService(TTSService):
"""Speechmatics TTS service implementation.
@@ -42,6 +55,8 @@ class SpeechmaticsTTSService(TTSService):
It converts text to speech and returns raw PCM audio data for real-time playback.
"""
_settings: SpeechmaticsTTSSettings
SPEECHMATICS_SAMPLE_RATE = 16000
class InputParams(BaseModel):
@@ -91,11 +106,11 @@ class SpeechmaticsTTSService(TTSService):
if not self._api_key:
raise ValueError("Missing Speechmatics API key")
# Default parameters
self._params = params or SpeechmaticsTTSService.InputParams()
# Set voice from constructor parameter
self.set_voice(voice_id)
params = params or SpeechmaticsTTSService.InputParams()
self._settings = SpeechmaticsTTSSettings(
voice=voice_id,
max_retries=params.max_retries,
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -131,7 +146,7 @@ class SpeechmaticsTTSService(TTSService):
}
# Complete HTTP URL
url = _get_endpoint_url(self._base_url, self._voice_id, self.sample_rate)
url = _get_endpoint_url(self._base_url, self._settings.voice, self.sample_rate)
try:
# Start TTS TTFB metrics
@@ -159,7 +174,7 @@ class SpeechmaticsTTSService(TTSService):
attempt += 1
# Check if we've exceeded the maximum number of attempts
if attempt >= self._params.max_retries:
if attempt >= self._settings.max_retries:
raise ValueError()
# Report error frame

100
src/pipecat/services/stt_service.py
View File

@@ -9,9 +9,10 @@
import asyncio
import io
import time
import warnings
import wave
from abc import abstractmethod
from typing import Any, AsyncGenerator, Dict, Mapping, Optional
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from websockets.protocol import State
@@ -32,6 +33,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import STTSettings, is_given
from pipecat.services.stt_latency import DEFAULT_TTFS_P99
from pipecat.services.websocket_service import WebsocketService
from pipecat.transcriptions.language import Language
@@ -73,6 +75,8 @@ class STTService(AIService):
logger.error(f"STT connection error: {error}")
"""
_settings: STTSettings
def __init__(
self,
*,
@@ -111,7 +115,8 @@ class STTService(AIService):
self._audio_passthrough = audio_passthrough
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._settings: Dict[str, Any] = {}
self._settings = STTSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._muted: bool = False
self._user_id: str = ""
self._ttfs_p99_latency = ttfs_p99_latency
@@ -179,18 +184,53 @@ class STTService(AIService):
async def set_model(self, model: str):
"""Set the speech recognition model.
.. deprecated:: 0.0.103
Use ``STTUpdateSettingsFrame(model=...)`` instead.
Args:
model: The name of the model to use for speech recognition.
"""
self.set_model_name(model)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated, use 'STTUpdateSettingsFrame(model=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching STT model to: [{model}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(model=model))
async def set_language(self, language: Language):
"""Set the language for speech recognition.
.. deprecated:: 0.0.103
Use ``STTUpdateSettingsFrame(language=...)`` instead.
Args:
language: The language to use for speech recognition.
"""
pass
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_language' is deprecated, use 'STTUpdateSettingsFrame(language=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching STT language to: [{language}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(language=language))
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a language to the service-specific language format.
Args:
language: The language to convert.
Returns:
The service-specific language identifier, or None if not supported.
"""
return Language(language)
@abstractmethod
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
@@ -222,20 +262,29 @@ class STTService(AIService):
await self._cancel_ttfb_timeout()
await self._cancel_keepalive_task()
async def _update_settings(self, settings: Mapping[str, Any]):
logger.info(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating STT setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
await self.set_language(value)
elif key == "language":
await self.set_language(value)
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for STT service: {key}")
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply an STT settings update.
Handles ``model`` (via parent). Translates ``Language`` enum values
before applying so the stored value is a service-specific string.
Concrete services should override this method and handle language
changes (including any reconnect logic) based on the returned
changed-field dict.
Args:
update: An STT settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
# Translate language *before* applying so the stored value is canonical
if is_given(update.language) and isinstance(update.language, Language):
converted = self.language_to_service_language(update.language)
if converted is not None:
update.language = converted
changed = await super()._update_settings(update)
return changed
async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
"""Process an audio frame for speech recognition.
@@ -300,7 +349,20 @@ class STTService(AIService):
await self._handle_vad_user_stopped_speaking(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, STTUpdateSettingsFrame):
await self._update_settings(frame.settings)
if frame.update is not None:
await self._update_settings(frame.update)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via STTUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.103, use STTUpdateSettingsFrame(update=STTSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
update = type(self._settings).from_mapping(frame.settings)
await self._update_settings(update)
elif isinstance(frame, STTMuteFrame):
self._muted = frame.mute
logger.debug(f"STT service {'muted' if frame.mute else 'unmuted'}")

100
src/pipecat/services/tts_service.py
View File

@@ -8,6 +8,7 @@
import asyncio
import uuid
import warnings
from abc import abstractmethod
from dataclasses import dataclass
from typing import (
@@ -18,7 +19,6 @@ from typing import (
Callable,
Dict,
List,
Mapping,
Optional,
Sequence,
Tuple,
@@ -52,6 +52,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import TTSSettings, is_given
from pipecat.services.websocket_service import WebsocketService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
@@ -103,6 +104,8 @@ class TTSService(AIService):
logger.debug(f"TTS request: {context_id} - {text}")
"""
_settings: TTSSettings
def __init__(
self,
*,
@@ -188,8 +191,9 @@ class TTSService(AIService):
self._append_trailing_space: bool = append_trailing_space
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._settings = TTSSettings(
voice=""
) # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
if text_aggregator:
import warnings
@@ -261,18 +265,42 @@ class TTSService(AIService):
async def set_model(self, model: str):
"""Set the TTS model to use.
.. deprecated:: 0.0.103
Use ``TTSUpdateSettingsFrame(model=...)`` instead.
Args:
model: The name of the TTS model.
"""
self.set_model_name(model)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated, use 'TTSUpdateSettingsFrame(model=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching TTS model to: [{model}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(model=model))
def set_voice(self, voice: str):
async def set_voice(self, voice: str):
"""Set the voice for speech synthesis.
.. deprecated:: 0.0.103
Use ``TTSUpdateSettingsFrame(voice=...)`` instead.
Args:
voice: The voice identifier or name.
"""
self._voice_id = voice
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_voice' is deprecated, use 'TTSUpdateSettingsFrame(voice=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching TTS voice to: [{voice}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(voice=voice))
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request.
@@ -324,15 +352,6 @@ class TTSService(AIService):
return text + " "
return text
async def update_setting(self, key: str, value: Any):
"""Update a service-specific setting.
Args:
key: The setting key to update.
value: The new value for the setting.
"""
pass
async def flush_audio(self):
"""Flush any buffered audio data."""
pass
@@ -403,22 +422,26 @@ class TTSService(AIService):
if not (agg_type == aggregation_type and func == transform_function)
]
async def _update_settings(self, settings: Mapping[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating TTS setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
self._settings[key] = self.language_to_service_language(value)
elif key == "model":
self.set_model_name(value)
elif key == "voice" or key == "voice_id":
self.set_voice(value)
elif key == "text_filter":
for filter in self._text_filters:
await filter.update_settings(value)
else:
logger.warning(f"Unknown setting for TTS service: {key}")
async def _update_settings(self, update: TTSSettings) -> dict[str, Any]:
"""Apply a TTS settings update.
Translates language to service-specific value before applying.
Args:
update: A TTS settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
# Translate language *before* applying so the stored value is canonical
if is_given(update.language) and isinstance(update.language, Language):
converted = self.language_to_service_language(update.language)
if converted is not None:
update.language = converted
changed = await super()._update_settings(update)
return changed
async def say(self, text: str):
"""Immediately speak the provided text.
@@ -501,7 +524,20 @@ class TTSService(AIService):
await self.flush_audio()
self._processing_text = processing_text
elif isinstance(frame, TTSUpdateSettingsFrame):
await self._update_settings(frame.settings)
if frame.update is not None:
await self._update_settings(frame.update)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via TTSUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.103, use TTSUpdateSettingsFrame(update=TTSSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
update = type(self._settings).from_mapping(frame.settings)
await self._update_settings(update)
elif isinstance(frame, BotStoppedSpeakingFrame):
await self._maybe_resume_frame_processing()
await self.push_frame(frame, direction)

27
src/pipecat/services/ultravox/llm.py
View File

@@ -15,6 +15,7 @@ import asyncio
import datetime
import json
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, Literal, Optional, Union
import aiohttp
@@ -34,7 +35,6 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
TranscriptionFrame,
TTSAudioRawFrame,
@@ -56,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
try:
@@ -66,6 +67,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class UltravoxRealtimeLLMSettings(LLMSettings):
"""Settings for UltravoxRealtimeLLMService.
Parameters:
output_medium: The output medium for the model ("voice" or "text").
"""
output_medium: str | _NotGiven = field(default=NOT_GIVEN)
class AgentInputParams(BaseModel):
"""Input parameters for Ultravox Realtime generation using a pre-defined Agent.
@@ -146,6 +158,8 @@ class UltravoxRealtimeLLMService(LLMService):
by the model and may not always align with its understanding of user input.
"""
_settings: UltravoxRealtimeLLMSettings
def __init__(
self,
*,
@@ -163,6 +177,7 @@ class UltravoxRealtimeLLMService(LLMService):
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
self._settings = UltravoxRealtimeLLMSettings()
self._params = params
if one_shot_selected_tools:
if not isinstance(self._params, OneShotInputParams):
@@ -310,6 +325,13 @@ class UltravoxRealtimeLLMService(LLMService):
await self.cancel_task(self._receive_task, timeout=1.0)
self._receive_task = None
async def _update_settings(self, update: UltravoxRealtimeLLMSettings):
changed = await super()._update_settings(update)
if "output_medium" in changed:
await self._update_output_medium(self._settings.output_medium)
self._warn_unhandled_updated_settings(changed.keys() - {"output_medium"})
return changed
#
# frame processing
# StartFrame, StopFrame, CancelFrame implemented in base class
@@ -331,9 +353,6 @@ class UltravoxRealtimeLLMService(LLMService):
else LLMContext.from_openai_context(frame.context)
)
await self._handle_context(context)
elif isinstance(frame, LLMUpdateSettingsFrame):
if "output_medium" in frame.settings:
await self._update_output_medium(frame.settings.get("output_medium"))
elif isinstance(frame, InputTextRawFrame):
await self._send_user_text(frame.text)
await self.push_frame(frame, direction)

67
src/pipecat/services/whisper/base_stt.py
View File

@@ -10,13 +10,15 @@ This module provides common functionality for services implementing the Whisper
interface, including language mapping, metrics generation, and error handling.
"""
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from openai import AsyncOpenAI
from openai.types.audio import Transcription
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import WHISPER_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -24,6 +26,22 @@ from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
@dataclass
class BaseWhisperSTTSettings(STTSettings):
"""Settings for Whisper API-based STT services.
Parameters:
base_url: API base URL.
prompt: Optional text to guide the model's style or continue
a previous segment.
temperature: Sampling temperature between 0 and 1.
"""
base_url: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prompt: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
def language_to_whisper_language(language: Language) -> Optional[str]:
"""Maps pipecat Language enum to Whisper API language codes.
@@ -106,6 +124,8 @@ class BaseWhisperSTTService(SegmentedSTTService):
including metrics generation and error handling.
"""
_settings: BaseWhisperSTTSettings
def __init__(
self,
*,
@@ -136,33 +156,43 @@ class BaseWhisperSTTService(SegmentedSTTService):
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self.set_model_name(model)
self._client = self._create_client(api_key, base_url)
self._language = self.language_to_service_language(language or Language.EN)
self._prompt = prompt
self._temperature = temperature
self._include_prob_metrics = include_prob_metrics
self._settings = {
"base_url": base_url,
"language": self._language,
"prompt": self._prompt,
"temperature": self._temperature,
}
self._settings = BaseWhisperSTTSettings(
model=model,
language=self._language,
base_url=base_url,
prompt=self._prompt,
temperature=self._temperature,
)
self._sync_model_name_to_metrics()
def _create_client(self, api_key: Optional[str], base_url: Optional[str]):
return AsyncOpenAI(api_key=api_key, base_url=base_url)
async def set_model(self, model: str):
"""Set the model name for transcription.
async def _update_settings(self, update: STTSettings) -> dict[str, Any]:
"""Apply a settings update, syncing instance variables.
Args:
model: The name of the model to use.
Keeps ``_language``, ``_prompt``, and ``_temperature`` in sync with
the settings fields.
"""
self.set_model_name(model)
changed = await super()._update_settings(update)
if "language" in changed:
self._language = self._settings.language
if "prompt" in changed:
self._prompt = self._settings.prompt
if "temperature" in changed:
self._temperature = self._settings.temperature
return changed
def can_generate_metrics(self) -> bool:
"""Indicates whether this service can generate metrics.
"""Whether this service can generate processing metrics.
Returns:
bool: True, as this service supports metric generation.
@@ -180,15 +210,6 @@ class BaseWhisperSTTService(SegmentedSTTService):
"""
return language_to_whisper_language(language)
async def set_language(self, language: Language):
"""Set the language for transcription.
Args:
language: The Language enum value to use for transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._language = self.language_to_service_language(language)
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None

93
src/pipecat/services/whisper/stt.py
View File

@@ -11,6 +11,7 @@ supporting both Faster Whisper and MLX Whisper backends for efficient inference.
"""
import asyncio
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, Optional
@@ -19,6 +20,7 @@ from loguru import logger
from typing_extensions import TYPE_CHECKING, override
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.time import time_now_iso8601
@@ -172,6 +174,36 @@ def language_to_whisper_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class WhisperSTTSettings(STTSettings):
"""Settings for the local Whisper (Faster Whisper) STT service.
Parameters:
device: Inference device ('cpu', 'cuda', or 'auto').
compute_type: Compute type for inference ('default', 'int8', etc.).
no_speech_prob: Probability threshold for filtering non-speech segments.
"""
device: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
compute_type: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
no_speech_prob: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class WhisperMLXSTTSettings(STTSettings):
"""Settings for the MLX Whisper STT service.
Parameters:
no_speech_prob: Probability threshold for filtering non-speech segments.
temperature: Sampling temperature (0.0-1.0).
engine: Whisper engine identifier.
"""
no_speech_prob: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class WhisperSTTService(SegmentedSTTService):
"""Class to transcribe audio with a locally-downloaded Whisper model.
@@ -179,6 +211,8 @@ class WhisperSTTService(SegmentedSTTService):
segments. It supports multiple languages and various model sizes.
"""
_settings: WhisperSTTSettings
def __init__(
self,
*,
@@ -202,16 +236,17 @@ class WhisperSTTService(SegmentedSTTService):
super().__init__(**kwargs)
self._device: str = device
self._compute_type = compute_type
self.set_model_name(model if isinstance(model, str) else model.value)
self._no_speech_prob = no_speech_prob
self._model: Optional[WhisperModel] = None
self._settings = {
"language": language,
"device": self._device,
"compute_type": self._compute_type,
"no_speech_prob": self._no_speech_prob,
}
self._settings = WhisperSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
device=self._device,
compute_type=self._compute_type,
no_speech_prob=self._no_speech_prob,
)
self._sync_model_name_to_metrics()
self._load()
@@ -234,15 +269,6 @@ class WhisperSTTService(SegmentedSTTService):
"""
return language_to_whisper_language(language)
async def set_language(self, language: Language):
"""Set the language for transcription.
Args:
language: The Language enum value to use for transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
def _load(self):
"""Loads the Whisper model.
@@ -255,7 +281,7 @@ class WhisperSTTService(SegmentedSTTService):
logger.debug("Loading Whisper model...")
self._model = WhisperModel(
self.model_name, device=self._device, compute_type=self._compute_type
self._settings.model, device=self._device, compute_type=self._compute_type
)
logger.debug("Loaded Whisper model")
except ModuleNotFoundError as e:
@@ -293,9 +319,8 @@ class WhisperSTTService(SegmentedSTTService):
# Divide by 32768 because we have signed 16-bit data.
audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0
whisper_lang = self.language_to_service_language(self._settings["language"])
segments, _ = await asyncio.to_thread(
self._model.transcribe, audio_float, language=whisper_lang
self._model.transcribe, audio_float, language=self._settings.language
)
text: str = ""
for segment in segments:
@@ -305,13 +330,13 @@ class WhisperSTTService(SegmentedSTTService):
await self.stop_processing_metrics()
if text:
await self._handle_transcription(text, True, self._settings["language"])
await self._handle_transcription(text, True, self._settings.language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
)
@@ -322,6 +347,8 @@ class WhisperSTTServiceMLX(WhisperSTTService):
segments. It's optimized for Apple Silicon and supports multiple languages and quantizations.
"""
_settings: WhisperMLXSTTSettings
def __init__(
self,
*,
@@ -343,16 +370,17 @@ class WhisperSTTServiceMLX(WhisperSTTService):
# Skip WhisperSTTService.__init__ and call its parent directly
SegmentedSTTService.__init__(self, **kwargs)
self.set_model_name(model if isinstance(model, str) else model.value)
self._no_speech_prob = no_speech_prob
self._temperature = temperature
self._settings = {
"language": language,
"no_speech_prob": self._no_speech_prob,
"temperature": self._temperature,
"engine": "mlx",
}
self._settings = WhisperMLXSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
no_speech_prob=self._no_speech_prob,
temperature=self._temperature,
engine="mlx",
)
self._sync_model_name_to_metrics()
# No need to call _load() as MLX Whisper loads models on demand
@@ -390,13 +418,12 @@ class WhisperSTTServiceMLX(WhisperSTTService):
# Divide by 32768 because we have signed 16-bit data.
audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0
whisper_lang = self.language_to_service_language(self._settings["language"])
chunk = await asyncio.to_thread(
mlx_whisper.transcribe,
audio_float,
path_or_hf_repo=self.model_name,
path_or_hf_repo=self._settings.model,
temperature=self._temperature,
language=whisper_lang,
language=self._settings.language,
)
text: str = ""
for segment in chunk.get("segments", []):
@@ -413,13 +440,13 @@ class WhisperSTTServiceMLX(WhisperSTTService):
await self.stop_processing_metrics()
if text:
await self._handle_transcription(text, True, self._settings["language"])
await self._handle_transcription(text, True, self._settings.language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
)
except Exception as e:

35
src/pipecat/services/xtts/tts.py
View File

@@ -10,7 +10,8 @@ This module provides integration with Coqui XTTS streaming server for
text-to-speech synthesis using local Docker deployment.
"""
from typing import Any, AsyncGenerator, Dict, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, Dict, Optional
import aiohttp
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -68,6 +70,17 @@ def language_to_xtts_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class XTTSTTSSettings(TTSSettings):
"""Settings for XTTS TTS service.
Parameters:
base_url: Base URL of the XTTS streaming server.
"""
base_url: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class XTTSService(TTSService):
"""Coqui XTTS text-to-speech service.
@@ -76,6 +89,8 @@ class XTTSService(TTSService):
studio speakers configuration.
"""
_settings: XTTSTTSSettings
def __init__(
self,
*,
@@ -98,11 +113,11 @@ class XTTSService(TTSService):
"""
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"language": self.language_to_service_language(language),
"base_url": base_url,
}
self.set_voice(voice_id)
self._settings = XTTSTTSSettings(
voice=voice_id,
language=self.language_to_service_language(language),
base_url=base_url,
)
self._studio_speakers: Optional[Dict[str, Any]] = None
self._aiohttp_session = aiohttp_session
@@ -138,7 +153,7 @@ class XTTSService(TTSService):
if self._studio_speakers:
return
async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r:
async with self._aiohttp_session.get(self._settings.base_url + "/studio_speakers") as r:
if r.status != 200:
text = await r.text()
await self.push_error(
@@ -164,13 +179,13 @@ class XTTSService(TTSService):
logger.error(f"{self} no studio speakers available")
return
embeddings = self._studio_speakers[self._voice_id]
embeddings = self._studio_speakers[self._settings.voice]
url = self._settings["base_url"] + "/tts_stream"
url = self._settings.base_url + "/tts_stream"
payload = {
"text": text.replace(".", "").replace("*", ""),
"language": self._settings["language"],
"language": self._settings.language,
"speaker_embedding": embeddings["speaker_embedding"],
"gpt_cond_latent": embeddings["gpt_cond_latent"],
"add_wav_header": False,

43
src/pipecat/utils/tracing/service_decorators.py
View File

@@ -42,6 +42,23 @@ T = TypeVar("T")
R = TypeVar("R")
def _get_model_name(service) -> str:
"""Get the model name from a service instance.
This is a bit of a mess — there were multiple places a model name could live.
Soon, self._settings should be the only source of truth about model name.
In fact...it might already be the case, but juuuuust to be safe, we'll
check all the places we used to store it.
"""
return (
getattr(getattr(service, "_settings", None), "model", None)
or getattr(service, "_full_model_name", None)
or getattr(service, "model_name", None)
or getattr(service, "_model_name", None)
or "unknown"
)
def _noop_decorator(func):
"""No-op fallback decorator when tracing is unavailable.
@@ -202,13 +219,14 @@ def traced_tts(func: Optional[Callable] = None, *, name: Optional[str] = None) -
tracer = trace.get_tracer("pipecat")
with tracer.start_as_current_span(span_name, context=parent_context) as span:
try:
settings = getattr(self, "_settings", {})
add_tts_span_attributes(
span=span,
service_name=service_class_name,
model=getattr(self, "model_name") or "unknown",
voice_id=getattr(self, "_voice_id", "unknown"),
model=_get_model_name(self),
voice_id=getattr(settings, "voice", "unknown"),
text=text,
settings=getattr(self, "_settings", {}),
settings=settings,
character_count=len(text),
operation_name="tts",
cartesia_version=getattr(self, "_cartesia_version", None),
@@ -325,7 +343,7 @@ def traced_stt(func: Optional[Callable] = None, *, name: Optional[str] = None) -
add_stt_span_attributes(
span=current_span,
service_name=service_class_name,
model=getattr(self, "model_name") or settings.get("model", "unknown"),
model=_get_model_name(self),
transcript=transcript,
is_final=is_final,
language=str(language) if language else None,
@@ -506,10 +524,7 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) -
# Add all available attributes to the span
attribute_kwargs = {
"service_name": service_class_name,
"model": getattr(self, "_full_model_name", None)
or getattr(self, "model_name", None)
or params.get("model")
or "unknown",
"model": _get_model_name(self),
"stream": True, # Most LLM services use streaming
"parameters": params,
}
@@ -609,11 +624,7 @@ def traced_gemini_live(operation: str) -> Callable:
) as current_span:
try:
# Base service attributes
model_name = (
getattr(self, "model_name", None)
or getattr(self, "_model_name", None)
or "unknown"
)
model_name = _get_model_name(self)
voice_id = getattr(self, "_voice_id", None)
language_code = getattr(self, "_language_code", None)
settings = getattr(self, "_settings", {})
@@ -917,11 +928,7 @@ def traced_openai_realtime(operation: str) -> Callable:
) as current_span:
try:
# Base service attributes
model_name = (
getattr(self, "model_name", None)
or getattr(self, "_model_name", None)
or "unknown"
)
model_name = _get_model_name(self)
# Operation-specific attribute collection
operation_attrs = {}