Merge pull request #3946 from pipecat-ai/mb/tts-settings-review

Review TTS settings
This commit is contained in:
Mark Backman
2026-03-07 07:48:26 -05:00
committed by GitHub
50 changed files with 284 additions and 286 deletions

1
changelog/3946.added.md Normal file
View File

@@ -0,0 +1 @@
- Runtime settings updates (via `STTUpdateSettingsFrame`) now work for AWS Transcribe, Azure, Cartesia, Deepgram, ElevenLabs Realtime, Gradium, and Soniox STT services. Previously, changing settings at runtime only stored the new values without reconnecting.

View File

@@ -25,7 +25,7 @@ from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport from pipecat.runner.utils import create_transport
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService, OpenAILLMSettings from pipecat.services.openai.llm import OpenAILLMService, OpenAILLMSettings
from pipecat.services.xtts.tts import XTTSService, XTTSSettings from pipecat.services.xtts.tts import XTTSService, XTTSTTSSettings
from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -59,7 +59,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
tts = XTTSService( tts = XTTSService(
aiohttp_session=session, aiohttp_session=session,
settings=XTTSSettings( settings=XTTSTTSSettings(
voice="Claribel Dervla", voice="Claribel Dervla",
), ),
base_url="http://localhost:8000", base_url="http://localhost:8000",

View File

@@ -104,7 +104,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
# Create Strands agent processor # Create Strands agent processor
try: try:
agent = build_agent(model_id="us.anthropic.claude-3-5-haiku-20241022-v1:0", max_tokens=8000) agent = build_agent(model_id="us.anthropic.claude-sonnet-4-6", max_tokens=8000)
llm = StrandsAgentsProcessor(agent=agent) llm = StrandsAgentsProcessor(agent=agent)
logger.info("Successfully created Strands agent for NAB customer service coaching") logger.info("Successfully created Strands agent for NAB customer service coaching")
except Exception as e: except Exception as e:
@@ -152,7 +152,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
messages=[ messages=[
{ {
"role": "user", "role": "user",
"content": f"Greet the user and introduce yourself.", "content": f"Greet the user and introduce yourself. Don't use emojis.",
} }
], ],
run_llm=True, run_llm=True,

View File

@@ -64,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm = AWSBedrockLLMService( llm = AWSBedrockLLMService(
aws_region="us-west-2", aws_region="us-west-2",
settings=AWSBedrockLLMSettings( settings=AWSBedrockLLMSettings(
model="us.anthropic.claude-haiku-4-5-20251001-v1:0", model="us.anthropic.claude-sonnet-4-6",
temperature=0.8, temperature=0.8,
system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
), ),

View File

@@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
stt = GoogleSTTService( stt = GoogleSTTService(
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
settings=GoogleSTTSettings( settings=GoogleSTTSettings(
languages=Language.EN_US, languages=[Language.EN_US],
), ),
) )

View File

@@ -55,7 +55,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
stt = GoogleSTTService( stt = GoogleSTTService(
settings=GoogleSTTSettings( settings=GoogleSTTSettings(
languages=Language.EN_US, languages=[Language.EN_US],
), ),
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
) )

View File

@@ -55,8 +55,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
stt = GoogleSTTService( stt = GoogleSTTService(
settings=GoogleSTTSettings( settings=GoogleSTTSettings(
languages=Language.EN_US, languages=[Language.EN_US],
model="chirp_3", # Add model to use a specific model
# model="chirp_3",
), ),
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
location="us", location="us",

View File

@@ -58,7 +58,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
tts = FishAudioTTSService( tts = FishAudioTTSService(
api_key=os.getenv("FISH_API_KEY"), api_key=os.getenv("FISH_API_KEY"),
settings=FishAudioTTSSettings( settings=FishAudioTTSSettings(
model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama voice="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama
), ),
) )

View File

@@ -24,7 +24,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
from pipecat.runner.types import RunnerArguments from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport from pipecat.runner.utils import create_transport
from pipecat.services.openai.llm import OpenAILLMService, OpenAILLMSettings from pipecat.services.openai.llm import OpenAILLMService, OpenAILLMSettings
from pipecat.services.sarvam.stt import SarvamSTTService from pipecat.services.sarvam.stt import SarvamSTTService, SarvamSTTSettings
from pipecat.services.sarvam.tts import SarvamHttpTTSService, SarvamHttpTTSSettings from pipecat.services.sarvam.tts import SarvamHttpTTSService, SarvamHttpTTSSettings
from pipecat.transcriptions.language import Language from pipecat.transcriptions.language import Language
from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -59,14 +59,16 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
stt = SarvamSTTService( stt = SarvamSTTService(
api_key=os.getenv("SARVAM_API_KEY"), api_key=os.getenv("SARVAM_API_KEY"),
model="saarika:v2.5", settings=SarvamSTTSettings(
model="saarika:v2.5",
),
) )
tts = SarvamHttpTTSService( tts = SarvamHttpTTSService(
api_key=os.getenv("SARVAM_API_KEY"), api_key=os.getenv("SARVAM_API_KEY"),
aiohttp_session=session, aiohttp_session=session,
settings=SarvamHttpTTSSettings( settings=SarvamHttpTTSSettings(
language=Language.EN, language=Language.EN_IN,
), ),
) )

View File

@@ -61,11 +61,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm = AWSBedrockLLMService( llm = AWSBedrockLLMService(
aws_region="us-west-2", aws_region="us-west-2",
settings=AWSBedrockLLMSettings( settings=AWSBedrockLLMSettings(
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", model="us.anthropic.claude-sonnet-4-6",
# Note: usually, prefer providing latency="optimized" param. temperature=0.8,
# Here we can't because AWS Bedrock doesn't support it for Claude 3.7,
# which we need for image input.
params=AWSBedrockLLMService.InputParams(temperature=0.8),
system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are also able to describe images.", system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. You are also able to describe images.",
), ),
) )

View File

@@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm = AWSBedrockLLMService( llm = AWSBedrockLLMService(
aws_region="us-west-2", aws_region="us-west-2",
settings=AWSBedrockLLMSettings( settings=AWSBedrockLLMSettings(
model="us.anthropic.claude-haiku-4-5-20251001-v1:0", model="us.anthropic.claude-sonnet-4-6",
temperature=0.8, temperature=0.8,
system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
), ),

View File

@@ -7,7 +7,6 @@
import os import os
from deepgram import LiveOptions
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
@@ -28,7 +27,7 @@ from pipecat.processors.filters.function_filter import FunctionFilter
from pipecat.runner.types import RunnerArguments from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport from pipecat.runner.utils import create_transport
from pipecat.services.cartesia.tts import CartesiaTTSService, CartesiaTTSSettings from pipecat.services.cartesia.tts import CartesiaTTSService, CartesiaTTSSettings
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSettings
from pipecat.services.llm_service import FunctionCallParams from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.llm import OpenAILLMService, OpenAILLMSettings from pipecat.services.openai.llm import OpenAILLMService, OpenAILLMSettings
from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -102,7 +101,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot") logger.info(f"Starting bot")
stt = DeepgramSTTService( stt = DeepgramSTTService(
api_key=os.getenv("DEEPGRAM_API_KEY"), live_options=LiveOptions(language="multi") api_key=os.getenv("DEEPGRAM_API_KEY"),
settings=DeepgramSTTSettings(
language="multi",
),
) )
tts = SwitchLanguage() tts = SwitchLanguage()

View File

@@ -7,7 +7,6 @@
import asyncio import asyncio
import os import os
from deepgram import LiveOptions
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
@@ -114,7 +113,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
STTUpdateSettingsFrame( STTUpdateSettingsFrame(
delta=DeepgramSageMakerSTTSettings( delta=DeepgramSageMakerSTTSettings(
language=Language.ES, language=Language.ES,
live_options=LiveOptions(punctuate=False), punctuate=False,
) )
) )
) )

View File

@@ -7,7 +7,6 @@
import asyncio import asyncio
import os import os
from deepgram import LiveOptions
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
@@ -108,7 +107,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
STTUpdateSettingsFrame( STTUpdateSettingsFrame(
delta=DeepgramSTTSettings( delta=DeepgramSTTSettings(
language=Language.ES, language=Language.ES,
live_options=LiveOptions(punctuate=False), punctuate=False,
) )
) )
) )

View File

@@ -62,7 +62,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm = AWSBedrockLLMService( llm = AWSBedrockLLMService(
aws_region="us-west-2", aws_region="us-west-2",
settings=AWSBedrockLLMSettings( settings=AWSBedrockLLMSettings(
model="us.anthropic.claude-haiku-4-5-20251001-v1:0", model="us.anthropic.claude-sonnet-4-6",
temperature=0.8, temperature=0.8,
system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
), ),

View File

@@ -104,7 +104,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
await asyncio.sleep(10) await asyncio.sleep(10)
logger.info("Updating Gradium STT settings: delay_in_frames=5") logger.info("Updating Gradium STT settings: delay_in_frames=5")
await task.queue_frame(STTUpdateSettingsFrame(delta=GradiumSTTSettings(delay_in_frames=5))) await task.queue_frame(STTUpdateSettingsFrame(delta=GradiumSTTSettings(delay_in_frames=16)))
@transport.event_handler("on_client_disconnected") @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client): async def on_client_disconnected(transport, client):

View File

@@ -146,6 +146,7 @@ TESTS_07 = [
("07zg-interruptible-camb.py", EVAL_SIMPLE_MATH), ("07zg-interruptible-camb.py", EVAL_SIMPLE_MATH),
("07zi-interruptible-piper.py", EVAL_SIMPLE_MATH), ("07zi-interruptible-piper.py", EVAL_SIMPLE_MATH),
("07zj-interruptible-kokoro.py", EVAL_SIMPLE_MATH), ("07zj-interruptible-kokoro.py", EVAL_SIMPLE_MATH),
("07zk-interruptible-resembleai.py", EVAL_SIMPLE_MATH),
# Needs a local XTTS docker instance running. # Needs a local XTTS docker instance running.
# ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH), # ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH),
] ]

View File

@@ -74,7 +74,7 @@ def language_to_async_language(language: Language) -> Optional[str]:
@dataclass @dataclass
class AsyncAITTSSettings(TTSSettings): class AsyncAITTSSettings(TTSSettings):
"""Settings for Async AI TTS services.""" """Settings for AsyncAITTSService and AsyncAIHttpTTSService."""
pass pass

View File

@@ -107,7 +107,7 @@ class AWSTranscribeSTTService(WebsocketSTTService):
_warn_deprecated_param("language", AWSTranscribeSTTSettings, "language") _warn_deprecated_param("language", AWSTranscribeSTTSettings, "language")
default_settings.language = self.language_to_service_language(language) default_settings.language = self.language_to_service_language(language)
# 3. No params to apply # 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
@@ -158,22 +158,12 @@ class AWSTranscribeSTTService(WebsocketSTTService):
return encoding_map.get(encoding, encoding) return encoding_map.get(encoding, encoding)
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]: async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta. """Apply a settings delta and reconnect if anything changed."""
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta) changed = await super()._update_settings(delta)
if not changed: if changed and self._websocket:
return changed await self._disconnect()
await self._connect()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if changed and self._websocket:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed return changed

View File

@@ -123,7 +123,7 @@ def language_to_aws_language(language: Language) -> Optional[str]:
@dataclass @dataclass
class AWSPollyTTSSettings(TTSSettings): class AWSPollyTTSSettings(TTSSettings):
"""Settings for AWS Polly TTS service. """Settings for AWSPollyTTSService.
Parameters: Parameters:
engine: TTS engine to use ('standard', 'neural', etc.). engine: TTS engine to use ('standard', 'neural', etc.).

View File

@@ -112,7 +112,7 @@ class AzureSTTService(STTService):
_warn_deprecated_param("language", AzureSTTSettings, "language") _warn_deprecated_param("language", AzureSTTSettings, "language")
default_settings.language = language_to_azure_language(language) default_settings.language = language_to_azure_language(language)
# 3. No params to apply # 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
@@ -159,23 +159,16 @@ class AzureSTTService(STTService):
return language_to_azure_language(language) return language_to_azure_language(language)
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]: async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta. """Apply a settings delta and reconnect if language changed."""
Settings are stored but not applied to the active recognizer.
"""
changed = await super()._update_settings(delta) changed = await super()._update_settings(delta)
# TODO: someday we could reconnect here to apply updated settings. if "language" in changed:
# Code might look something like the below: self._speech_config.speech_recognition_language = (
# if "language" in changed: self._settings.language or language_to_azure_language(Language.EN_US)
# self._speech_config.speech_recognition_language = self._settings.language )
# if self._speech_recognizer: if self._audio_stream:
# # Requires refactoring to set up and tear down recognizer, as await self._disconnect()
# # language is applied at recognizer initialization await self._connect()
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed return changed
@@ -202,14 +195,32 @@ class AzureSTTService(STTService):
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the speech recognition service. """Start the speech recognition service.
Initializes the Azure speech recognizer with audio stream configuration
and begins continuous speech recognition.
Args: Args:
frame: Frame indicating the start of processing. frame: Frame indicating the start of processing.
""" """
await super().start(frame) await super().start(frame)
await self._connect()
async def stop(self, frame: EndFrame):
"""Stop the speech recognition service.
Args:
frame: Frame indicating the end of processing.
"""
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
"""Cancel the speech recognition service.
Args:
frame: Frame indicating cancellation.
"""
await super().cancel(frame)
await self._disconnect()
async def _connect(self):
"""Initialize the Azure speech recognizer and begin continuous recognition."""
if self._audio_stream: if self._audio_stream:
return return
@@ -231,37 +242,15 @@ class AzureSTTService(STTService):
error_msg=f"Uncaught exception during initialization: {e}", exception=e error_msg=f"Uncaught exception during initialization: {e}", exception=e
) )
async def stop(self, frame: EndFrame): async def _disconnect(self):
"""Stop the speech recognition service. """Stop recognition and close audio streams."""
Cleanly shuts down the Azure speech recognizer and closes audio streams.
Args:
frame: Frame indicating the end of processing.
"""
await super().stop(frame)
if self._speech_recognizer:
self._speech_recognizer.stop_continuous_recognition_async()
if self._audio_stream:
self._audio_stream.close()
async def cancel(self, frame: CancelFrame):
"""Cancel the speech recognition service.
Immediately stops recognition and closes resources.
Args:
frame: Frame indicating cancellation.
"""
await super().cancel(frame)
if self._speech_recognizer: if self._speech_recognizer:
self._speech_recognizer.stop_continuous_recognition_async() self._speech_recognizer.stop_continuous_recognition_async()
self._speech_recognizer = None
if self._audio_stream: if self._audio_stream:
self._audio_stream.close() self._audio_stream.close()
self._audio_stream = None
@traced_stt @traced_stt
async def _handle_transcription( async def _handle_transcription(

View File

@@ -68,7 +68,7 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma
@dataclass @dataclass
class AzureTTSSettings(TTSSettings): class AzureTTSSettings(TTSSettings):
"""Settings for Azure TTS services. """Settings for AzureTTSService and AzureHttpTTSService.
Parameters: Parameters:
emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). emphasis: Emphasis level for speech ("strong", "moderate", "reduced").

View File

@@ -135,7 +135,7 @@ def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]:
@dataclass @dataclass
class CambTTSSettings(TTSSettings): class CambTTSSettings(TTSSettings):
"""Settings for Camb.ai TTS service. """Settings for CambTTSService.
Parameters: Parameters:
user_instructions: Custom instructions for mars-instruct model only. user_instructions: Custom instructions for mars-instruct model only.

View File

@@ -23,7 +23,6 @@ from pipecat.frames.frames import (
Frame, Frame,
StartFrame, StartFrame,
TTSAudioRawFrame, TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame, TTSStoppedFrame,
) )
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, _warn_deprecated_param from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, _warn_deprecated_param
@@ -188,7 +187,7 @@ class CartesiaEmotion(str, Enum):
@dataclass @dataclass
class CartesiaTTSSettings(TTSSettings): class CartesiaTTSSettings(TTSSettings):
"""Settings for Cartesia TTS services. """Settings for CartesiaTTSService and CartesiaHttpTTSService.
Parameters: Parameters:
generation_config: Generation configuration for Sonic-3 models. Includes volume, generation_config: Generation configuration for Sonic-3 models. Includes volume,
@@ -705,7 +704,7 @@ class CartesiaHttpTTSService(TTSService):
voice_id: Optional[str] = None, voice_id: Optional[str] = None,
model: Optional[str] = None, model: Optional[str] = None,
base_url: str = "https://api.cartesia.ai", base_url: str = "https://api.cartesia.ai",
cartesia_version: str = "2024-11-13", cartesia_version: str = "2026-03-01",
aiohttp_session: Optional[aiohttp.ClientSession] = None, aiohttp_session: Optional[aiohttp.ClientSession] = None,
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
encoding: str = "pcm_s16le", encoding: str = "pcm_s16le",

View File

@@ -40,7 +40,7 @@ from pipecat.utils.tracing.service_decorators import traced_tts
@dataclass @dataclass
class DeepgramSageMakerTTSSettings(TTSSettings): class DeepgramSageMakerTTSSettings(TTSSettings):
"""Settings for Deepgram SageMaker TTS service.""" """Settings for DeepgramSageMakerTTSService."""
pass pass

View File

@@ -365,7 +365,9 @@ class DeepgramSTTService(STTService):
vad_events=False, vad_events=False,
) )
# 2. Apply live_options overrides — only if settings not provided # 2. (No step 2, as there are no deprecated direct args)
# 3. Apply live_options overrides — only if settings not provided
if live_options is not None: if live_options is not None:
_warn_deprecated_param("live_options", DeepgramSTTSettings) _warn_deprecated_param("live_options", DeepgramSTTSettings)
if not settings: if not settings:
@@ -402,7 +404,7 @@ class DeepgramSTTService(STTService):
delta = DeepgramSTTSettings.from_mapping(lo_dict) delta = DeepgramSTTSettings.from_mapping(lo_dict)
default_settings.apply_update(delta) default_settings.apply_update(delta)
# 3. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
default_settings.apply_update(settings) default_settings.apply_update(settings)
@@ -494,8 +496,9 @@ class DeepgramSTTService(STTService):
if isinstance(self._settings, DeepgramSTTSettings): if isinstance(self._settings, DeepgramSTTSettings):
self._settings._sync_extra_to_fields() self._settings._sync_extra_to_fields()
await self._disconnect() if self._connection:
await self._connect() await self._disconnect()
await self._connect()
return changed return changed
@@ -594,13 +597,16 @@ class DeepgramSTTService(STTService):
return return
logger.debug("Disconnecting from Deepgram") logger.debug("Disconnecting from Deepgram")
# Ask Deepgram to close the stream gracefully before cancelling the task. # Clear self._connection first to prevent run_stt from sending audio
if self._connection: # during the close handshake, then close gracefully on the saved ref.
await self._connection.send_close_stream() connection = self._connection
self._connection = None
if connection:
await connection.send_close_stream()
await self.cancel_task(self._connection_task) await self.cancel_task(self._connection_task)
self._connection_task = None self._connection_task = None
self._connection = None
async def _connection_handler(self): async def _connection_handler(self):
"""Manages the full WebSocket lifecycle inside a single async with block. """Manages the full WebSocket lifecycle inside a single async with block.

View File

@@ -45,7 +45,7 @@ except ModuleNotFoundError as e:
@dataclass @dataclass
class DeepgramTTSSettings(TTSSettings): class DeepgramTTSSettings(TTSSettings):
"""Settings for Deepgram TTS service.""" """Settings for DeepgramTTSService and DeepgramHttpTTSService."""
pass pass
@@ -110,6 +110,8 @@ class DeepgramTTSService(WebsocketTTSService):
default_settings.model = voice default_settings.model = voice
default_settings.voice = voice default_settings.voice = voice
# 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
default_settings.apply_update(settings) default_settings.apply_update(settings)
@@ -423,6 +425,8 @@ class DeepgramHttpTTSService(TTSService):
default_settings.model = voice default_settings.model = voice
default_settings.voice = voice default_settings.voice = voice
# 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
default_settings.apply_update(settings) default_settings.apply_update(settings)

View File

@@ -200,18 +200,12 @@ class ElevenLabsRealtimeSTTSettings(STTSettings):
vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive). vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive).
min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms). min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms).
min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms). min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms).
include_timestamps: Whether to include word-level timestamps in transcripts.
enable_logging: Whether to enable logging on ElevenLabs' side.
include_language_detection: Whether to include language detection in transcripts.
""" """
vad_silence_threshold_secs: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) vad_silence_threshold_secs: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_speech_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) min_speech_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_silence_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) min_silence_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_timestamps: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_logging: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_language_detection: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class ElevenLabsSTTService(SegmentedSTTService): class ElevenLabsSTTService(SegmentedSTTService):
@@ -496,6 +490,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
commit_strategy: CommitStrategy = CommitStrategy.MANUAL, commit_strategy: CommitStrategy = CommitStrategy.MANUAL,
model: Optional[str] = None, model: Optional[str] = None,
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
include_timestamps: bool = False,
enable_logging: bool = False,
include_language_detection: bool = False,
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
settings: Optional[ElevenLabsRealtimeSTTSettings] = None, settings: Optional[ElevenLabsRealtimeSTTSettings] = None,
ttfs_p99_latency: Optional[float] = ELEVENLABS_REALTIME_TTFS_P99, ttfs_p99_latency: Optional[float] = ELEVENLABS_REALTIME_TTFS_P99,
@@ -515,6 +512,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Use ``settings=ElevenLabsRealtimeSTTSettings(model=...)`` instead. Use ``settings=ElevenLabsRealtimeSTTSettings(model=...)`` instead.
sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
include_timestamps: Whether to include word-level timestamps in transcripts.
enable_logging: Whether to enable logging on ElevenLabs' side.
include_language_detection: Whether to include language detection in transcripts.
params: Configuration parameters for the STT service. params: Configuration parameters for the STT service.
.. deprecated:: 0.0.105 .. deprecated:: 0.0.105
@@ -534,9 +534,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
vad_threshold=None, vad_threshold=None,
min_speech_duration_ms=None, min_speech_duration_ms=None,
min_silence_duration_ms=None, min_silence_duration_ms=None,
include_timestamps=False,
enable_logging=False,
include_language_detection=False,
) )
# 2. Apply direct init arg overrides (deprecated) # 2. Apply direct init arg overrides (deprecated)
@@ -555,9 +552,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
default_settings.vad_threshold = params.vad_threshold default_settings.vad_threshold = params.vad_threshold
default_settings.min_speech_duration_ms = params.min_speech_duration_ms default_settings.min_speech_duration_ms = params.min_speech_duration_ms
default_settings.min_silence_duration_ms = params.min_silence_duration_ms default_settings.min_silence_duration_ms = params.min_silence_duration_ms
default_settings.include_timestamps = params.include_timestamps include_timestamps = params.include_timestamps
default_settings.enable_logging = params.enable_logging enable_logging = params.enable_logging
default_settings.include_language_detection = params.include_language_detection include_language_detection = params.include_language_detection
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
@@ -579,6 +576,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
# Init-only config (not runtime-updatable). # Init-only config (not runtime-updatable).
self._commit_strategy = commit_strategy self._commit_strategy = commit_strategy
self._include_timestamps = include_timestamps
self._enable_logging = enable_logging
self._include_language_detection = include_language_detection
self._connected_event = asyncio.Event() self._connected_event = asyncio.Event()
self._connected_event.set() self._connected_event.set()
@@ -605,8 +605,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
if not changed: if not changed:
return changed return changed
await self._disconnect() if self._websocket:
await self._connect() await self._disconnect()
await self._connect()
return changed return changed
@@ -762,17 +763,15 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
params.append(f"commit_strategy={self._commit_strategy.value}") params.append(f"commit_strategy={self._commit_strategy.value}")
# Add optional parameters # Add optional parameters
if self._settings.include_timestamps: if self._include_timestamps:
params.append( params.append(f"include_timestamps={str(self._include_timestamps).lower()}")
f"include_timestamps={str(self._settings.include_timestamps).lower()}"
)
if self._settings.enable_logging: if self._enable_logging:
params.append(f"enable_logging={str(self._settings.enable_logging).lower()}") params.append(f"enable_logging={str(self._enable_logging).lower()}")
if self._settings.include_language_detection: if self._include_language_detection:
params.append( params.append(
f"include_language_detection={str(self._settings.include_language_detection).lower()}" f"include_language_detection={str(self._include_language_detection).lower()}"
) )
# Add VAD parameters if using VAD commit strategy and values are specified # Add VAD parameters if using VAD commit strategy and values are specified
@@ -920,7 +919,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
""" """
# If timestamps are enabled, skip this message and wait for the # If timestamps are enabled, skip this message and wait for the
# committed_transcript_with_timestamps message which contains all the data # committed_transcript_with_timestamps message which contains all the data
if self._settings.include_timestamps: if self._include_timestamps:
return return
text = data.get("text", "").strip() text = data.get("text", "").strip()

View File

@@ -187,7 +187,7 @@ class PronunciationDictionaryLocator(BaseModel):
@dataclass @dataclass
class ElevenLabsTTSSettings(TTSSettings): class ElevenLabsTTSSettings(TTSSettings):
"""Settings for the ElevenLabs WebSocket TTS service. """Settings for ElevenLabsTTSService.
Fields that appear in the WebSocket URL (``voice``, ``model``, Fields that appear in the WebSocket URL (``voice``, ``model``,
``language``) require a full reconnect when changed. Fields that ``language``) require a full reconnect when changed. Fields that
@@ -225,7 +225,7 @@ class ElevenLabsTTSSettings(TTSSettings):
@dataclass @dataclass
class ElevenLabsHttpTTSSettings(TTSSettings): class ElevenLabsHttpTTSSettings(TTSSettings):
"""Settings for the ElevenLabs HTTP TTS service. """Settings for ElevenLabsHttpTTSService.
Parameters: Parameters:
optimize_streaming_latency: Latency optimization level (0-4). optimize_streaming_latency: Latency optimization level (0-4).
@@ -358,6 +358,9 @@ class ElevenLabsTTSService(WebsocketTTSService):
model: Optional[str] = None, model: Optional[str] = None,
url: str = "wss://api.elevenlabs.io", url: str = "wss://api.elevenlabs.io",
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
auto_mode: bool = True,
enable_ssml_parsing: Optional[bool] = None,
enable_logging: Optional[bool] = None,
pronunciation_dictionary_locators: Optional[List[PronunciationDictionaryLocator]] = None, pronunciation_dictionary_locators: Optional[List[PronunciationDictionaryLocator]] = None,
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
settings: Optional[ElevenLabsTTSSettings] = None, settings: Optional[ElevenLabsTTSSettings] = None,
@@ -381,6 +384,9 @@ class ElevenLabsTTSService(WebsocketTTSService):
url: WebSocket URL for ElevenLabs TTS API. url: WebSocket URL for ElevenLabs TTS API.
sample_rate: Audio sample rate. If None, uses default. sample_rate: Audio sample rate. If None, uses default.
auto_mode: Whether to enable automatic mode optimization.
enable_ssml_parsing: Whether to parse SSML tags in text.
enable_logging: Whether to enable ElevenLabs server-side logging.
pronunciation_dictionary_locators: List of pronunciation dictionary pronunciation_dictionary_locators: List of pronunciation dictionary
locators to use. locators to use.
params: Additional input parameters for voice customization. params: Additional input parameters for voice customization.
@@ -428,11 +434,6 @@ class ElevenLabsTTSService(WebsocketTTSService):
apply_text_normalization=None, apply_text_normalization=None,
) )
# Track init-only URL params through the override chain
_auto_mode = True
_enable_ssml_parsing = None
_enable_logging = None
# 2. Apply direct init arg overrides (deprecated) # 2. Apply direct init arg overrides (deprecated)
if voice_id is not None: if voice_id is not None:
_warn_deprecated_param("voice_id", ElevenLabsTTSSettings, "voice") _warn_deprecated_param("voice_id", ElevenLabsTTSSettings, "voice")
@@ -459,11 +460,11 @@ class ElevenLabsTTSService(WebsocketTTSService):
if params.speed is not None: if params.speed is not None:
default_settings.speed = params.speed default_settings.speed = params.speed
if params.auto_mode is not None: if params.auto_mode is not None:
_auto_mode = str(params.auto_mode).lower() auto_mode = params.auto_mode
if params.enable_ssml_parsing is not None: if params.enable_ssml_parsing is not None:
_enable_ssml_parsing = params.enable_ssml_parsing enable_ssml_parsing = params.enable_ssml_parsing
if params.enable_logging is not None: if params.enable_logging is not None:
_enable_logging = params.enable_logging enable_logging = params.enable_logging
if params.apply_text_normalization is not None: if params.apply_text_normalization is not None:
default_settings.apply_text_normalization = params.apply_text_normalization default_settings.apply_text_normalization = params.apply_text_normalization
if _pronunciation_dictionary_locators is None: if _pronunciation_dictionary_locators is None:
@@ -488,9 +489,9 @@ class ElevenLabsTTSService(WebsocketTTSService):
self._url = url self._url = url
# Init-only WebSocket URL params (not runtime-updatable). # Init-only WebSocket URL params (not runtime-updatable).
self._auto_mode = _auto_mode self._auto_mode = auto_mode
self._enable_ssml_parsing = _enable_ssml_parsing self._enable_ssml_parsing = enable_ssml_parsing
self._enable_logging = _enable_logging self._enable_logging = enable_logging
self._output_format = "" # initialized in start() self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings() self._voice_settings = self._set_voice_settings()
@@ -664,7 +665,7 @@ class ElevenLabsTTSService(WebsocketTTSService):
voice_id = self._settings.voice voice_id = self._settings.voice
model = self._settings.model model = self._settings.model
output_format = self._output_format output_format = self._output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._auto_mode}" url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={str(self._auto_mode).lower()}"
if self._enable_ssml_parsing: if self._enable_ssml_parsing:
url += f"&enable_ssml_parsing={self._enable_ssml_parsing}" url += f"&enable_ssml_parsing={self._enable_ssml_parsing}"

View File

@@ -10,9 +10,8 @@ This module provides integration with Fish Audio's real-time TTS WebSocket API
for streaming text-to-speech synthesis with customizable voice parameters. for streaming text-to-speech synthesis with customizable voice parameters.
""" """
import uuid
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Literal, Mapping, Optional, Self from typing import Any, AsyncGenerator, Literal, Mapping, Optional, Self
from loguru import logger from loguru import logger
from pydantic import BaseModel from pydantic import BaseModel
@@ -25,7 +24,6 @@ from pipecat.frames.frames import (
InterruptionFrame, InterruptionFrame,
StartFrame, StartFrame,
TTSAudioRawFrame, TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame, TTSStoppedFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
@@ -49,21 +47,23 @@ FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"]
@dataclass @dataclass
class FishAudioTTSSettings(TTSSettings): class FishAudioTTSSettings(TTSSettings):
"""Settings for Fish Audio TTS service. """Settings for FishAudioTTSService.
Parameters: Parameters:
latency: Latency mode ("normal" or "balanced"). Defaults to "normal". latency: Latency mode ("normal" or "balanced"). Defaults to "balanced".
normalize: Whether to normalize audio output. Defaults to True. normalize: Whether to normalize audio output. Defaults to True.
temperature: Controls randomness in speech generation (0.0-1.0).
top_p: Controls diversity via nucleus sampling (0.0-1.0).
prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0. prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0.
prosody_volume: Volume adjustment in dB. Defaults to 0. prosody_volume: Volume adjustment in dB (-20 to 20). Defaults to 0.
reference_id: Reference ID of the voice model.
""" """
latency: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) latency: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
normalize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) normalize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) prosody_speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_volume: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) prosody_volume: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
reference_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod @classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> Self: def from_mapping(cls, settings: Mapping[str, Any]) -> Self:
@@ -174,18 +174,18 @@ class FishAudioTTSService(InterruptibleTTSService):
model="s1", model="s1",
voice=None, voice=None,
language=None, language=None,
latency="normal", latency="balanced",
normalize=True, normalize=True,
temperature=None,
top_p=None,
prosody_speed=1.0, prosody_speed=1.0,
prosody_volume=0, prosody_volume=0,
reference_id=None,
) )
# 2. Apply direct init arg overrides (deprecated) # 2. Apply direct init arg overrides (deprecated)
if reference_id is not None: if reference_id is not None:
_warn_deprecated_param("reference_id", FishAudioTTSSettings, "voice") _warn_deprecated_param("reference_id", FishAudioTTSSettings, "voice")
default_settings.voice = reference_id default_settings.voice = reference_id
default_settings.reference_id = reference_id
if model_id is not None: if model_id is not None:
_warn_deprecated_param("model_id", FishAudioTTSSettings, "model") _warn_deprecated_param("model_id", FishAudioTTSSettings, "model")
default_settings.model = model_id default_settings.model = model_id
@@ -317,8 +317,12 @@ class FishAudioTTSService(InterruptibleTTSService):
"speed": self._settings.prosody_speed, "speed": self._settings.prosody_speed,
"volume": self._settings.prosody_volume, "volume": self._settings.prosody_volume,
}, },
"reference_id": self._settings.reference_id, "reference_id": self._settings.voice,
} }
if self._settings.temperature is not None:
request_settings["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
request_settings["top_p"] = self._settings.top_p
start_message = {"event": "start", "request": {"text": "", **request_settings}} start_message = {"event": "start", "request": {"text": "", **request_settings}}
await self._websocket.send(ormsgpack.packb(start_message)) await self._websocket.send(ormsgpack.packb(start_message))
logger.debug("Sent start message to Fish Audio") logger.debug("Sent start message to Fish Audio")
@@ -375,7 +379,14 @@ class FishAudioTTSService(InterruptibleTTSService):
frame = TTSAudioRawFrame(audio_data, self.sample_rate, 1) frame = TTSAudioRawFrame(audio_data, self.sample_rate, 1)
await self.push_frame(frame) await self.push_frame(frame)
await self.stop_ttfb_metrics() await self.stop_ttfb_metrics()
continue elif event == "finish":
reason = msg.get("reason", "unknown")
if reason == "error":
await self.push_error(
error_msg="Fish Audio server error during synthesis"
)
else:
logger.debug(f"Fish Audio session finished: {reason}")
except Exception as e: except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)

View File

@@ -12,7 +12,7 @@ WebSocket API for streaming audio transcription.
import base64 import base64
import json import json
from dataclasses import dataclass from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional from typing import Any, AsyncGenerator, Optional
from loguru import logger from loguru import logger
@@ -28,7 +28,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame, VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import STTSettings, _warn_deprecated_param from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, _warn_deprecated_param
from pipecat.services.stt_latency import GRADIUM_TTFS_P99 from pipecat.services.stt_latency import GRADIUM_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language from pipecat.transcriptions.language import Language, resolve_language
@@ -68,9 +68,16 @@ def language_to_gradium_language(language: Language) -> Optional[str]:
@dataclass @dataclass
class GradiumSTTSettings(STTSettings): class GradiumSTTSettings(STTSettings):
"""Settings for GradiumSTTService.""" """Settings for GradiumSTTService.
pass Parameters:
delay_in_frames: Delay in audio frames (80ms each) before text is
generated. Higher delays allow more context but increase latency.
Allowed values: 7, 8, 10, 12, 14, 16, 20, 24, 36, 48.
Default is 10 (800ms). Lower values like 7-8 give faster response.
"""
delay_in_frames: Optional[int] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GradiumSTTService(WebsocketSTTService): class GradiumSTTService(WebsocketSTTService):
@@ -107,7 +114,6 @@ class GradiumSTTService(WebsocketSTTService):
*, *,
api_key: str, api_key: str,
api_endpoint_base_url: str = "wss://eu.api.gradium.ai/api/speech/asr", api_endpoint_base_url: str = "wss://eu.api.gradium.ai/api/speech/asr",
delay_in_frames: Optional[int] = None,
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
json_config: Optional[str] = None, json_config: Optional[str] = None,
settings: Optional[GradiumSTTSettings] = None, settings: Optional[GradiumSTTSettings] = None,
@@ -119,9 +125,6 @@ class GradiumSTTService(WebsocketSTTService):
Args: Args:
api_key: Gradium API key for authentication. api_key: Gradium API key for authentication.
api_endpoint_base_url: WebSocket endpoint URL. Defaults to Gradium's streaming endpoint. api_endpoint_base_url: WebSocket endpoint URL. Defaults to Gradium's streaming endpoint.
delay_in_frames: Delay in audio frames (80ms each) before text is
generated. Higher delays allow more context but increase latency.
Allowed values: 7, 8, 10, 12, 14, 16, 20, 24, 36, 48.
params: Configuration parameters for language and delay settings. params: Configuration parameters for language and delay settings.
.. deprecated:: 0.0.105 .. deprecated:: 0.0.105
@@ -151,9 +154,10 @@ class GradiumSTTService(WebsocketSTTService):
default_settings = GradiumSTTSettings( default_settings = GradiumSTTSettings(
model=None, model=None,
language=None, language=None,
delay_in_frames=None,
) )
# 2. (no deprecated direct args for this service) # 2. (No step 2, as there are no deprecated direct args)
# 3. Apply params overrides — only if settings not provided # 3. Apply params overrides — only if settings not provided
if params is not None: if params is not None:
@@ -161,7 +165,7 @@ class GradiumSTTService(WebsocketSTTService):
if not settings: if not settings:
default_settings.language = params.language default_settings.language = params.language
if params.delay_in_frames is not None: if params.delay_in_frames is not None:
delay_in_frames = params.delay_in_frames default_settings.delay_in_frames = params.delay_in_frames
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
@@ -178,7 +182,6 @@ class GradiumSTTService(WebsocketSTTService):
self._api_endpoint_base_url = api_endpoint_base_url self._api_endpoint_base_url = api_endpoint_base_url
self._websocket = None self._websocket = None
self._json_config = json_config self._json_config = json_config
self._config_delay_in_frames = delay_in_frames
self._receive_task = None self._receive_task = None
@@ -212,8 +215,9 @@ class GradiumSTTService(WebsocketSTTService):
if not changed: if not changed:
return changed return changed
await self._disconnect() if self._websocket:
await self._connect() await self._disconnect()
await self._connect()
return changed return changed
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
@@ -358,8 +362,8 @@ class GradiumSTTService(WebsocketSTTService):
gradium_language = language_to_gradium_language(self._settings.language) gradium_language = language_to_gradium_language(self._settings.language)
if gradium_language: if gradium_language:
json_config["language"] = gradium_language json_config["language"] = gradium_language
if self._config_delay_in_frames: if self._settings.delay_in_frames:
json_config["delay_in_frames"] = self._config_delay_in_frames json_config["delay_in_frames"] = self._settings.delay_in_frames
if json_config: if json_config:
setup_msg["json_config"] = json_config setup_msg["json_config"] = json_config
await self._websocket.send(json.dumps(setup_msg)) await self._websocket.send(json.dumps(setup_msg))

View File

@@ -39,7 +39,7 @@ SAMPLE_RATE = 48000
@dataclass @dataclass
class GradiumTTSSettings(TTSSettings): class GradiumTTSSettings(TTSSettings):
"""Settings for the Gradium TTS service.""" """Settings for GradiumTTSService."""
pass pass

View File

@@ -34,7 +34,7 @@ except ModuleNotFoundError as e:
@dataclass @dataclass
class GroqTTSSettings(TTSSettings): class GroqTTSSettings(TTSSettings):
"""Settings for the Groq TTS service. """Settings for GroqTTSService.
Parameters: Parameters:
speed: Speech speed multiplier. Defaults to 1.0. speed: Speech speed multiplier. Defaults to 1.0.

View File

@@ -50,7 +50,7 @@ DEFAULT_HEADERS = {
@dataclass @dataclass
class HumeTTSSettings(TTSSettings): class HumeTTSSettings(TTSSettings):
"""Settings for Hume TTS service. """Settings for HumeTTSService.
Parameters: Parameters:
description: Natural-language acting directions (up to 100 characters). description: Natural-language acting directions (up to 100 characters).

View File

@@ -68,32 +68,19 @@ from pipecat.utils.tracing.service_decorators import traced_tts
@dataclass @dataclass
class InworldTTSSettings(TTSSettings): class InworldTTSSettings(TTSSettings):
"""Settings for Inworld TTS services. """Settings for InworldTTSService and InworldHttpTTSService.
Parameters: Parameters:
speaking_rate: Speaking rate for speech synthesis. speaking_rate: Speaking rate for speech synthesis.
temperature: Temperature for speech synthesis. temperature: Temperature for speech synthesis.
auto_mode: Whether to use auto mode. Recommended when texts are sent
in full sentences/phrases. When enabled, the server controls
flushing of buffered text to achieve minimal latency while
maintaining high quality audio output. If None (default),
automatically set based on aggregate_sentences.
apply_text_normalization: Whether to apply text normalization.
timestamp_transport_strategy: Strategy for timestamp transport ("ASYNC" or "SYNC").
""" """
speaking_rate: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) speaking_rate: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN) temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
auto_mode: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
timestamp_transport_strategy: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = { _aliases: ClassVar[Dict[str, str]] = {
"voiceId": "voice", "voiceId": "voice",
"modelId": "model", "modelId": "model",
"applyTextNormalization": "apply_text_normalization",
"autoMode": "auto_mode",
"timestampTransportStrategy": "timestamp_transport_strategy",
} }
@classmethod @classmethod
@@ -141,6 +128,7 @@ class InworldHttpTTSService(TTSService):
streaming: bool = True, streaming: bool = True,
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
encoding: str = "LINEAR16", encoding: str = "LINEAR16",
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = "ASYNC",
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
settings: Optional[InworldTTSSettings] = None, settings: Optional[InworldTTSSettings] = None,
**kwargs, **kwargs,
@@ -163,6 +151,8 @@ class InworldHttpTTSService(TTSService):
streaming: Whether to use streaming mode. streaming: Whether to use streaming mode.
sample_rate: Audio sample rate in Hz. sample_rate: Audio sample rate in Hz.
encoding: Audio encoding format. encoding: Audio encoding format.
timestamp_transport_strategy: Strategy for timestamp transport
("ASYNC" or "SYNC"). Defaults to "ASYNC".
params: Input parameters for Inworld TTS configuration. params: Input parameters for Inworld TTS configuration.
.. deprecated:: 0.0.105 .. deprecated:: 0.0.105
@@ -179,9 +169,6 @@ class InworldHttpTTSService(TTSService):
language=None, language=None,
speaking_rate=None, speaking_rate=None,
temperature=None, temperature=None,
timestamp_transport_strategy="ASYNC",
auto_mode=None, # Not applicable for HTTP TTS
apply_text_normalization=None, # Not applicable for HTTP TTS
) )
# 2. Apply direct init arg overrides (deprecated) # 2. Apply direct init arg overrides (deprecated)
@@ -201,9 +188,7 @@ class InworldHttpTTSService(TTSService):
if params.temperature is not None: if params.temperature is not None:
default_settings.temperature = params.temperature default_settings.temperature = params.temperature
if params.timestamp_transport_strategy is not None: if params.timestamp_transport_strategy is not None:
default_settings.timestamp_transport_strategy = ( timestamp_transport_strategy = params.timestamp_transport_strategy
params.timestamp_transport_strategy
)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
@@ -230,9 +215,10 @@ class InworldHttpTTSService(TTSService):
self._cumulative_time = 0.0 self._cumulative_time = 0.0
# Init-only audio format config (not runtime-updatable). # Init-only config (not runtime-updatable).
self._audio_encoding = encoding self._audio_encoding = encoding
self._audio_sample_rate = 0 # Set in start() self._audio_sample_rate = 0 # Set in start()
self._timestamp_transport_strategy = timestamp_transport_strategy
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.
@@ -251,22 +237,6 @@ class InworldHttpTTSService(TTSService):
await super().start(frame) await super().start(frame)
self._audio_sample_rate = self.sample_rate self._audio_sample_rate = self.sample_rate
async def stop(self, frame: EndFrame):
"""Stop the Inworld TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame)
async def cancel(self, frame: CancelFrame):
"""Cancel the Inworld TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame and handle state changes. """Push a frame and handle state changes.
@@ -347,8 +317,8 @@ class InworldHttpTTSService(TTSService):
# Use WORD timestamps for simplicity and correct spacing/capitalization # Use WORD timestamps for simplicity and correct spacing/capitalization
payload["timestampType"] = self._timestamp_type payload["timestampType"] = self._timestamp_type
if self._settings.timestamp_transport_strategy is not None: if self._timestamp_transport_strategy is not None:
payload["timestampTransportStrategy"] = self._settings.timestamp_transport_strategy payload["timestampTransportStrategy"] = self._timestamp_transport_strategy
request_id = str(uuid.uuid4()) request_id = str(uuid.uuid4())
headers = { headers = {
@@ -556,6 +526,9 @@ class InworldTTSService(WebsocketTTSService):
url: str = "wss://api.inworld.ai/tts/v1/voice:streamBidirectional", url: str = "wss://api.inworld.ai/tts/v1/voice:streamBidirectional",
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
encoding: str = "LINEAR16", encoding: str = "LINEAR16",
auto_mode: Optional[bool] = None,
apply_text_normalization: Optional[str] = None,
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = "ASYNC",
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
settings: Optional[InworldTTSSettings] = None, settings: Optional[InworldTTSSettings] = None,
aggregate_sentences: Optional[bool] = None, aggregate_sentences: Optional[bool] = None,
@@ -580,6 +553,12 @@ class InworldTTSService(WebsocketTTSService):
url: URL of the Inworld WebSocket API. url: URL of the Inworld WebSocket API.
sample_rate: Audio sample rate in Hz. sample_rate: Audio sample rate in Hz.
encoding: Audio encoding format. encoding: Audio encoding format.
auto_mode: Whether to use auto mode. When enabled, the server
controls flushing of buffered text. If None (default),
automatically set based on ``aggregate_sentences``.
apply_text_normalization: Whether to apply text normalization.
timestamp_transport_strategy: Strategy for timestamp transport
("ASYNC" or "SYNC"). Defaults to "ASYNC".
params: Input parameters for Inworld WebSocket TTS configuration. params: Input parameters for Inworld WebSocket TTS configuration.
.. deprecated:: 0.0.105 .. deprecated:: 0.0.105
@@ -596,6 +575,10 @@ class InworldTTSService(WebsocketTTSService):
append_trailing_space: Whether to append a trailing space to text before sending to TTS. append_trailing_space: Whether to append a trailing space to text before sending to TTS.
**kwargs: Additional arguments passed to the parent class. **kwargs: Additional arguments passed to the parent class.
""" """
# Derive auto_mode from aggregate_sentences if not explicitly set
if auto_mode is None:
auto_mode = True if aggregate_sentences is None else aggregate_sentences
# 1. Initialize default_settings with hardcoded defaults # 1. Initialize default_settings with hardcoded defaults
default_settings = InworldTTSSettings( default_settings = InworldTTSSettings(
model="inworld-tts-1.5-max", model="inworld-tts-1.5-max",
@@ -603,9 +586,6 @@ class InworldTTSService(WebsocketTTSService):
language=None, language=None,
speaking_rate=None, speaking_rate=None,
temperature=None, temperature=None,
apply_text_normalization=None,
timestamp_transport_strategy="ASYNC",
auto_mode=True if aggregate_sentences is None else aggregate_sentences,
) )
# 2. Apply direct init arg overrides (deprecated) # 2. Apply direct init arg overrides (deprecated)
@@ -627,13 +607,11 @@ class InworldTTSService(WebsocketTTSService):
if params.temperature is not None: if params.temperature is not None:
default_settings.temperature = params.temperature default_settings.temperature = params.temperature
if params.apply_text_normalization is not None: if params.apply_text_normalization is not None:
default_settings.apply_text_normalization = params.apply_text_normalization apply_text_normalization = params.apply_text_normalization
if params.timestamp_transport_strategy is not None: if params.timestamp_transport_strategy is not None:
default_settings.timestamp_transport_strategy = ( timestamp_transport_strategy = params.timestamp_transport_strategy
params.timestamp_transport_strategy
)
if params.auto_mode is not None: if params.auto_mode is not None:
default_settings.auto_mode = params.auto_mode auto_mode = params.auto_mode
_buffer_max_delay_ms = params.max_buffer_delay_ms _buffer_max_delay_ms = params.max_buffer_delay_ms
_buffer_char_threshold = params.buffer_char_threshold _buffer_char_threshold = params.buffer_char_threshold
@@ -673,9 +651,12 @@ class InworldTTSService(WebsocketTTSService):
# Track the end time of the last word in the current generation # Track the end time of the last word in the current generation
self._generation_end_time = 0.0 self._generation_end_time = 0.0
# Init-only audio format config (not runtime-updatable). # Init-only config (not runtime-updatable).
self._audio_encoding = encoding self._audio_encoding = encoding
self._audio_sample_rate = 0 # Set in start() self._audio_sample_rate = 0 # Set in start()
self._auto_mode = auto_mode
self._apply_text_normalization = apply_text_normalization
self._timestamp_transport_strategy = timestamp_transport_strategy
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.
@@ -926,7 +907,7 @@ class InworldTTSService(WebsocketTTSService):
for k in ["contextCreated", "audioChunk", "flushCompleted", "contextClosed"] for k in ["contextCreated", "audioChunk", "flushCompleted", "contextClosed"]
if k in result if k in result
] ]
logger.debug(f"{self}: Received message types={msg_types}, ctx_id={ctx_id}") logger.trace(f"{self}: Received message types={msg_types}, ctx_id={ctx_id}")
# Check for errors # Check for errors
status = result.get("status", {}) status = result.get("status", {})
@@ -1036,14 +1017,12 @@ class InworldTTSService(WebsocketTTSService):
if self._settings.temperature is not None: if self._settings.temperature is not None:
create_config["temperature"] = self._settings.temperature create_config["temperature"] = self._settings.temperature
if self._settings.apply_text_normalization is not None: if self._apply_text_normalization is not None:
create_config["applyTextNormalization"] = self._settings.apply_text_normalization create_config["applyTextNormalization"] = self._apply_text_normalization
if self._settings.auto_mode is not None: if self._auto_mode is not None:
create_config["autoMode"] = self._settings.auto_mode create_config["autoMode"] = self._auto_mode
if self._settings.timestamp_transport_strategy is not None: if self._timestamp_transport_strategy is not None:
create_config["timestampTransportStrategy"] = ( create_config["timestampTransportStrategy"] = self._timestamp_transport_strategy
self._settings.timestamp_transport_strategy
)
# Set buffer settings for timely audio generation. # Set buffer settings for timely audio generation.
# Use provided values or defaults that work well for streaming LLM output. # Use provided values or defaults that work well for streaming LLM output.

View File

@@ -89,7 +89,7 @@ def language_to_kokoro_language(language: Language) -> str:
@dataclass @dataclass
class KokoroTTSSettings(TTSSettings): class KokoroTTSSettings(TTSSettings):
"""Settings for the Kokoro TTS service.""" """Settings for KokoroTTSService."""
pass pass

View File

@@ -19,7 +19,6 @@ from pipecat.frames.frames import (
Frame, Frame,
StartFrame, StartFrame,
TTSAudioRawFrame, TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame, TTSStoppedFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
@@ -48,6 +47,7 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
The corresponding LMNT language code, or None if not supported. The corresponding LMNT language code, or None if not supported.
""" """
LANGUAGE_MAP = { LANGUAGE_MAP = {
Language.AR: "ar",
Language.DE: "de", Language.DE: "de",
Language.EN: "en", Language.EN: "en",
Language.ES: "es", Language.ES: "es",
@@ -65,6 +65,7 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
Language.TH: "th", Language.TH: "th",
Language.TR: "tr", Language.TR: "tr",
Language.UK: "uk", Language.UK: "uk",
Language.UR: "ur",
Language.VI: "vi", Language.VI: "vi",
Language.ZH: "zh", Language.ZH: "zh",
} }
@@ -74,7 +75,7 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
@dataclass @dataclass
class LmntTTSSettings(TTSSettings): class LmntTTSSettings(TTSSettings):
"""Settings for LMNT TTS service.""" """Settings for LmntTTSService."""
pass pass
@@ -96,6 +97,7 @@ class LmntTTSService(InterruptibleTTSService):
voice_id: Optional[str] = None, voice_id: Optional[str] = None,
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
language: Language = Language.EN, language: Language = Language.EN,
output_format: str = "pcm_s16le",
model: Optional[str] = None, model: Optional[str] = None,
settings: Optional[LmntTTSSettings] = None, settings: Optional[LmntTTSSettings] = None,
**kwargs, **kwargs,
@@ -111,6 +113,8 @@ class LmntTTSService(InterruptibleTTSService):
sample_rate: Audio sample rate. If None, uses default. sample_rate: Audio sample rate. If None, uses default.
language: Language for synthesis. Defaults to English. language: Language for synthesis. Defaults to English.
output_format: Audio output format. One of "pcm_s16le", "pcm_f32le",
"mp3", "ulaw", "webm". Defaults to "pcm_s16le".
model: TTS model to use. model: TTS model to use.
.. deprecated:: 0.0.105 .. deprecated:: 0.0.105
@@ -122,7 +126,7 @@ class LmntTTSService(InterruptibleTTSService):
""" """
# 1. Initialize default_settings with hardcoded defaults # 1. Initialize default_settings with hardcoded defaults
default_settings = LmntTTSSettings( default_settings = LmntTTSSettings(
model="blizzard", model="aurora",
voice=None, voice=None,
language=self.language_to_service_language(language), language=self.language_to_service_language(language),
) )
@@ -135,7 +139,7 @@ class LmntTTSService(InterruptibleTTSService):
_warn_deprecated_param("model", LmntTTSSettings, "model") _warn_deprecated_param("model", LmntTTSSettings, "model")
default_settings.model = model default_settings.model = model
# 3. No params for this service # 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
@@ -151,7 +155,7 @@ class LmntTTSService(InterruptibleTTSService):
) )
self._api_key = api_key self._api_key = api_key
self._output_format = "raw" self._output_format = output_format
self._receive_task = None self._receive_task = None
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:

View File

@@ -87,10 +87,9 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
@dataclass @dataclass
class MiniMaxTTSSettings(TTSSettings): class MiniMaxTTSSettings(TTSSettings):
"""Settings for MiniMax TTS service. """Settings for MiniMaxHttpTTSService.
Parameters: Parameters:
stream: Whether to use streaming mode.
speed: Speech speed (range: 0.5 to 2.0). speed: Speech speed (range: 0.5 to 2.0).
volume: Speech volume (range: 0 to 10). volume: Speech volume (range: 0 to 10).
pitch: Pitch adjustment (range: -12 to 12). pitch: Pitch adjustment (range: -12 to 12).
@@ -101,7 +100,6 @@ class MiniMaxTTSSettings(TTSSettings):
language_boost: Language boost string for multilingual support. language_boost: Language boost string for multilingual support.
""" """
stream: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) volume: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) pitch: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@@ -189,6 +187,7 @@ class MiniMaxHttpTTSService(TTSService):
voice_id: Optional[str] = None, voice_id: Optional[str] = None,
aiohttp_session: aiohttp.ClientSession, aiohttp_session: aiohttp.ClientSession,
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
stream: bool = True,
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
settings: Optional[MiniMaxTTSSettings] = None, settings: Optional[MiniMaxTTSSettings] = None,
**kwargs, **kwargs,
@@ -217,6 +216,7 @@ class MiniMaxHttpTTSService(TTSService):
aiohttp_session: aiohttp.ClientSession for API communication. aiohttp_session: aiohttp.ClientSession for API communication.
sample_rate: Output audio sample rate in Hz. If None, uses pipeline default. sample_rate: Output audio sample rate in Hz. If None, uses pipeline default.
stream: Whether to use streaming mode. Defaults to True.
params: Additional configuration parameters. params: Additional configuration parameters.
.. deprecated:: 0.0.105 .. deprecated:: 0.0.105
@@ -231,7 +231,6 @@ class MiniMaxHttpTTSService(TTSService):
model="speech-02-turbo", model="speech-02-turbo",
voice="Calm_Woman", voice="Calm_Woman",
language=None, language=None,
stream=True,
speed=1.0, speed=1.0,
volume=1.0, volume=1.0,
pitch=0, pitch=0,
@@ -311,6 +310,7 @@ class MiniMaxHttpTTSService(TTSService):
self._api_key = api_key self._api_key = api_key
self._group_id = group_id self._group_id = group_id
self._stream = stream
self._base_url = f"{base_url}?GroupId={group_id}" self._base_url = f"{base_url}?GroupId={group_id}"
self._session = aiohttp_session self._session = aiohttp_session
@@ -392,7 +392,7 @@ class MiniMaxHttpTTSService(TTSService):
# Create payload from settings # Create payload from settings
payload = { payload = {
"stream": self._settings.stream, "stream": self._stream,
"voice_setting": voice_setting, "voice_setting": voice_setting,
"audio_setting": audio_setting, "audio_setting": audio_setting,
"model": self._settings.model, "model": self._settings.model,

View File

@@ -26,12 +26,10 @@ from pipecat.frames.frames import (
EndFrame, EndFrame,
ErrorFrame, ErrorFrame,
Frame, Frame,
InterruptionFrame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
StartFrame, StartFrame,
TTSAudioRawFrame, TTSAudioRawFrame,
TTSSpeakFrame, TTSSpeakFrame,
TTSStartedFrame,
TTSStoppedFrame, TTSStoppedFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
@@ -76,7 +74,7 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
@dataclass @dataclass
class NeuphonicTTSSettings(TTSSettings): class NeuphonicTTSSettings(TTSSettings):
"""Settings for Neuphonic TTS service. """Settings for NeuphonicTTSService and NeuphonicHttpTTSService.
Parameters: Parameters:
speed: Speech speed multiplier. Defaults to 1.0. speed: Speech speed multiplier. Defaults to 1.0.
@@ -487,7 +485,7 @@ class NeuphonicHttpTTSService(TTSService):
default_settings = NeuphonicTTSSettings( default_settings = NeuphonicTTSSettings(
model=None, model=None,
voice=None, voice=None,
language=self.language_to_service_language(Language.EN) or "en", language=self.language_to_service_language(Language.EN),
speed=1.0, speed=1.0,
) )
@@ -501,9 +499,7 @@ class NeuphonicHttpTTSService(TTSService):
_warn_deprecated_param("params", NeuphonicTTSSettings) _warn_deprecated_param("params", NeuphonicTTSSettings)
if not settings: if not settings:
if params.language is not None: if params.language is not None:
default_settings.language = ( default_settings.language = self.language_to_service_language(params.language)
self.language_to_service_language(params.language) or "en"
)
if params.speed is not None: if params.speed is not None:
default_settings.speed = params.speed default_settings.speed = params.speed

View File

@@ -44,7 +44,7 @@ except ModuleNotFoundError as e:
@dataclass @dataclass
class NvidiaTTSSettings(TTSSettings): class NvidiaTTSSettings(TTSSettings):
"""Settings for NVIDIA Riva TTS service. """Settings for NvidiaTTSService.
Parameters: Parameters:
quality: Audio quality setting (0-100). quality: Audio quality setting (0-100).

View File

@@ -62,7 +62,7 @@ VALID_VOICES: Dict[str, ValidVoice] = {
@dataclass @dataclass
class OpenAITTSSettings(TTSSettings): class OpenAITTSSettings(TTSSettings):
"""Settings for OpenAI TTS service. """Settings for OpenAITTSService.
Parameters: Parameters:
instructions: Instructions to guide voice synthesis behavior. instructions: Instructions to guide voice synthesis behavior.

View File

@@ -33,7 +33,7 @@ except ModuleNotFoundError as e:
@dataclass @dataclass
class PiperTTSSettings(TTSSettings): class PiperTTSSettings(TTSSettings):
"""Settings for Piper TTS service.""" """Settings for PiperTTSService."""
pass pass
@@ -82,7 +82,7 @@ class PiperTTSService(TTSService):
_warn_deprecated_param("voice_id", PiperTTSSettings, "voice") _warn_deprecated_param("voice_id", PiperTTSSettings, "voice")
default_settings.voice = voice_id default_settings.voice = voice_id
# 3. No params for this service # 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
@@ -186,7 +186,7 @@ class PiperTTSService(TTSService):
# #
@dataclass @dataclass
class PiperHttpTTSSettings(TTSSettings): class PiperHttpTTSSettings(TTSSettings):
"""Settings for Piper HTTP TTS service.""" """Settings for PiperHttpTTSService."""
pass pass
@@ -232,7 +232,7 @@ class PiperHttpTTSService(TTSService):
_warn_deprecated_param("voice_id", PiperHttpTTSSettings, "voice") _warn_deprecated_param("voice_id", PiperHttpTTSSettings, "voice")
default_settings.voice = voice_id default_settings.voice = voice_id
# 3. No params for this service # 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:

View File

@@ -38,7 +38,7 @@ except ModuleNotFoundError as e:
@dataclass @dataclass
class ResembleAITTSSettings(TTSSettings): class ResembleAITTSSettings(TTSSettings):
"""Settings for Resemble AI TTS service.""" """Settings for ResembleAITTSService."""
pass pass
@@ -94,7 +94,7 @@ class ResembleAITTSService(WebsocketTTSService):
_warn_deprecated_param("voice_id", ResembleAITTSSettings, "voice") _warn_deprecated_param("voice_id", ResembleAITTSSettings, "voice")
default_settings.voice = voice_id default_settings.voice = voice_id
# 3. No params for this service # 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:

View File

@@ -73,7 +73,7 @@ def language_to_rime_language(language: Language) -> str:
@dataclass @dataclass
class RimeTTSSettings(TTSSettings): class RimeTTSSettings(TTSSettings):
"""Settings for Rime WS JSON and HTTP TTS services. """Settings for RimeTTSService and RimeHttpTTSService.
Parameters: Parameters:
segment: Text segmentation mode ("immediate", "bySentence", "never"). segment: Text segmentation mode ("immediate", "bySentence", "never").
@@ -106,7 +106,7 @@ class RimeTTSSettings(TTSSettings):
@dataclass @dataclass
class RimeNonJsonTTSSettings(TTSSettings): class RimeNonJsonTTSSettings(TTSSettings):
"""Settings for Rime non-JSON WS TTS service. """Settings for RimeNonJsonTTSService.
Parameters: Parameters:
segment: Text segmentation mode ("immediate", "bySentence", "never"). segment: Text segmentation mode ("immediate", "bySentence", "never").

View File

@@ -400,12 +400,13 @@ class SarvamSTTService(STTService):
changed = await super()._update_settings(delta) changed = await super()._update_settings(delta)
# Prompt is a WebSocket connect-time parameter; reconnect to apply. # Language and prompt are WebSocket connect-time parameters; reconnect to apply.
if "prompt" in changed: reconnect_fields = {"language", "prompt"}
if changed.keys() & reconnect_fields:
await self._disconnect() await self._disconnect()
await self._connect() await self._connect()
unhandled = {k: v for k, v in changed.items() if k != "prompt"} unhandled = {k: v for k, v in changed.items() if k not in reconnect_fields}
if unhandled: if unhandled:
self._warn_unhandled_updated_settings(unhandled) self._warn_unhandled_updated_settings(unhandled)
@@ -483,7 +484,6 @@ class SarvamSTTService(STTService):
Frame: None (transcription results come via WebSocket callbacks). Frame: None (transcription results come via WebSocket callbacks).
""" """
if not self._socket_client: if not self._socket_client:
logger.warning("WebSocket not connected, cannot process audio")
yield None yield None
return return
@@ -636,18 +636,22 @@ class SarvamSTTService(STTService):
await self.cancel_task(self._receive_task) await self.cancel_task(self._receive_task)
self._receive_task = None self._receive_task = None
if self._websocket_context and self._socket_client: # Clear references first to prevent run_stt from sending audio
# during the close handshake.
socket_client = self._socket_client
websocket_context = self._websocket_context
self._socket_client = None
self._websocket_context = None
if websocket_context and socket_client:
try: try:
# Exit the async context manager await websocket_context.__aexit__(None, None, None)
await self._websocket_context.__aexit__(None, None, None)
except Exception as e: except Exception as e:
await self.push_error( await self.push_error(
error_msg=f"Error closing WebSocket connection: {e}", exception=e error_msg=f"Error closing WebSocket connection: {e}", exception=e
) )
finally: finally:
logger.debug("Disconnected from Sarvam WebSocket") logger.debug("Disconnected from Sarvam WebSocket")
self._socket_client = None
self._websocket_context = None
async def _receive_task_handler(self): async def _receive_task_handler(self):
"""Handle incoming messages from Sarvam WebSocket. """Handle incoming messages from Sarvam WebSocket.

View File

@@ -53,11 +53,9 @@ from pipecat.frames.frames import (
EndFrame, EndFrame,
ErrorFrame, ErrorFrame,
Frame, Frame,
InterruptionFrame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
StartFrame, StartFrame,
TTSAudioRawFrame, TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame, TTSStoppedFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
@@ -230,16 +228,27 @@ def language_to_sarvam_language(language: Language) -> Optional[str]:
""" """
LANGUAGE_MAP = { LANGUAGE_MAP = {
Language.BN: "bn-IN", # Bengali Language.BN: "bn-IN", # Bengali
Language.BN_IN: "bn-IN",
Language.EN: "en-IN", # English (India) Language.EN: "en-IN", # English (India)
Language.EN_IN: "en-IN",
Language.GU: "gu-IN", # Gujarati Language.GU: "gu-IN", # Gujarati
Language.GU_IN: "gu-IN",
Language.HI: "hi-IN", # Hindi Language.HI: "hi-IN", # Hindi
Language.HI_IN: "hi-IN",
Language.KN: "kn-IN", # Kannada Language.KN: "kn-IN", # Kannada
Language.KN_IN: "kn-IN",
Language.ML: "ml-IN", # Malayalam Language.ML: "ml-IN", # Malayalam
Language.ML_IN: "ml-IN",
Language.MR: "mr-IN", # Marathi Language.MR: "mr-IN", # Marathi
Language.MR_IN: "mr-IN",
Language.OR: "od-IN", # Odia Language.OR: "od-IN", # Odia
Language.OR_IN: "od-IN",
Language.PA: "pa-IN", # Punjabi Language.PA: "pa-IN", # Punjabi
Language.PA_IN: "pa-IN",
Language.TA: "ta-IN", # Tamil Language.TA: "ta-IN", # Tamil
Language.TA_IN: "ta-IN",
Language.TE: "te-IN", # Telugu Language.TE: "te-IN", # Telugu
Language.TE_IN: "te-IN",
} }
return resolve_language(language, LANGUAGE_MAP, use_base_code=False) return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@@ -247,7 +256,7 @@ def language_to_sarvam_language(language: Language) -> Optional[str]:
@dataclass @dataclass
class SarvamHttpTTSSettings(TTSSettings): class SarvamHttpTTSSettings(TTSSettings):
"""Settings for Sarvam HTTP TTS service. """Settings for SarvamHttpTTSService.
Parameters: Parameters:
enable_preprocessing: Whether to enable text preprocessing. Defaults to False. enable_preprocessing: Whether to enable text preprocessing. Defaults to False.
@@ -273,7 +282,7 @@ class SarvamHttpTTSSettings(TTSSettings):
@dataclass @dataclass
class SarvamTTSSettings(SarvamHttpTTSSettings): class SarvamTTSSettings(SarvamHttpTTSSettings):
"""Settings for Sarvam WebSocket TTS service. """Settings for SarvamTTSService.
Extends :class:`SarvamHttpTTSSettings` with WebSocket-specific buffering parameters. Extends :class:`SarvamHttpTTSSettings` with WebSocket-specific buffering parameters.
@@ -481,6 +490,10 @@ class SarvamHttpTTSService(TTSService):
if settings is not None: if settings is not None:
default_settings.apply_update(settings) default_settings.apply_update(settings)
# Convert Language enum to service-specific string
if isinstance(default_settings.language, Language):
default_settings.language = self.language_to_service_language(default_settings.language)
# Get model configuration (validates model exists) # Get model configuration (validates model exists)
resolved_model = default_settings.model resolved_model = default_settings.model
if resolved_model not in TTS_MODEL_CONFIGS: if resolved_model not in TTS_MODEL_CONFIGS:
@@ -900,6 +913,10 @@ class SarvamTTSService(InterruptibleTTSService):
if settings is not None: if settings is not None:
default_settings.apply_update(settings) default_settings.apply_update(settings)
# Convert Language enum to service-specific string
if isinstance(default_settings.language, Language):
default_settings.language = self.language_to_service_language(default_settings.language)
# Get model configuration (validates model exists) # Get model configuration (validates model exists)
resolved_model = default_settings.model resolved_model = default_settings.model
if resolved_model not in TTS_MODEL_CONFIGS: if resolved_model not in TTS_MODEL_CONFIGS:

View File

@@ -297,9 +297,7 @@ class SonioxSTTService(WebsocketSTTService):
await self._connect() await self._connect()
async def _update_settings(self, delta: SonioxSTTSettings) -> dict[str, Any]: async def _update_settings(self, delta: SonioxSTTSettings) -> dict[str, Any]:
"""Apply settings delta. """Apply settings delta and reconnect if anything changed.
Settings are stored but not applied to the active connection.
Args: Args:
delta: A settings delta. delta: A settings delta.
@@ -309,15 +307,9 @@ class SonioxSTTService(WebsocketSTTService):
""" """
changed = await super()._update_settings(delta) changed = await super()._update_settings(delta)
if not changed: if changed:
return changed await self._disconnect()
await self._connect()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed return changed

View File

@@ -37,7 +37,7 @@ except ModuleNotFoundError as e:
@dataclass @dataclass
class SpeechmaticsTTSSettings(TTSSettings): class SpeechmaticsTTSSettings(TTSSettings):
"""Settings for Speechmatics TTS service. """Settings for SpeechmaticsTTSService.
Parameters: Parameters:
max_retries: Maximum number of retries for HTTP requests. max_retries: Maximum number of retries for HTTP requests.

View File

@@ -70,7 +70,7 @@ def language_to_xtts_language(language: Language) -> Optional[str]:
@dataclass @dataclass
class XTTSTTSSettings(TTSSettings): class XTTSTTSSettings(TTSSettings):
"""Settings for XTTS TTS service.""" """Settings for XTTSService."""
pass pass
@@ -124,6 +124,8 @@ class XTTSService(TTSService):
_warn_deprecated_param("voice_id", XTTSTTSSettings, "voice") _warn_deprecated_param("voice_id", XTTSTTSSettings, "voice")
default_settings.voice = voice_id default_settings.voice = voice_id
# 3. (No step 3, as there's no params object to apply)
# 4. Apply settings delta (canonical API, always wins) # 4. Apply settings delta (canonical API, always wins)
if settings is not None: if settings is not None:
default_settings.apply_update(settings) default_settings.apply_update(settings)

View File

@@ -34,7 +34,6 @@ new services are covered automatically with no per-service maintenance.
import importlib import importlib
import inspect import inspect
import pkgutil import pkgutil
import warnings
from dataclasses import fields from dataclasses import fields
import pytest import pytest