wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3834 from pipecat-ai/pk/make-ai-service-exclusive-syncer-of-model-name-to-metrics

Browse Source 
Make it so that `AIService` is the exclusive "syncer" of model name t…
This commit is contained in:
kompfner
2026-02-25 15:53:59 -05:00
committed by GitHub
parent 937c691f2a bca42f7d68
commit b1df1ba5d4
72 changed files with 1207 additions and 1023 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
COMMUNITY_INTEGRATIONS.md
View File

@@ -257,15 +257,16 @@ The service stores its current settings in `self._settings` and declares the typ
```python
class MySTTService(STTService):
_settings: MySTTSettings
def __init__(self, *, model: str, language: str, region: str, **kwargs):
super().__init__(**kwargs)
# Initial value must be provided for every field in self._settings
# before service is started
self._settings = MySTTSettings(model=model, language=language, region=region)
self._sync_model_name_to_metrics()
# An initial value should be provided for every settings field.
# This will be validated at service start.
# (If you track sample_rate, it can be a placeholder value like 0; see
# "Sample Rate Handling").
super().__init__(
settings=MySTTSettings(model=model, language=language, region=region), **kwargs
)
```
To react to runtime setting changes, override `_update_settings`. The base implementation applies the delta to `self._settings` and returns a `dict` mapping each changed field name to its **pre-update** value. Your override should call `super()` first, then act on the changed fields. A common implementation might look like:

6
examples/foundational/55zg-update-settings-hathora-tts.py
View File

@@ -54,7 +54,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
tts = HathoraTTSService(
api_key=os.getenv("HATHORA_API_KEY"),
model="hathora-ai/polar",
model="hexgrad-kokoro-82m",
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
@@ -100,8 +100,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
await task.queue_frames([LLMRunFrame()])
await asyncio.sleep(10)
logger.info("Updating Hathora TTS settings: speed=1.3")
await task.queue_frame(TTSUpdateSettingsFrame(delta=HathoraTTSSettings(speed=1.3)))
logger.info("Updating Hathora TTS settings: speed=1.5")
await task.queue_frame(TTSUpdateSettingsFrame(delta=HathoraTTSSettings(speed=1.5)))
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):

4
examples/foundational/55zs-update-settings-hathora-stt.py
View File

@@ -52,7 +52,9 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
stt = HathoraSTTService(api_key=os.getenv("HATHORA_API_KEY"), model="deepgram-nova3")
stt = HathoraSTTService(
api_key=os.getenv("HATHORA_API_KEY"), model="nvidia-parakeet-tdt-0.6b-v3"
)
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),

20
src/pipecat/services/ai_service.py
View File

@@ -35,14 +35,21 @@ class AIService(FrameProcessor):
this base infrastructure.
"""
def __init__(self, **kwargs):
def __init__(self, settings: ServiceSettings | None = None, **kwargs):
"""Initialize the AI service.
Args:
settings: The runtime-updatable settings for the AI service.
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
self._settings: ServiceSettings = ServiceSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._settings: ServiceSettings = (
settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ServiceSettings()
)
self._sync_model_name_to_metrics()
self._session_properties: Dict[str, Any] = {}
self._tracing_enabled: bool = False
self._tracing_context = None
@@ -54,15 +61,12 @@ class AIService(FrameProcessor):
of truth for it in `self._settings.model`. This method is just for
syncing the model name to the metrics data.
TODO: as a next step we should make it so that service classes pass
model into `super().__init__` and `AIService` can be responsible for
syncing its initial value to metrics, just as it's responsible for
syncing any updates to its value to metrics via `_update_settings`.
Args:
model: The name of the AI model to use.
"""
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._settings.model))
self.set_core_metrics_data(
MetricsData(processor=self.name, model=self._settings.model or "")
)
async def start(self, frame: StartFrame):
"""Start the AI service.

52
src/pipecat/services/anthropic/llm.py
View File

@@ -232,37 +232,39 @@ class AnthropicLLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AnthropicLLMService.InputParams()
super().__init__(
settings=AnthropicLLMSettings(
model=model,
max_tokens=params.max_tokens,
enable_prompt_caching=(
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._client = client or AsyncAnthropic(
api_key=api_key
) # if the client is provided, use it and remove it, otherwise create a new one
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = AnthropicLLMSettings(
model=model,
max_tokens=params.max_tokens,
enable_prompt_caching=(
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate usage metrics.

14
src/pipecat/services/assemblyai/stt.py
View File

@@ -111,15 +111,17 @@ class AssemblyAISTTService(WebsocketSTTService):
connection_params = self._configure_manual_turn_mode(connection_params)
super().__init__(
sample_rate=connection_params.sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs
sample_rate=connection_params.sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AssemblyAISTTSettings(
model=None,
language=language,
connection_params=connection_params,
),
**kwargs,
)
self._api_key = api_key
self._settings = AssemblyAISTTSettings(
model=None,
language=language,
connection_params=connection_params,
)
self._api_endpoint_base_url = api_endpoint_base_url
self._vad_force_turn_endpoint = vad_force_turn_endpoint

53
src/pipecat/services/asyncai/tts.py
View File

@@ -147,30 +147,29 @@ class AsyncAITTSService(AudioContextTTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to the parent service.
"""
params = params or AsyncAITTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs,
)
params = params or AsyncAITTSService.InputParams()
self._api_key = api_key
self._api_version = version
self._url = url
self._settings = AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._keepalive_task = None
@@ -501,24 +500,26 @@ class AsyncAIHttpTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AsyncAIHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs,
)
self._api_key = api_key
self._base_url = url
self._api_version = version
self._settings = AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
)
self._sync_model_name_to_metrics()
self._session = aiohttp_session

39
src/pipecat/services/aws/llm.py
View File

@@ -797,10 +797,28 @@ class AWSBedrockLLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AWSBedrockLLMService.InputParams()
super().__init__(
settings=AWSBedrockLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
latency=params.latency,
additional_model_request_fields=params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
),
**kwargs,
)
# Initialize the AWS Bedrock client
if not client_config:
client_config = Config(
@@ -822,23 +840,6 @@ class AWSBedrockLLMService(LLMService):
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = AWSBedrockLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
latency=params.latency,
additional_model_request_fields=params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
)
self._sync_model_name_to_metrics()
logger.info(f"Using AWS Bedrock model: {model}")

36
src/pipecat/services/aws/nova_sonic/llm.py
View File

@@ -254,28 +254,30 @@ class AWSNovaSonicLLMService(LLMService):
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs)
params = params or Params()
super().__init__(
settings=AWSNovaSonicLLMSettings(
model=model,
voice_id=voice_id,
temperature=params.temperature,
max_tokens=params.max_tokens,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
endpointing_sensitivity=params.endpointing_sensitivity,
),
**kwargs,
)
self._secret_access_key = secret_access_key
self._access_key_id = access_key_id
self._session_token = session_token
self._region = region
self._client: Optional[BedrockRuntimeClient] = None
params = params or Params()
self._settings = AWSNovaSonicLLMSettings(
model=model,
voice_id=voice_id,
temperature=params.temperature,
max_tokens=params.max_tokens,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
endpointing_sensitivity=params.endpointing_sensitivity,
)
self._sync_model_name_to_metrics()
# Audio I/O config (hardware settings, not runtime-tunable)
self._input_sample_rate = params.input_sample_rate

20
src/pipecat/services/aws/stt.py
View File

@@ -99,15 +99,17 @@ class AWSTranscribeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._settings = AWSTranscribeSTTSettings(
language=self.language_to_service_language(language) or "en-US",
sample_rate=sample_rate,
media_encoding="linear16",
number_of_channels=1,
show_speaker_label=False,
enable_channel_identification=False,
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=AWSTranscribeSTTSettings(
language=self.language_to_service_language(language) or "en-US",
sample_rate=sample_rate,
media_encoding="linear16",
number_of_channels=1,
show_speaker_label=False,
enable_channel_identification=False,
),
**kwargs,
)
# Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz

31
src/pipecat/services/aws/tts.py
View File

@@ -195,10 +195,25 @@ class AWSPollyTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AWSPollyTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AWSPollyTTSSettings(
model=None,
voice=voice_id,
engine=params.engine,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
volume=params.volume,
lexicon_names=params.lexicon_names,
),
**kwargs,
)
# Get credentials from environment variables if not provided
self._aws_params = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -208,18 +223,6 @@ class AWSPollyTTSService(TTSService):
}
self._aws_session = aioboto3.Session()
self._settings = AWSPollyTTSSettings(
model=None,
voice=voice_id,
engine=params.engine,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
volume=params.volume,
lexicon_names=params.lexicon_names,
)
self._resampler = create_stream_resampler()

15
src/pipecat/services/azure/image.py
View File

@@ -12,6 +12,7 @@ using REST endpoints for creating images from text prompts.
import asyncio
import io
from dataclasses import dataclass
from typing import AsyncGenerator
import aiohttp
@@ -19,6 +20,16 @@ from PIL import Image
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class AzureImageGenSettings(ImageGenSettings):
"""Settings for the Azure image generation service.
Parameters:
model: Azure image generation model identifier.
"""
class AzureImageGenServiceREST(ImageGenService):
@@ -49,13 +60,11 @@ class AzureImageGenServiceREST(ImageGenService):
aiohttp_session: Shared aiohttp session for HTTP requests.
api_version: Azure API version string. Defaults to "2023-06-01-preview".
"""
super().__init__()
super().__init__(settings=AzureImageGenSettings(model=model))
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self._settings.model = model
self._sync_model_name_to_metrics()
self._image_size = image_size
self._aiohttp_session = aiohttp_session

18
src/pipecat/services/azure/stt.py
View File

@@ -96,7 +96,17 @@ class AzureSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AzureSTTSettings(
model=None,
region=region,
language=language_to_azure_language(language),
sample_rate=sample_rate,
),
**kwargs,
)
self._speech_config = SpeechConfig(
subscription=api_key,
@@ -109,12 +119,6 @@ class AzureSTTService(STTService):
self._audio_stream = None
self._speech_recognizer = None
self._settings = AzureSTTSettings(
model=None,
region=region,
language=language_to_azure_language(language),
sample_rate=sample_rate,
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate performance metrics.

60
src/pipecat/services/azure/tts.py
View File

@@ -141,7 +141,6 @@ class AzureBaseTTSService:
api_key: str,
region: str,
voice: str = "en-US-SaraNeural",
params: Optional[InputParams] = None,
):
"""Initialize Azure-specific configuration.
@@ -151,25 +150,7 @@ class AzureBaseTTSService:
api_key: Azure Cognitive Services subscription key.
region: Azure region identifier (e.g., "eastus", "westus2").
voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
params: Voice and synthesis parameters configuration.
"""
params = params or AzureBaseTTSService.InputParams()
self._settings = AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
)
self._api_key = api_key
self._region = region
self._speech_synthesizer = None
@@ -289,6 +270,8 @@ class AzureTTSService(TTSService, AzureBaseTTSService):
aggregate_sentences: Whether to aggregate sentences before synthesis.
**kwargs: Additional arguments passed to the parent TTSService.
"""
params = params or AzureBaseTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=False, # We'll push text frames based on word timestamps
@@ -296,11 +279,25 @@ class AzureTTSService(TTSService, AzureBaseTTSService):
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs,
)
# Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None
self._speech_synthesizer = None
@@ -734,10 +731,29 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
params: Voice and synthesis parameters configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AzureBaseTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs,
)
# Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None
self._speech_synthesizer = None

28
src/pipecat/services/camb/tts.py
View File

@@ -213,11 +213,6 @@ class CambTTSService(TTSService):
params: Additional voice parameters. If None, uses defaults.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._timeout = timeout
params = params or CambTTSService.InputParams()
# Warn if sample rate doesn't match model's supported rate
@@ -227,16 +222,23 @@ class CambTTSService(TTSService):
f"sample rate. Current rate of {sample_rate}Hz may cause issues."
)
# Build settings
self._settings = CambTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else "en-us"
super().__init__(
sample_rate=sample_rate,
settings=CambTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language)
if params.language
else "en-us"
),
user_instructions=params.user_instructions,
),
user_instructions=params.user_instructions,
**kwargs,
)
self._sync_model_name_to_metrics()
self._api_key = api_key
self._timeout = timeout
self._client = None

24
src/pipecat/services/cartesia/stt.py
View File

@@ -173,13 +173,6 @@ class CartesiaSTTService(WebsocketSTTService):
**kwargs: Additional arguments passed to parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=120,
keepalive_interval=30,
**kwargs,
)
default_options = CartesiaLiveOptions(
model="ink-whisper",
@@ -196,12 +189,19 @@ class CartesiaSTTService(WebsocketSTTService):
k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None"
}
self._settings = CartesiaSTTSettings(
model=merged_options["model"],
language=merged_options.get("language"),
encoding=merged_options.get("encoding", "pcm_s16le"),
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=120,
keepalive_interval=30,
settings=CartesiaSTTSettings(
model=merged_options["model"],
language=merged_options.get("language"),
encoding=merged_options.get("encoding", "pcm_s16le"),
),
**kwargs,
)
self._sync_model_name_to_metrics()
self._api_key = api_key
self._base_url = base_url or "api.cartesia.ai"
self._receive_task = None

69
src/pipecat/services/cartesia/tts.py
View File

@@ -305,6 +305,8 @@ class CartesiaTTSService(AudioContextTTSService):
# if we're interrupted. Cartesia gives us word-by-word timestamps. We
# can use those to generate text frames ourselves aligned with the
# playout timing of the audio!
params = params or CartesiaTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
@@ -312,6 +314,20 @@ class CartesiaTTSService(AudioContextTTSService):
supports_word_timestamps=True,
sample_rate=sample_rate,
text_aggregator=text_aggregator,
settings=CartesiaTTSSettings(
model=model,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
voice=voice_id,
),
**kwargs,
)
@@ -323,26 +339,9 @@ class CartesiaTTSService(AudioContextTTSService):
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("<spell>", "</spell>")])
params = params or CartesiaTTSService.InputParams()
self._api_key = api_key
self._cartesia_version = cartesia_version
self._url = url
self._settings = CartesiaTTSSettings(
model=model,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
voice=voice_id,
)
self._sync_model_name_to_metrics()
self._receive_task = None
@@ -727,28 +726,30 @@ class CartesiaHttpTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or CartesiaHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=CartesiaTTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._cartesia_version = cartesia_version
self._settings = CartesiaTTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
)
self._sync_model_name_to_metrics()
self._client = AsyncCartesia(
api_key=api_key,

28
src/pipecat/services/deepgram/flux/stt.py
View File

@@ -207,26 +207,24 @@ class DeepgramFluxSTTService(WebsocketSTTService):
# was never destroyed.
# So we can keep it here as false, because inside the method send_with_retry, it will
# already try to reconnect if needed.
params = params or DeepgramFluxSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
reconnect_on_error=False,
settings=DeepgramFluxSTTSettings(
model=model,
language=Language.EN,
encoding=flux_encoding,
eager_eot_threshold=params.eager_eot_threshold,
eot_threshold=params.eot_threshold,
eot_timeout_ms=params.eot_timeout_ms,
keyterm=params.keyterm or [],
mip_opt_out=params.mip_opt_out,
tag=params.tag or [],
min_confidence=params.min_confidence,
),
**kwargs,
)
params = params or DeepgramFluxSTTService.InputParams()
self._settings = DeepgramFluxSTTSettings(
model=model,
language=Language.EN,
encoding=flux_encoding,
eager_eot_threshold=params.eager_eot_threshold,
eot_threshold=params.eot_threshold,
eot_timeout_ms=params.eot_timeout_ms,
keyterm=params.keyterm or [],
mip_opt_out=params.mip_opt_out,
tag=params.tag or [],
min_confidence=params.min_confidence,
)
self._sync_model_name_to_metrics()
self._api_key = api_key
self._url = url
self._should_interrupt = should_interrupt

15
src/pipecat/services/deepgram/stt.py
View File

@@ -117,7 +117,6 @@ class DeepgramSTTService(STTService):
The `vad_events` option in LiveOptions is deprecated as of version 0.0.99 and will be removed in a future version. Please use the Silero VAD instead.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if url:
import warnings
@@ -155,12 +154,16 @@ class DeepgramSTTService(STTService):
merged_options["language"] = merged_options["language"].value
merged_live_options = LiveOptions(**merged_options)
self._settings = DeepgramSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=DeepgramSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
),
**kwargs,
)
self._sync_model_name_to_metrics()
self._addons = addons
self._should_interrupt = should_interrupt

21
src/pipecat/services/deepgram/stt_sagemaker.py
View File

@@ -115,10 +115,6 @@ class DeepgramSageMakerSTTService(STTService):
**kwargs: Additional arguments passed to the parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._endpoint_name = endpoint_name
self._region = region
# Create default options similar to DeepgramSTTService
default_options = LiveOptions(
@@ -144,12 +140,19 @@ class DeepgramSageMakerSTTService(STTService):
merged_options["language"] = merged_options["language"].value
merged_live_options = LiveOptions(**merged_options)
self._settings = DeepgramSageMakerSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=DeepgramSageMakerSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
),
**kwargs,
)
self._sync_model_name_to_metrics()
self._endpoint_name = endpoint_name
self._region = region
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None

31
src/pipecat/services/deepgram/tts.py
View File

@@ -101,18 +101,17 @@ class DeepgramTTSService(WebsocketTTSService):
pause_frame_processing=True,
push_stop_frames=True,
append_trailing_space=True,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._settings = DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._context_id: Optional[str] = None
@@ -394,18 +393,20 @@ class DeepgramHttpTTSService(TTSService):
encoding: Audio encoding format. Defaults to "linear16".
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
super().__init__(
sample_rate=sample_rate,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = base_url
self._settings = DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.

13
src/pipecat/services/deepgram/tts_sagemaker.py
View File

@@ -99,18 +99,17 @@ class DeepgramSageMakerTTSService(TTSService):
push_stop_frames=True,
pause_frame_processing=True,
append_trailing_space=True,
settings=DeepgramSageMakerTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._endpoint_name = endpoint_name
self._region = region
self._settings = DeepgramSageMakerTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
)
self._sync_model_name_to_metrics()
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None

50
src/pipecat/services/elevenlabs/stt.py
View File

@@ -261,28 +261,26 @@ class ElevenLabsSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
params = params or ElevenLabsSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=ElevenLabsSTTSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
tag_audio_events=params.tag_audio_events,
),
**kwargs,
)
params = params or ElevenLabsSTTService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
self._model_id = model
self._settings = ElevenLabsSTTSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
tag_audio_events=params.tag_audio_events,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -500,16 +498,28 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to WebsocketSTTService.
"""
params = params or ElevenLabsRealtimeSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=10,
keepalive_interval=5,
settings=ElevenLabsRealtimeSTTSettings(
model=model,
language=params.language_code,
commit_strategy=params.commit_strategy,
vad_silence_threshold_secs=params.vad_silence_threshold_secs,
vad_threshold=params.vad_threshold,
min_speech_duration_ms=params.min_speech_duration_ms,
min_silence_duration_ms=params.min_silence_duration_ms,
include_timestamps=params.include_timestamps,
enable_logging=params.enable_logging,
include_language_detection=params.include_language_detection,
),
**kwargs,
)
params = params or ElevenLabsRealtimeSTTService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._model_id = model
@@ -519,20 +529,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
self._connected_event = asyncio.Event()
self._connected_event.set()
self._settings = ElevenLabsRealtimeSTTSettings(
model=model,
language=params.language_code,
commit_strategy=params.commit_strategy,
vad_silence_threshold_secs=params.vad_silence_threshold_secs,
vad_threshold=params.vad_threshold,
min_speech_duration_ms=params.min_speech_duration_ms,
min_silence_duration_ms=params.min_silence_duration_ms,
include_timestamps=params.include_timestamps,
enable_logging=params.enable_logging,
include_language_detection=params.include_language_detection,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.

71
src/pipecat/services/elevenlabs/tts.py
View File

@@ -394,6 +394,8 @@ class ElevenLabsTTSService(AudioContextTTSService):
# Finally, ElevenLabs doesn't provide information on when the bot stops
# speaking for a while, so we want the parent class to send TTSStopFrame
# after a short period not receiving any audio.
params = params or ElevenLabsTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
@@ -401,30 +403,27 @@ class ElevenLabsTTSService(AudioContextTTSService):
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=ElevenLabsTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else None
),
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
auto_mode=str(params.auto_mode).lower(),
enable_ssml_parsing=params.enable_ssml_parsing,
enable_logging=params.enable_logging,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs,
)
params = params or ElevenLabsTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = ElevenLabsTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else None
),
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
auto_mode=str(params.auto_mode).lower(),
enable_ssml_parsing=params.enable_ssml_parsing,
enable_logging=params.enable_logging,
apply_text_normalization=params.apply_text_normalization,
)
self._sync_model_name_to_metrics()
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
@@ -910,37 +909,35 @@ class ElevenLabsHttpTTSService(TTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to the parent service.
"""
params = params or ElevenLabsHttpTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=ElevenLabsHttpTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(params.language)
if params.language
else None,
optimize_streaming_latency=params.optimize_streaming_latency,
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs,
)
params = params or ElevenLabsHttpTTSService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._params = params
self._session = aiohttp_session
self._settings = ElevenLabsHttpTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(params.language)
if params.language
else None,
optimize_streaming_latency=params.optimize_streaming_latency,
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
apply_text_normalization=params.apply_text_normalization,
)
self._sync_model_name_to_metrics()
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators

15
src/pipecat/services/fal/image.py
View File

@@ -13,6 +13,7 @@ for creating images from text prompts using various AI models.
import asyncio
import io
import os
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Optional, Union
import aiohttp
@@ -22,6 +23,7 @@ from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try:
import fal_client
@@ -31,6 +33,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class FalImageGenSettings(ImageGenSettings):
"""Settings for the Fal image generation service.
Parameters:
model: Fal.ai model identifier.
"""
class FalImageGenService(ImageGenService):
"""Fal's image generation service.
@@ -77,9 +88,7 @@ class FalImageGenService(ImageGenService):
key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable.
**kwargs: Additional arguments passed to parent ImageGenService.
"""
super().__init__(**kwargs)
self._settings.model = model
self._sync_model_name_to_metrics()
super().__init__(settings=FalImageGenSettings(model=model), **kwargs)
self._params = params
self._aiohttp_session = aiohttp_session
if key:

22
src/pipecat/services/fal/stt.py
View File

@@ -207,14 +207,23 @@ class FalSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
params = params or FalSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=FalSTTSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en",
task=params.task,
chunk_level=params.chunk_level,
version=params.version,
),
**kwargs,
)
params = params or FalSTTService.InputParams()
if api_key:
os.environ["FAL_KEY"] = api_key
elif "FAL_KEY" not in os.environ:
@@ -223,15 +232,6 @@ class FalSTTService(SegmentedSTTService):
)
self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY"))
self._settings = FalSTTSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en",
task=params.task,
chunk_level=params.chunk_level,
version=params.version,
)
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.

38
src/pipecat/services/fish/tts.py
View File

@@ -138,13 +138,6 @@ class FishAudioTTSService(InterruptibleTTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent service.
"""
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or FishAudioTTSService.InputParams()
# Validation for model and reference_id parameters
@@ -169,25 +162,30 @@ class FishAudioTTSService(InterruptibleTTSService):
)
reference_id = model
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
settings=FishAudioTTSSettings(
model=model_id,
voice=reference_id,
fish_sample_rate=0,
latency=params.latency,
format=output_format,
normalize=params.normalize,
prosody_speed=params.prosody_speed,
prosody_volume=params.prosody_volume,
reference_id=reference_id,
),
**kwargs,
)
self._api_key = api_key
self._base_url = "wss://api.fish.audio/v1/tts/live"
self._websocket = None
self._receive_task = None
self._request_id = None
self._settings = FishAudioTTSSettings(
model=model_id,
voice=reference_id,
fish_sample_rate=0,
latency=params.latency,
format=output_format,
normalize=params.normalize,
prosody_speed=params.prosody_speed,
prosody_volume=params.prosody_volume,
reference_id=reference_id,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

54
src/pipecat/services/gladia/stt.py
View File

@@ -278,14 +278,6 @@ class GladiaSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService parent class.
"""
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=20,
keepalive_interval=5,
**kwargs,
)
params = params or GladiaInputParams()
if params.language is not None:
@@ -308,11 +300,6 @@ class GladiaSTTService(WebsocketSTTService):
stacklevel=2,
)
self._api_key = api_key
self._region = region
self._url = url
self._receive_task = None
# Resolve deprecated language → language_config at init time
language_config = params.language_config
if not language_config and params.language:
@@ -320,22 +307,33 @@ class GladiaSTTService(WebsocketSTTService):
if language_code:
language_config = LanguageConfig(languages=[language_code], code_switching=False)
self._settings = GladiaSTTSettings(
model=model,
language=None,
encoding=params.encoding,
bit_depth=params.bit_depth,
channels=params.channels,
custom_metadata=params.custom_metadata,
endpointing=params.endpointing,
maximum_duration_without_endpointing=params.maximum_duration_without_endpointing,
language_config=language_config,
pre_processing=params.pre_processing,
realtime_processing=params.realtime_processing,
messages_config=params.messages_config,
enable_vad=params.enable_vad,
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=20,
keepalive_interval=5,
settings=GladiaSTTSettings(
model=model,
language=None,
encoding=params.encoding,
bit_depth=params.bit_depth,
channels=params.channels,
custom_metadata=params.custom_metadata,
endpointing=params.endpointing,
maximum_duration_without_endpointing=params.maximum_duration_without_endpointing,
language_config=language_config,
pre_processing=params.pre_processing,
realtime_processing=params.realtime_processing,
messages_config=params.messages_config,
enable_vad=params.enable_vad,
),
**kwargs,
)
self._sync_model_name_to_metrics()
self._api_key = api_key
self._region = region
self._url = url
self._receive_task = None
# Session management
self._session_url = None

57
src/pipecat/services/google/gemini_live/llm.py
View File

@@ -695,10 +695,38 @@ class GeminiLiveLLMService(LLMService):
stacklevel=2,
)
super().__init__(base_url=base_url, **kwargs)
params = params or InputParams()
super().__init__(
base_url=base_url,
settings=GeminiLiveLLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
max_tokens=params.max_tokens,
presence_penalty=params.presence_penalty,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
modalities=params.modalities,
language=language_to_gemini_language(params.language)
if params.language
else "en-US",
media_resolution=params.media_resolution,
vad=params.vad,
context_window_compression=params.context_window_compression.model_dump()
if params.context_window_compression
else {},
thinking=params.thinking or {},
enable_affective_dialog=params.enable_affective_dialog or False,
proactivity=params.proactivity or {},
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._last_sent_time = 0
self._base_url = base_url
self._voice_id = voice_id
@@ -742,31 +770,6 @@ class GeminiLiveLLMService(LLMService):
self._consecutive_failures = 0
self._connection_start_time = None
self._settings = GeminiLiveLLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
max_tokens=params.max_tokens,
presence_penalty=params.presence_penalty,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
modalities=params.modalities,
language=self._language_code,
media_resolution=params.media_resolution,
vad=params.vad,
context_window_compression=params.context_window_compression.model_dump()
if params.context_window_compression
else {},
thinking=params.thinking or {},
enable_affective_dialog=params.enable_affective_dialog or False,
proactivity=params.proactivity or {},
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
self._file_api_base_url = file_api_base_url
self._file_api: Optional[GeminiFileAPI] = None

19
src/pipecat/services/google/image.py
View File

@@ -16,6 +16,7 @@ import os
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -25,6 +26,7 @@ from pydantic import BaseModel, Field
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.google.utils import update_google_client_http_options
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try:
from google import genai
@@ -35,6 +37,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class GoogleImageGenSettings(ImageGenSettings):
"""Settings for the Google image generation service.
Parameters:
model: Google Imagen model identifier.
"""
class GoogleImageGenService(ImageGenService):
"""Google AI image generation service using Imagen models.
@@ -72,17 +83,15 @@ class GoogleImageGenService(ImageGenService):
http_options: HTTP options for the client.
**kwargs: Additional arguments passed to the parent ImageGenService.
"""
super().__init__(**kwargs)
self._params = params or GoogleImageGenService.InputParams()
params = params or GoogleImageGenService.InputParams()
super().__init__(settings=GoogleImageGenSettings(model=params.model), **kwargs)
self._params = params
# Add client header
http_options = update_google_client_http_options(http_options)
self._client = genai.Client(api_key=api_key, http_options=http_options)
self._settings.model = self._params.model
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

36
src/pipecat/services/google/llm.py
View File

@@ -793,29 +793,29 @@ class GoogleLLMService(LLMService):
http_options: HTTP options for the client.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
params = params or GoogleLLMService.InputParams()
super().__init__(
settings=GoogleLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._api_key = api_key
self._system_instruction = system_instruction
self._http_options = update_google_client_http_options(http_options)
self._settings = GoogleLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
self._tools = tools
self._tool_config = tool_config

39
src/pipecat/services/google/stt.py
View File

@@ -499,10 +499,29 @@ class GoogleSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or GoogleSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=GoogleSTTSettings(
language=None,
languages=list(params.language_list),
language_codes=None,
model=params.model,
use_separate_recognition_per_channel=params.use_separate_recognition_per_channel,
enable_automatic_punctuation=params.enable_automatic_punctuation,
enable_spoken_punctuation=params.enable_spoken_punctuation,
enable_spoken_emojis=params.enable_spoken_emojis,
profanity_filter=params.profanity_filter,
enable_word_time_offsets=params.enable_word_time_offsets,
enable_word_confidence=params.enable_word_confidence,
enable_interim_results=params.enable_interim_results,
enable_voice_activity_events=params.enable_voice_activity_events,
),
**kwargs,
)
self._location = location
self._stream = None
self._config = None
@@ -553,22 +572,6 @@ class GoogleSTTService(STTService):
self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options)
self._settings = GoogleSTTSettings(
language=None,
languages=list(params.language_list),
language_codes=None,
model=params.model,
use_separate_recognition_per_channel=params.use_separate_recognition_per_channel,
enable_automatic_punctuation=params.enable_automatic_punctuation,
enable_spoken_punctuation=params.enable_spoken_punctuation,
enable_spoken_emojis=params.enable_spoken_emojis,
profanity_filter=params.profanity_filter,
enable_word_time_offsets=params.enable_word_time_offsets,
enable_word_confidence=params.enable_word_confidence,
enable_interim_results=params.enable_interim_results,
enable_voice_activity_events=params.enable_voice_activity_events,
)
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.

86
src/pipecat/services/google/tts.py
View File

@@ -602,25 +602,28 @@ class GoogleHttpTTSService(TTSService):
params: Voice customization parameters including pitch, rate, volume, etc.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleHttpTTSService.InputParams()
self._location = location
self._settings = GoogleHttpTTSSettings(
model=None,
pitch=params.pitch,
rate=params.rate,
speaking_rate=params.speaking_rate,
volume=params.volume,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
gender=params.gender,
google_style=params.google_style,
voice=voice_id,
super().__init__(
sample_rate=sample_rate,
settings=GoogleHttpTTSSettings(
model=None,
pitch=params.pitch,
rate=params.rate,
speaking_rate=params.speaking_rate,
volume=params.volume,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
gender=params.gender,
google_style=params.google_style,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
@@ -1016,19 +1019,22 @@ class GoogleTTSService(GoogleBaseTTSService):
params: Language configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleTTSService.InputParams()
self._location = location
self._settings = GoogleStreamTTSSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
speaking_rate=params.speaking_rate,
voice=voice_id,
super().__init__(
sample_rate=sample_rate,
settings=GoogleStreamTTSSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
speaking_rate=params.speaking_rate,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._voice_cloning_key = voice_cloning_key
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
@@ -1222,26 +1228,28 @@ class GeminiTTSService(GoogleBaseTTSService):
f"Google TTS only supports {self.GOOGLE_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GeminiTTSService.InputParams()
if voice_id not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
self._location = location
self._model = model
self._settings = GeminiTTSSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
prompt=params.prompt,
multi_speaker=params.multi_speaker,
speaker_configs=params.speaker_configs,
voice=voice_id,
super().__init__(
sample_rate=sample_rate,
settings=GeminiTTSSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
prompt=params.prompt,
multi_speaker=params.multi_speaker,
speaker_configs=params.speaker_configs,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._model = model
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)

23
src/pipecat/services/gradium/stt.py
View File

@@ -129,8 +129,6 @@ class GradiumSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(sample_rate=SAMPLE_RATE, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if json_config is not None:
import warnings
@@ -140,19 +138,24 @@ class GradiumSTTService(WebsocketSTTService):
stacklevel=2,
)
params = params or GradiumSTTService.InputParams()
super().__init__(
sample_rate=SAMPLE_RATE,
ttfs_p99_latency=ttfs_p99_latency,
settings=GradiumSTTSettings(
model=None,
language=params.language,
delay_in_frames=params.delay_in_frames or None,
),
**kwargs,
)
self._api_key = api_key
self._api_endpoint_base_url = api_endpoint_base_url
self._websocket = None
self._json_config = json_config
params = params or GradiumSTTService.InputParams()
self._settings = GradiumSTTSettings(
model=None,
language=params.language,
delay_in_frames=params.delay_in_frames or None,
)
self._receive_task = None
self._audio_buffer = bytearray()

16
src/pipecat/services/gradium/tts.py
View File

@@ -85,27 +85,27 @@ class GradiumTTSService(AudioContextTTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent class.
"""
params = params or GradiumTTSService.InputParams()
super().__init__(
push_stop_frames=True,
push_text_frames=False,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=SAMPLE_RATE,
settings=GradiumTTSSettings(
model=model,
voice=voice_id,
language=None,
output_format="pcm",
),
**kwargs,
)
params = params or GradiumTTSService.InputParams()
# Store service configuration
self._api_key = api_key
self._url = url
self._json_config = json_config
self._settings = GradiumTTSSettings(
model=model,
voice=voice_id,
language=None,
output_format="pcm",
)
# State tracking
self._receive_task = None

32
src/pipecat/services/grok/realtime/llm.py
View File

@@ -145,25 +145,27 @@ class GrokRealtimeLLMService(LLMService):
start_audio_paused: Whether to start with audio input paused. Defaults to False.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(base_url=base_url, **kwargs)
super().__init__(
base_url=base_url,
settings=GrokRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = base_url
self._settings = GrokRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
)
self._audio_input_paused = start_audio_paused
self._websocket = None
self._receive_task = None

23
src/pipecat/services/groq/tts.py
View File

@@ -99,27 +99,24 @@ class GroqTTSService(TTSService):
if sample_rate != self.GROQ_SAMPLE_RATE:
logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ")
params = params or GroqTTSService.InputParams()
super().__init__(
pause_frame_processing=True,
sample_rate=sample_rate,
settings=GroqTTSSettings(
model=model_name,
voice=voice_id,
language=str(params.language) if params.language else "en",
output_format=output_format,
speed=params.speed,
groq_sample_rate=sample_rate,
),
**kwargs,
)
params = params or GroqTTSService.InputParams()
self._api_key = api_key
self._output_format = output_format
self._params = params
self._settings = GroqTTSSettings(
model=model_name,
voice=voice_id,
language=str(params.language) if params.language else "en",
output_format=output_format,
speed=params.speed,
groq_sample_rate=sample_rate,
)
self._sync_model_name_to_metrics()
self._client = AsyncGroq(api_key=self._api_key)

17
src/pipecat/services/hathora/stt.py
View File

@@ -89,24 +89,23 @@ class HathoraSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the parent class.
"""
params = params or HathoraSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=HathoraSTTSettings(
model=model,
language=params.language,
config=params.config,
),
**kwargs,
)
self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
params = params or HathoraSTTService.InputParams()
self._settings = HathoraSTTSettings(
model=model,
language=params.language,
config=params.config,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

19
src/pipecat/services/hathora/tts.py
View File

@@ -107,24 +107,23 @@ class HathoraTTSService(TTSService):
params: Configuration parameters.
**kwargs: Additional arguments passed to the parent class.
"""
params = params or HathoraTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=HathoraTTSSettings(
model=model,
voice=voice_id,
language=None, # Not applicable here
speed=params.speed,
config=params.config,
),
**kwargs,
)
self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
params = params or HathoraTTSService.InputParams()
self._settings = HathoraTTSSettings(
model=model,
voice=voice_id,
speed=params.speed,
config=params.config,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

19
src/pipecat/services/hume/tts.py
View File

@@ -121,11 +121,21 @@ class HumeTTSService(TTSService):
f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}"
)
params = params or HumeTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
settings=HumeTTSSettings(
model=None,
voice=voice_id,
language=None, # Not applicable here
description=params.description,
speed=params.speed,
trailing_silence=params.trailing_silence,
),
**kwargs,
)
@@ -135,15 +145,6 @@ class HumeTTSService(TTSService):
self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client)
params = params or HumeTTSService.InputParams()
self._settings = HumeTTSSettings(
model=None,
voice=voice_id,
description=params.description,
speed=params.speed,
trailing_silence=params.trailing_silence,
)
self._audio_bytes = b""
# Track cumulative time for word timestamps across utterances

14
src/pipecat/services/image_service.py
View File

@@ -11,11 +11,12 @@ text prompts into images.
"""
from abc import abstractmethod
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from pipecat.frames.frames import Frame, TextFrame
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import ImageGenSettings
class ImageGenService(AIService):
@@ -26,13 +27,20 @@ class ImageGenService(AIService):
generation functionality using their specific AI service.
"""
def __init__(self, **kwargs):
def __init__(self, *, settings: Optional[ImageGenSettings] = None, **kwargs):
"""Initialize the image generation service.
Args:
settings: The runtime-updatable settings for the image generation service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ImageGenSettings(),
**kwargs,
)
# Renders the image. Returns an Image object.
@abstractmethod

61
src/pipecat/services/inworld/tts.py
View File

@@ -150,16 +150,28 @@ class InworldHttpTTSService(TTSService):
params: Input parameters for Inworld TTS configuration.
**kwargs: Additional arguments passed to the parent class.
"""
params = params or InworldHttpTTSService.InputParams()
super().__init__(
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=None, # Not applicable for HTTP TTS
apply_text_normalization=None, # Not applicable for HTTP TTS
),
**kwargs,
)
params = params or InworldHttpTTSService.InputParams()
self._api_key = api_key
self._session = aiohttp_session
self._streaming = streaming
@@ -170,23 +182,8 @@ class InworldHttpTTSService(TTSService):
else:
self._base_url = "https://api.inworld.ai/tts/v1/voice"
self._settings = InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=None, # Not applicable for HTTP TTS
apply_text_normalization=None, # Not applicable for HTTP TTS
)
self._cumulative_time = 0.0
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -530,6 +527,8 @@ class InworldTTSService(AudioContextTTSService):
append_trailing_space: Whether to append a trailing space to text before sending to TTS.
**kwargs: Additional arguments passed to the parent class.
"""
params = params or InworldTTSService.InputParams()
super().__init__(
push_text_frames=False,
push_stop_frames=True,
@@ -538,25 +537,23 @@ class InworldTTSService(AudioContextTTSService):
sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences,
append_trailing_space=append_trailing_space,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
apply_text_normalization=params.apply_text_normalization,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=params.auto_mode if params.auto_mode is not None else aggregate_sentences,
),
**kwargs,
)
params = params or InworldTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
apply_text_normalization=params.apply_text_normalization,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=params.auto_mode if params.auto_mode is not None else aggregate_sentences,
)
self._timestamp_type = "WORD"
self._buffer_settings = {
@@ -575,8 +572,6 @@ class InworldTTSService(AudioContextTTSService):
# Track the end time of the last word in the current generation
self._generation_end_time = 0.0
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

19
src/pipecat/services/kokoro/tts.py
View File

@@ -137,19 +137,20 @@ class KokoroTTSService(TTSService):
**kwargs: Additional arguments passed to parent `TTSService`.
"""
super().__init__(**kwargs)
params = params or KokoroTTSService.InputParams()
self._lang_code = language_to_kokoro_language(params.language)
self._settings = KokoroTTSSettings(
model=None,
voice=voice_id,
language=language_to_kokoro_language(params.language),
lang_code=language_to_kokoro_language(params.language),
super().__init__(
settings=KokoroTTSSettings(
model=None,
voice=voice_id,
language=language_to_kokoro_language(params.language),
lang_code=language_to_kokoro_language(params.language),
),
**kwargs,
)
self._lang_code = language_to_kokoro_language(params.language)
model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx"
voices = Path(voices_path) if voices_path else KOKORO_CACHE_DIR / "voices-v1.0.bin"

16
src/pipecat/services/llm_service.py
View File

@@ -181,7 +181,11 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
def __init__(
self, run_in_parallel: bool = True, function_call_timeout_secs: float = 10.0, **kwargs
self,
run_in_parallel: bool = True,
function_call_timeout_secs: float = 10.0,
settings: Optional[LLMSettings] = None,
**kwargs,
):
"""Initialize the LLM service.
@@ -190,10 +194,17 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
Defaults to True.
function_call_timeout_secs: Timeout in seconds for deferred function calls.
Defaults to 10.0 seconds.
settings: The runtime-updatable settings for the LLM service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or LLMSettings(),
**kwargs,
)
self._run_in_parallel = run_in_parallel
self._function_call_timeout_secs = function_call_timeout_secs
self._filter_incomplete_user_turns: bool = False
@@ -204,7 +215,6 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
self._sequential_runner_task: Optional[asyncio.Task] = None
self._skip_tts: Optional[bool] = None
self._summary_task: Optional[asyncio.Task] = None
self._settings = LLMSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._register_event_handler("on_function_calls_started")
self._register_event_handler("on_completion_timeout")

13
src/pipecat/services/lmnt/tts.py
View File

@@ -118,17 +118,16 @@ class LmntTTSService(InterruptibleTTSService):
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
settings=LmntTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(language),
format="raw",
),
**kwargs,
)
self._api_key = api_key
self._settings = LmntTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(language),
format="raw",
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._context_id: Optional[str] = None

44
src/pipecat/services/minimax/tts.py
View File

@@ -227,35 +227,35 @@ class MiniMaxHttpTTSService(TTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or MiniMaxHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=MiniMaxTTSSettings(
model=model,
voice=voice_id,
language=None,
stream=True,
speed=params.speed,
volume=params.volume,
pitch=params.pitch,
language_boost=None,
emotion=None,
text_normalization=None,
latex_read=None,
audio_bitrate=128000,
audio_format="pcm",
audio_channel=1,
audio_sample_rate=0,
),
**kwargs,
)
self._api_key = api_key
self._group_id = group_id
self._base_url = f"{base_url}?GroupId={group_id}"
self._session = aiohttp_session
# Create voice settings
self._settings = MiniMaxTTSSettings(
model=model,
voice=voice_id,
language=None,
stream=True,
speed=params.speed,
volume=params.volume,
pitch=params.pitch,
language_boost=None,
emotion=None,
text_normalization=None,
latex_read=None,
audio_bitrate=128000,
audio_format="pcm",
audio_channel=1,
audio_sample_rate=0,
)
self._sync_model_name_to_metrics()
# Add language boost if provided
if params.language:
service_lang = self.language_to_service_language(params.language)

16
src/pipecat/services/moondream/vision.py
View File

@@ -11,6 +11,7 @@ for image analysis and description generation.
"""
import asyncio
from dataclasses import dataclass
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
VisionFullResponseStartFrame,
VisionTextFrame,
)
from pipecat.services.settings import VisionSettings
from pipecat.services.vision_service import VisionService
try:
@@ -60,6 +62,15 @@ def detect_device():
return torch.device("cpu"), torch.float32
@dataclass
class MoondreamSettings(VisionSettings):
"""Settings for the Moondream vision service.
Parameters:
model: Moondream model identifier.
"""
class MoondreamService(VisionService):
"""Moondream vision-language model service.
@@ -79,10 +90,7 @@ class MoondreamService(VisionService):
use_cpu: Whether to force CPU usage instead of hardware acceleration.
**kwargs: Additional arguments passed to the parent VisionService.
"""
super().__init__(**kwargs)
self._settings.model = model
self._sync_model_name_to_metrics()
super().__init__(settings=MoondreamSettings(model=model), **kwargs)
if not use_cpu:
device, dtype = detect_device()

43
src/pipecat/services/neuphonic/tts.py
View File

@@ -134,26 +134,26 @@ class NeuphonicTTSService(InterruptibleTTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
params = params or NeuphonicTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
push_stop_frames=True,
stop_frame_timeout_s=2.0,
sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
language=self.language_to_service_language(params.language),
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
voice=voice_id,
),
**kwargs,
)
params = params or NeuphonicTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = NeuphonicTTSSettings(
model=None,
language=self.language_to_service_language(params.language),
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
voice=voice_id,
)
self._cumulative_time = 0
@@ -443,21 +443,24 @@ class NeuphonicHttpTTSService(TTSService):
params: Additional input parameters for TTS configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NeuphonicHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(params.language) or "en",
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = url.rstrip("/")
self._settings = NeuphonicTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(params.language) or "en",
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

47
src/pipecat/services/nvidia/stt.py
View File

@@ -164,10 +164,18 @@ class NvidiaSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSTTSettings(
model=model_function_map.get("model_name"),
language=params.language,
),
**kwargs,
)
self._server = server
self._api_key = api_key
self._use_ssl = use_ssl
@@ -180,12 +188,6 @@ class NvidiaSTTService(STTService):
self._custom_configuration = ""
self._function_id = model_function_map.get("function_id")
self._settings = NvidiaSTTSettings(
model=model_function_map.get("model_name"),
language=params.language,
)
self._sync_model_name_to_metrics()
self._asr_service = None
self._queue = None
self._config = None
@@ -463,10 +465,24 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSegmentedSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSegmentedSTTSettings(
model=model_function_map.get("model_name"),
language=self.language_to_service_language(params.language or Language.EN_US)
or "en-US",
profanity_filter=params.profanity_filter,
automatic_punctuation=params.automatic_punctuation,
verbatim_transcripts=params.verbatim_transcripts,
boosted_lm_words=params.boosted_lm_words,
boosted_lm_score=params.boosted_lm_score,
),
**kwargs,
)
# Initialize NVIDIA Riva settings
self._api_key = api_key
self._server = server
@@ -484,17 +500,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
self._config = None
self._asr_service = None
self._settings = NvidiaSegmentedSTTSettings(
model=model_function_map.get("model_name"),
language=self.language_to_service_language(params.language or Language.EN_US)
or "en-US",
profanity_filter=params.profanity_filter,
automatic_punctuation=params.automatic_punctuation,
verbatim_transcripts=params.verbatim_transcripts,
boosted_lm_words=params.boosted_lm_words,
boosted_lm_score=params.boosted_lm_score,
)
self._sync_model_name_to_metrics()
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to NVIDIA Riva's language code.

20
src/pipecat/services/nvidia/tts.py
View File

@@ -103,21 +103,23 @@ class NvidiaTTSService(TTSService):
use_ssl: Whether to use SSL for the NVIDIA Riva server. Defaults to True.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NvidiaTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NvidiaTTSSettings(
model=model_function_map.get("model_name"),
voice=voice_id,
language=params.language,
quality=params.quality,
),
**kwargs,
)
self._server = server
self._api_key = api_key
self._function_id = model_function_map.get("function_id")
self._use_ssl = use_ssl
self._settings = NvidiaTTSSettings(
model=model_function_map.get("model_name"),
voice=voice_id,
language=params.language,
quality=params.quality,
)
self._sync_model_name_to_metrics()
self._service = None
self._config = None

34
src/pipecat/services/openai/base_llm.py
View File

@@ -133,28 +133,28 @@ class BaseOpenAILLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs)
params = params or BaseOpenAILLMService.InputParams()
self._settings = OpenAILLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
presence_penalty=params.presence_penalty,
seed=params.seed,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
max_tokens=params.max_tokens,
max_completion_tokens=params.max_completion_tokens,
service_tier=params.service_tier,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
extra=params.extra if isinstance(params.extra, dict) else {},
super().__init__(
settings=OpenAILLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
presence_penalty=params.presence_penalty,
seed=params.seed,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
max_tokens=params.max_tokens,
max_completion_tokens=params.max_completion_tokens,
service_tier=params.service_tier,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._sync_model_name_to_metrics()
self._full_model_name: str = ""
self._client = self.create_client(
api_key=api_key,

15
src/pipecat/services/openai/image.py
View File

@@ -11,6 +11,7 @@ for creating images from text prompts.
"""
import io
from dataclasses import dataclass
from typing import AsyncGenerator, Literal, Optional
import aiohttp
@@ -24,6 +25,16 @@ from pipecat.frames.frames import (
URLImageRawFrame,
)
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class OpenAIImageGenSettings(ImageGenSettings):
"""Settings for the OpenAI image generation service.
Parameters:
model: DALL-E model identifier.
"""
class OpenAIImageGenService(ImageGenService):
@@ -52,9 +63,7 @@ class OpenAIImageGenService(ImageGenService):
image_size: Target size for generated images.
model: DALL-E model to use for generation. Defaults to "dall-e-3".
"""
super().__init__()
self._settings.model = model
self._sync_model_name_to_metrics()
super().__init__(settings=OpenAIImageGenSettings(model=model))
self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
self._aiohttp_session = aiohttp_session

33
src/pipecat/services/openai/realtime/llm.py
View File

@@ -171,25 +171,26 @@ class OpenAIRealtimeLLMService(LLMService):
# Build WebSocket URL with model query parameter
# Source: https://platform.openai.com/docs/guides/realtime-websocket
full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs)
super().__init__(
base_url=full_url,
settings=OpenAIRealtimeLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = full_url
self._settings = OpenAIRealtimeLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
)
self._sync_model_name_to_metrics()
self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused
self._video_frame_detail = video_frame_detail

12
src/pipecat/services/openai/stt.py
View File

@@ -221,6 +221,11 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=OpenAIRealtimeSTTSettings(
model=model,
language=language,
prompt=prompt,
),
**kwargs,
)
@@ -232,13 +237,6 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
self._noise_reduction = noise_reduction
self._should_interrupt = should_interrupt
self._settings = OpenAIRealtimeSTTSettings(
model=model,
language=language,
prompt=prompt,
)
self._sync_model_name_to_metrics()
self._receive_task = None
self._session_ready = False
self._resampler = create_stream_resampler()

21
src/pipecat/services/openai/tts.py
View File

@@ -132,10 +132,6 @@ class OpenAITTSService(TTSService):
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
if instructions or speed:
import warnings
@@ -147,13 +143,18 @@ class OpenAITTSService(TTSService):
stacklevel=2,
)
self._settings = OpenAITTSSettings(
model=model,
voice=voice,
instructions=params.instructions if params else instructions,
speed=params.speed if params else speed,
super().__init__(
sample_rate=sample_rate,
settings=OpenAITTSSettings(
model=model,
voice=voice,
instructions=params.instructions if params else instructions,
speed=params.speed if params else speed,
),
**kwargs,
)
self._sync_model_name_to_metrics()
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

33
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -156,25 +156,26 @@ class OpenAIRealtimeBetaLLMService(LLMService):
)
full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs)
super().__init__(
base_url=full_url,
settings=OpenAIRealtimeBetaLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = full_url
self._settings = OpenAIRealtimeBetaLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
)
self._sync_model_name_to_metrics()
self._audio_input_paused = start_audio_paused
self._send_transcription_frames = send_transcription_frames
self._websocket = None

13
src/pipecat/services/piper/tts.py
View File

@@ -69,9 +69,10 @@ class PiperTTSService(TTSService):
use_cuda: Use CUDA for GPU-accelerated inference.
**kwargs: Additional arguments passed to the parent `TTSService`.
"""
super().__init__(**kwargs)
self._settings = PiperTTSSettings(model=None, voice=voice_id, language=None)
super().__init__(
settings=PiperTTSSettings(model=None, voice=voice_id, language=None),
**kwargs,
)
download_dir = download_dir or Path.cwd()
@@ -199,7 +200,10 @@ class PiperHttpTTSService(TTSService):
voice_id: Piper voice model identifier (e.g. `en_US-ryan-high`).
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(**kwargs)
super().__init__(
settings=PiperHttpTTSSettings(model=None, voice=voice_id, language=None),
**kwargs,
)
if base_url.endswith("/"):
logger.warning("Base URL ends with a slash, this is not allowed.")
@@ -207,7 +211,6 @@ class PiperHttpTTSService(TTSService):
self._base_url = base_url
self._session = aiohttp_session
self._settings = PiperHttpTTSSettings(model=None, voice=voice_id, language=None)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

16
src/pipecat/services/resembleai/tts.py
View File

@@ -92,19 +92,19 @@ class ResembleAITTSService(AudioContextTTSService):
sample_rate=sample_rate,
reuse_context_id_within_turn=False,
supports_word_timestamps=True,
settings=ResembleAITTSSettings(
model=None,
voice=voice_id,
language=None,
precision=precision,
output_format=output_format,
resemble_sample_rate=sample_rate,
),
**kwargs,
)
self._api_key = api_key
self._url = url
self._settings = ResembleAITTSSettings(
model=None,
voice=voice_id,
language=None,
precision=precision,
output_format=output_format,
resemble_sample_rate=sample_rate,
)
self._websocket = None
self._request_id_counter = 0

126
src/pipecat/services/rime/tts.py
View File

@@ -202,6 +202,8 @@ class RimeTTSService(AudioContextTTSService):
**kwargs: Additional arguments passed to parent class.
"""
# Initialize with parent class settings for proper frame handling
params = params or RimeTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
@@ -210,6 +212,28 @@ class RimeTTSService(AudioContextTTSService):
supports_word_timestamps=True,
append_trailing_space=True,
sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
voice=voice_id,
audioFormat="pcm",
samplingRate=0, # updated in start()
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
inlineSpeedAlpha=None, # Not applicable here
# Arcana params
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
# Mistv2 params
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=params.no_text_normalization,
saveOovs=params.save_oovs,
),
**kwargs,
)
@@ -221,34 +245,9 @@ class RimeTTSService(AudioContextTTSService):
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("spell(", ")")])
params = params or RimeTTSService.InputParams()
# Store service configuration
self._api_key = api_key
self._url = url
self._settings = RimeTTSSettings(
model=model,
voice=voice_id,
audioFormat="pcm",
samplingRate=0, # updated in start()
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
inlineSpeedAlpha=None, # Not applicable here
# Arcana params
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
# Mistv2 params
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=params.no_text_normalization,
saveOovs=params.save_oovs,
)
self._sync_model_name_to_metrics()
# State tracking
self._receive_task = None
@@ -657,34 +656,36 @@ class RimeHttpTTSService(TTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RimeHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
audioFormat="pcm",
samplingRate=0,
segment=None,
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=None,
saveOovs=None,
inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else None,
repetition_penalty=None,
temperature=None,
top_p=None,
voice=voice_id,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = RimeTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
audioFormat="pcm",
samplingRate=0,
segment=None,
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=None,
saveOovs=None,
inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else None,
repetition_penalty=None,
temperature=None,
top_p=None,
voice=voice_id,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -841,31 +842,30 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to parent class.
"""
params = params or RimeNonJsonTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences,
push_stop_frames=True,
pause_frame_processing=True,
append_trailing_space=True,
settings=RimeNonJsonTTSSettings(
voice=voice_id,
model=model,
audioFormat=audio_format,
samplingRate=sample_rate,
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
),
**kwargs,
)
params = params or RimeNonJsonTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = RimeNonJsonTTSSettings(
voice=voice_id,
model=model,
audioFormat=audio_format,
samplingRate=sample_rate,
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
)
self._sync_model_name_to_metrics()
# Add any extra parameters for future compatibility
if params.extra:
self._settings.extra.update(params.extra)

24
src/pipecat/services/sarvam/stt.py
View File

@@ -240,11 +240,22 @@ class SarvamSTTService(STTService):
f"Model '{model}' does not support language parameter (auto-detects language)."
)
# Resolve mode default from model config
mode = params.mode if params.mode is not None else self._config.default_mode
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=keepalive_timeout,
keepalive_interval=keepalive_interval,
settings=SarvamSTTSettings(
model=model,
language=params.language,
prompt=params.prompt,
mode=mode,
vad_signals=params.vad_signals,
high_vad_sensitivity=params.high_vad_sensitivity,
),
**kwargs,
)
@@ -268,19 +279,6 @@ class SarvamSTTService(STTService):
self._socket_client = None
self._receive_task = None
# Resolve mode default from model config
mode = params.mode if params.mode is not None else self._config.default_mode
self._settings = SarvamSTTSettings(
model=model,
language=params.language,
prompt=params.prompt,
mode=mode,
vad_signals=params.vad_signals,
high_vad_sensitivity=params.high_vad_sensitivity,
)
self._sync_model_name_to_metrics()
if params.vad_signals:
self._register_event_handler("on_speech_started")
self._register_event_handler("on_speech_stopped")

110
src/pipecat/services/sarvam/tts.py
View File

@@ -466,12 +466,6 @@ class SarvamHttpTTSService(TTSService):
if voice_id is None:
voice_id = self._config.default_speaker
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Validate and clamp pace to model's valid range
pace = params.pace
pace_min, pace_max = self._config.pace_range
@@ -479,22 +473,32 @@ class SarvamHttpTTSService(TTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = SarvamHttpTTSSettings(
language=(
self.language_to_service_language(params.language) if params.language else "en-IN"
super().__init__(
sample_rate=sample_rate,
settings=SarvamHttpTTSSettings(
language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
),
enable_preprocessing=(
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
**kwargs,
)
self._sync_model_name_to_metrics()
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Add parameters based on model support
if self._config.supports_pitch:
@@ -818,21 +822,8 @@ class SarvamTTSService(InterruptibleTTSService):
if voice_id is None:
voice_id = self._config.default_speaker
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or SarvamTTSService.InputParams()
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
# Validate and clamp pace to model's valid range
pace = params.pace
pace_min, pace_max = self._config.pace_range
@@ -840,27 +831,42 @@ class SarvamTTSService(InterruptibleTTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = SarvamTTSSettings(
language=(
self.language_to_service_language(params.language) if params.language else "en-IN"
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
settings=SarvamTTSSettings(
language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
speech_sample_rate=str(sample_rate),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
min_buffer_size=params.min_buffer_size,
max_chunk_length=params.max_chunk_length,
output_audio_codec=params.output_audio_codec,
output_audio_bitrate=params.output_audio_bitrate,
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
),
speech_sample_rate=str(sample_rate),
enable_preprocessing=(
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
min_buffer_size=params.min_buffer_size,
max_chunk_length=params.max_chunk_length,
output_audio_codec=params.output_audio_codec,
output_audio_bitrate=params.output_audio_bitrate,
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
**kwargs,
)
self._sync_model_name_to_metrics()
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
# Add parameters based on model support
if self._config.supports_pitch:

22
src/pipecat/services/settings.py
View File

@@ -319,6 +319,28 @@ class ServiceSettings:
# ---------------------------------------------------------------------------
@dataclass
class ImageGenSettings(ServiceSettings):
"""Runtime-updatable settings for image generation services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Image generation model identifier.
"""
@dataclass
class VisionSettings(ServiceSettings):
"""Runtime-updatable settings for vision services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Vision model identifier.
"""
@dataclass
class LLMSettings(ServiceSettings):
"""Runtime-updatable settings for LLM services.

29
src/pipecat/services/soniox/stt.py
View File

@@ -202,33 +202,32 @@ class SonioxSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService.
"""
params = params or SonioxInputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=1,
keepalive_interval=5,
settings=SonioxSTTSettings(
model=params.model,
language=None,
audio_format=params.audio_format,
num_channels=params.num_channels,
language_hints=params.language_hints,
language_hints_strict=params.language_hints_strict,
context=params.context,
enable_speaker_diarization=params.enable_speaker_diarization,
enable_language_identification=params.enable_language_identification,
client_reference_id=params.client_reference_id,
),
**kwargs,
)
params = params or SonioxInputParams()
self._api_key = api_key
self._url = url
self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._settings = SonioxSTTSettings(
model=params.model,
language=None,
audio_format=params.audio_format,
num_channels=params.num_channels,
language_hints=params.language_hints,
language_hints_strict=params.language_hints_strict,
context=params.context,
enable_speaker_diarization=params.enable_speaker_diarization,
enable_language_identification=params.enable_language_identification,
client_reference_id=params.client_reference_id,
)
self._sync_model_name_to_metrics()
self._final_transcription_buffer = []
self._last_tokens_received: Optional[float] = None

38
src/pipecat/services/speechmatics/stt.py
View File

@@ -398,8 +398,6 @@ class SpeechmaticsSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
# Service parameters
self._api_key: str = api_key or os.getenv("SPEECHMATICS_API_KEY")
self._base_url: str = (
@@ -428,8 +426,8 @@ class SpeechmaticsSTTService(STTService):
speaker_passive_format = params.speaker_passive_format or speaker_active_format
# Settings — seeded from InputParams
self._settings = SpeechmaticsSTTSettings(
model=None,
settings = SpeechmaticsSTTSettings(
model=None, # Will be resolved from operating_point after config is built
language=params.language,
domain=params.domain,
turn_detection_mode=params.turn_detection_mode,
@@ -455,9 +453,17 @@ class SpeechmaticsSTTService(STTService):
extra_params=params.extra_params,
)
# Build SDK config from settings
# Build SDK config from settings, then resolve model from operating_point
self._client: VoiceAgentClient | None = None
self._config: VoiceAgentConfig = self._build_config()
self._config: VoiceAgentConfig = self._build_config(settings)
settings.model = self._config.operating_point.value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
# Outbound frame queue
self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue()
@@ -468,10 +474,6 @@ class SpeechmaticsSTTService(STTService):
EndOfUtteranceMode.EXTERNAL,
]
# Model + metrics (operating_point comes from the SDK config/preset)
self._settings.model = self._config.operating_point.value
self._sync_model_name_to_metrics()
# Message queue
self._stt_msg_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
self._stt_msg_task: asyncio.Task | None = None
@@ -524,7 +526,7 @@ class SpeechmaticsSTTService(STTService):
logger.debug(f"{self} settings update requires reconnect: {changed.keys()}")
# Connection-level fields changed — rebuild the SDK config
# from the now-updated self._settings, then reconnect.
self._config = self._build_config()
self._config = self._build_config(self._settings)
await self._disconnect()
await self._connect()
elif changed.keys() & SpeechmaticsSTTSettings.HOT_FIELDS:
@@ -661,13 +663,17 @@ class SpeechmaticsSTTService(STTService):
# CONFIGURATION
# ============================================================================
def _build_config(self) -> VoiceAgentConfig:
"""Build a ``VoiceAgentConfig`` from the current ``self._settings``.
def _build_config(self, settings: SpeechmaticsSTTSettings) -> VoiceAgentConfig:
"""Build a ``VoiceAgentConfig`` from the given settings.
Used both at init time and before reconnecting so the connection
always reflects the latest settings.
Used both at init time (with explicit settings, before
``super().__init__`` has run) and before reconnecting so the
connection always reflects the latest settings.
Args:
settings: Settings to build from.
"""
s = self._settings
s = settings
# Preset from turn detection mode
config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value)

21
src/pipecat/services/speechmatics/tts.py
View File

@@ -95,7 +95,18 @@ class SpeechmaticsTTSService(TTSService):
f"Speechmatics TTS only supports {self.SPEECHMATICS_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or SpeechmaticsTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=SpeechmaticsTTSSettings(
model=None,
voice=voice_id,
language=None,
max_retries=params.max_retries,
),
**kwargs,
)
# Service parameters
self._api_key: str = api_key
@@ -106,14 +117,6 @@ class SpeechmaticsTTSService(TTSService):
if not self._api_key:
raise ValueError("Missing Speechmatics API key")
params = params or SpeechmaticsTTSService.InputParams()
self._settings = SpeechmaticsTTSSettings(
model=None,
voice=voice_id,
language=None,
max_retries=params.max_retries,
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

11
src/pipecat/services/stt_service.py
View File

@@ -86,6 +86,7 @@ class STTService(AIService):
ttfs_p99_latency: Optional[float] = None,
keepalive_timeout: Optional[float] = None,
keepalive_interval: float = 5.0,
settings: Optional[STTSettings] = None,
**kwargs,
):
"""Initialize the STT service.
@@ -109,14 +110,20 @@ class STTService(AIService):
connection alive. None disables keepalive. Useful for services that
close idle connections (e.g. behind a ServiceSwitcher).
keepalive_interval: Seconds between idle checks when keepalive is enabled.
settings: The runtime-updatable settings for the STT service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or STTSettings(),
**kwargs,
)
self._audio_passthrough = audio_passthrough
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._settings = STTSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._muted: bool = False
self._user_id: str = ""
self._ttfs_p99_latency = ttfs_p99_latency

11
src/pipecat/services/tts_service.py
View File

@@ -147,6 +147,7 @@ class TTSService(AIService):
text_filter: Optional[BaseTextFilter] = None,
# Audio transport destination of the generated frames.
transport_destination: Optional[str] = None,
settings: Optional[TTSSettings] = None,
**kwargs,
):
"""Initialize the TTS service.
@@ -183,9 +184,16 @@ class TTSService(AIService):
Use `text_filters` instead, which allows multiple filters.
transport_destination: Destination for generated audio frames.
settings: The runtime-updatable settings for the TTS service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or TTSSettings(),
**kwargs,
)
self._aggregate_sentences: bool = aggregate_sentences
self._push_text_frames: bool = push_text_frames
self._push_stop_frames: bool = push_stop_frames
@@ -196,7 +204,6 @@ class TTSService(AIService):
self._append_trailing_space: bool = append_trailing_space
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._settings = TTSSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
if text_aggregator:
import warnings

28
src/pipecat/services/ultravox/llm.py
View File

@@ -176,19 +176,21 @@ class UltravoxRealtimeLLMService(LLMService):
May only be set with OneShotInputParams.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
self._settings = UltravoxRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
output_medium=None,
super().__init__(
settings=UltravoxRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
output_medium=None,
),
**kwargs,
)
self._params = params
if one_shot_selected_tools:

14
src/pipecat/services/vision_service.py
View File

@@ -12,11 +12,12 @@ visual content.
"""
from abc import abstractmethod
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from pipecat.frames.frames import Frame, UserImageRawFrame
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import VisionSettings
class VisionService(AIService):
@@ -27,13 +28,20 @@ class VisionService(AIService):
with the AI service infrastructure for metrics and lifecycle management.
"""
def __init__(self, **kwargs):
def __init__(self, *, settings: Optional[VisionSettings] = None, **kwargs):
"""Initialize the vision service.
Args:
settings: The runtime-updatable settings for the vision service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or VisionSettings(),
**kwargs,
)
self._describe_text = None
@abstractmethod

23
src/pipecat/services/whisper/base_stt.py
View File

@@ -155,22 +155,23 @@ class BaseWhisperSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs)
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=BaseWhisperSTTSettings(
model=model,
language=self.language_to_service_language(language or Language.EN),
base_url=base_url,
prompt=prompt,
temperature=temperature,
),
**kwargs,
)
self._client = self._create_client(api_key, base_url)
self._language = self.language_to_service_language(language or Language.EN)
self._language = self._settings.language
self._prompt = prompt
self._temperature = temperature
self._include_prob_metrics = include_prob_metrics
self._settings = BaseWhisperSTTSettings(
model=model,
language=self._language,
base_url=base_url,
prompt=self._prompt,
temperature=self._temperature,
)
self._sync_model_name_to_metrics()
def _create_client(self, api_key: Optional[str], base_url: Optional[str]):
return AsyncOpenAI(api_key=api_key, base_url=base_url)

41
src/pipecat/services/whisper/stt.py
View File

@@ -233,21 +233,21 @@ class WhisperSTTService(SegmentedSTTService):
language: The default language for transcription.
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
super().__init__(**kwargs)
super().__init__(
settings=WhisperSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
device=device,
compute_type=compute_type,
no_speech_prob=no_speech_prob,
),
**kwargs,
)
self._device: str = device
self._compute_type = compute_type
self._no_speech_prob = no_speech_prob
self._model: Optional[WhisperModel] = None
self._settings = WhisperSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
device=self._device,
compute_type=self._compute_type,
no_speech_prob=self._no_speech_prob,
)
self._sync_model_name_to_metrics()
self._load()
def can_generate_metrics(self) -> bool:
@@ -368,20 +368,21 @@ class WhisperSTTServiceMLX(WhisperSTTService):
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
# Skip WhisperSTTService.__init__ and call its parent directly
SegmentedSTTService.__init__(self, **kwargs)
SegmentedSTTService.__init__(
self,
settings=WhisperMLXSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
no_speech_prob=no_speech_prob,
temperature=temperature,
engine="mlx",
),
**kwargs,
)
self._no_speech_prob = no_speech_prob
self._temperature = temperature
self._settings = WhisperMLXSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
no_speech_prob=self._no_speech_prob,
temperature=self._temperature,
engine="mlx",
)
self._sync_model_name_to_metrics()
# No need to call _load() as MLX Whisper loads models on demand
@override

16
src/pipecat/services/xtts/tts.py
View File

@@ -111,13 +111,15 @@ class XTTSService(TTSService):
sample_rate: Audio sample rate. If None, uses default.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = XTTSTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(language),
base_url=base_url,
super().__init__(
sample_rate=sample_rate,
settings=XTTSTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(language),
base_url=base_url,
),
**kwargs,
)
self._studio_speakers: Optional[Dict[str, Any]] = None
self._aiohttp_session = aiohttp_session