wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make it so that AIService is the exclusive "syncer" of model name to metrics.

Browse Source 
The only (rare) exception—where a service directly still needs to directly call `self._sync_model_name_to_metrics()`—is when the model name need to be "pulled" from another field (or nested field) in settings up to settings.model on a settings update. This only occurs in Deepgram services, where we use the voice as the model name.

This change has the side-effect of bringing model name to metrics for a number of services that were accidentally omitting it before.
This commit is contained in:
Paul Kompfner
2026-02-25 09:41:23 -05:00
parent 937c691f2a
commit 27940d83a2
70 changed files with 1200 additions and 1019 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
COMMUNITY_INTEGRATIONS.md
View File

@@ -257,15 +257,16 @@ The service stores its current settings in `self._settings` and declares the typ
```python ```python
class MySTTService(STTService): class MySTTService(STTService):
_settings: MySTTSettings _settings: MySTTSettings
def __init__(self, *, model: str, language: str, region: str, **kwargs): def __init__(self, *, model: str, language: str, region: str, **kwargs):
super().__init__(**kwargs) # An initial value should be provided for every settings field.
# Initial value must be provided for every field in self._settings # This will be validated at service start.
# before service is started # (If you track sample_rate, it can be a placeholder value like 0; see
self._settings = MySTTSettings(model=model, language=language, region=region) # "Sample Rate Handling").
self._sync_model_name_to_metrics() super().__init__(
settings=MySTTSettings(model=model, language=language, region=region), **kwargs
)
``` ```
To react to runtime setting changes, override `_update_settings`. The base implementation applies the delta to `self._settings` and returns a `dict` mapping each changed field name to its **pre-update** value. Your override should call `super()` first, then act on the changed fields. A common implementation might look like: To react to runtime setting changes, override `_update_settings`. The base implementation applies the delta to `self._settings` and returns a `dict` mapping each changed field name to its **pre-update** value. Your override should call `super()` first, then act on the changed fields. A common implementation might look like:

20
src/pipecat/services/ai_service.py
View File

@@ -35,14 +35,21 @@ class AIService(FrameProcessor):
this base infrastructure. this base infrastructure.
""" """
def __init__(self, **kwargs): def __init__(self, settings: ServiceSettings | None = None, **kwargs):
"""Initialize the AI service. """Initialize the AI service.
Args: Args:
settings: The runtime-updatable settings for the AI service.
**kwargs: Additional arguments passed to the parent FrameProcessor. **kwargs: Additional arguments passed to the parent FrameProcessor.
""" """
super().__init__(**kwargs) super().__init__(**kwargs)
self._settings: ServiceSettings = ServiceSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen) self._settings: ServiceSettings = (
settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ServiceSettings()
)
self._sync_model_name_to_metrics()
self._session_properties: Dict[str, Any] = {} self._session_properties: Dict[str, Any] = {}
self._tracing_enabled: bool = False self._tracing_enabled: bool = False
self._tracing_context = None self._tracing_context = None
@@ -54,15 +61,12 @@ class AIService(FrameProcessor):
of truth for it in `self._settings.model`. This method is just for of truth for it in `self._settings.model`. This method is just for
syncing the model name to the metrics data. syncing the model name to the metrics data.
TODO: as a next step we should make it so that service classes pass
model into `super().__init__` and `AIService` can be responsible for
syncing its initial value to metrics, just as it's responsible for
syncing any updates to its value to metrics via `_update_settings`.
Args: Args:
model: The name of the AI model to use. model: The name of the AI model to use.
""" """
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._settings.model)) self.set_core_metrics_data(
MetricsData(processor=self.name, model=self._settings.model or "")
)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the AI service. """Start the AI service.

52
src/pipecat/services/anthropic/llm.py
View File

@@ -232,37 +232,39 @@ class AnthropicLLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out. retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService. **kwargs: Additional arguments passed to parent LLMService.
""" """
super().__init__(**kwargs)
params = params or AnthropicLLMService.InputParams() params = params or AnthropicLLMService.InputParams()
super().__init__(
settings=AnthropicLLMSettings(
model=model,
max_tokens=params.max_tokens,
enable_prompt_caching=(
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._client = client or AsyncAnthropic( self._client = client or AsyncAnthropic(
api_key=api_key api_key=api_key
) # if the client is provided, use it and remove it, otherwise create a new one ) # if the client is provided, use it and remove it, otherwise create a new one
self._retry_timeout_secs = retry_timeout_secs self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout self._retry_on_timeout = retry_on_timeout
self._settings = AnthropicLLMSettings(
model=model,
max_tokens=params.max_tokens,
enable_prompt_caching=(
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate usage metrics. """Check if this service can generate usage metrics.

14
src/pipecat/services/assemblyai/stt.py
View File

@@ -111,15 +111,17 @@ class AssemblyAISTTService(WebsocketSTTService):
connection_params = self._configure_manual_turn_mode(connection_params) connection_params = self._configure_manual_turn_mode(connection_params)
super().__init__( super().__init__(
sample_rate=connection_params.sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs sample_rate=connection_params.sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AssemblyAISTTSettings(
model=None,
language=language,
connection_params=connection_params,
),
**kwargs,
) )
self._api_key = api_key self._api_key = api_key
self._settings = AssemblyAISTTSettings(
model=None,
language=language,
connection_params=connection_params,
)
self._api_endpoint_base_url = api_endpoint_base_url self._api_endpoint_base_url = api_endpoint_base_url
self._vad_force_turn_endpoint = vad_force_turn_endpoint self._vad_force_turn_endpoint = vad_force_turn_endpoint

53
src/pipecat/services/asyncai/tts.py
View File

@@ -147,30 +147,29 @@ class AsyncAITTSService(AudioContextTTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService. aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to the parent service. **kwargs: Additional arguments passed to the parent service.
""" """
params = params or AsyncAITTSService.InputParams()
super().__init__( super().__init__(
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
pause_frame_processing=True, pause_frame_processing=True,
push_stop_frames=True, push_stop_frames=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs, **kwargs,
) )
params = params or AsyncAITTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._api_version = version self._api_version = version
self._url = url self._url = url
self._settings = AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
)
self._sync_model_name_to_metrics()
self._receive_task = None self._receive_task = None
self._keepalive_task = None self._keepalive_task = None
@@ -501,24 +500,26 @@ class AsyncAIHttpTTSService(TTSService):
params: Additional input parameters for voice customization. params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService. **kwargs: Additional arguments passed to the parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AsyncAIHttpTTSService.InputParams() params = params or AsyncAIHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._base_url = url self._base_url = url
self._api_version = version self._api_version = version
self._settings = AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
)
self._sync_model_name_to_metrics()
self._session = aiohttp_session self._session = aiohttp_session

39
src/pipecat/services/aws/llm.py
View File

@@ -797,10 +797,28 @@ class AWSBedrockLLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out. retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService. **kwargs: Additional arguments passed to parent LLMService.
""" """
super().__init__(**kwargs)
params = params or AWSBedrockLLMService.InputParams() params = params or AWSBedrockLLMService.InputParams()
super().__init__(
settings=AWSBedrockLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
latency=params.latency,
additional_model_request_fields=params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
),
**kwargs,
)
# Initialize the AWS Bedrock client # Initialize the AWS Bedrock client
if not client_config: if not client_config:
client_config = Config( client_config = Config(
@@ -822,23 +840,6 @@ class AWSBedrockLLMService(LLMService):
self._retry_timeout_secs = retry_timeout_secs self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout self._retry_on_timeout = retry_on_timeout
self._settings = AWSBedrockLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
latency=params.latency,
additional_model_request_fields=params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
)
self._sync_model_name_to_metrics()
logger.info(f"Using AWS Bedrock model: {model}") logger.info(f"Using AWS Bedrock model: {model}")

36
src/pipecat/services/aws/nova_sonic/llm.py
View File

@@ -254,28 +254,30 @@ class AWSNovaSonicLLMService(LLMService):
**kwargs: Additional arguments passed to the parent LLMService. **kwargs: Additional arguments passed to the parent LLMService.
""" """
super().__init__(**kwargs) params = params or Params()
super().__init__(
settings=AWSNovaSonicLLMSettings(
model=model,
voice_id=voice_id,
temperature=params.temperature,
max_tokens=params.max_tokens,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
endpointing_sensitivity=params.endpointing_sensitivity,
),
**kwargs,
)
self._secret_access_key = secret_access_key self._secret_access_key = secret_access_key
self._access_key_id = access_key_id self._access_key_id = access_key_id
self._session_token = session_token self._session_token = session_token
self._region = region self._region = region
self._client: Optional[BedrockRuntimeClient] = None self._client: Optional[BedrockRuntimeClient] = None
params = params or Params()
self._settings = AWSNovaSonicLLMSettings(
model=model,
voice_id=voice_id,
temperature=params.temperature,
max_tokens=params.max_tokens,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
endpointing_sensitivity=params.endpointing_sensitivity,
)
self._sync_model_name_to_metrics()
# Audio I/O config (hardware settings, not runtime-tunable) # Audio I/O config (hardware settings, not runtime-tunable)
self._input_sample_rate = params.input_sample_rate self._input_sample_rate = params.input_sample_rate

20
src/pipecat/services/aws/stt.py
View File

@@ -99,15 +99,17 @@ class AWSTranscribeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class. **kwargs: Additional arguments passed to parent STTService class.
""" """
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs) super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
self._settings = AWSTranscribeSTTSettings( settings=AWSTranscribeSTTSettings(
language=self.language_to_service_language(language) or "en-US", language=self.language_to_service_language(language) or "en-US",
sample_rate=sample_rate, sample_rate=sample_rate,
media_encoding="linear16", media_encoding="linear16",
number_of_channels=1, number_of_channels=1,
show_speaker_label=False, show_speaker_label=False,
enable_channel_identification=False, enable_channel_identification=False,
),
**kwargs,
) )
# Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz # Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz

31
src/pipecat/services/aws/tts.py
View File

@@ -195,10 +195,25 @@ class AWSPollyTTSService(TTSService):
params: Additional input parameters for voice customization. params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService class. **kwargs: Additional arguments passed to parent TTSService class.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AWSPollyTTSService.InputParams() params = params or AWSPollyTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AWSPollyTTSSettings(
model=None,
voice=voice_id,
engine=params.engine,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
volume=params.volume,
lexicon_names=params.lexicon_names,
),
**kwargs,
)
# Get credentials from environment variables if not provided # Get credentials from environment variables if not provided
self._aws_params = { self._aws_params = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"), "aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -208,18 +223,6 @@ class AWSPollyTTSService(TTSService):
} }
self._aws_session = aioboto3.Session() self._aws_session = aioboto3.Session()
self._settings = AWSPollyTTSSettings(
model=None,
voice=voice_id,
engine=params.engine,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
volume=params.volume,
lexicon_names=params.lexicon_names,
)
self._resampler = create_stream_resampler() self._resampler = create_stream_resampler()

15
src/pipecat/services/azure/image.py
View File

@@ -12,6 +12,7 @@ using REST endpoints for creating images from text prompts.
import asyncio import asyncio
import io import io
from dataclasses import dataclass
from typing import AsyncGenerator from typing import AsyncGenerator
import aiohttp import aiohttp
@@ -19,6 +20,16 @@ from PIL import Image
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class AzureImageGenSettings(ImageGenSettings):
"""Settings for the Azure image generation service.
Parameters:
model: Azure image generation model identifier.
"""
class AzureImageGenServiceREST(ImageGenService): class AzureImageGenServiceREST(ImageGenService):
@@ -49,13 +60,11 @@ class AzureImageGenServiceREST(ImageGenService):
aiohttp_session: Shared aiohttp session for HTTP requests. aiohttp_session: Shared aiohttp session for HTTP requests.
api_version: Azure API version string. Defaults to "2023-06-01-preview". api_version: Azure API version string. Defaults to "2023-06-01-preview".
""" """
super().__init__() super().__init__(settings=AzureImageGenSettings(model=model))
self._api_key = api_key self._api_key = api_key
self._azure_endpoint = endpoint self._azure_endpoint = endpoint
self._api_version = api_version self._api_version = api_version
self._settings.model = model
self._sync_model_name_to_metrics()
self._image_size = image_size self._image_size = image_size
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session

18
src/pipecat/services/azure/stt.py
View File

@@ -96,7 +96,17 @@ class AzureSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService. **kwargs: Additional arguments passed to parent STTService.
""" """
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs) super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AzureSTTSettings(
model=None,
region=region,
language=language_to_azure_language(language),
sample_rate=sample_rate,
),
**kwargs,
)
self._speech_config = SpeechConfig( self._speech_config = SpeechConfig(
subscription=api_key, subscription=api_key,
@@ -109,12 +119,6 @@ class AzureSTTService(STTService):
self._audio_stream = None self._audio_stream = None
self._speech_recognizer = None self._speech_recognizer = None
self._settings = AzureSTTSettings(
model=None,
region=region,
language=language_to_azure_language(language),
sample_rate=sample_rate,
)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate performance metrics. """Check if this service can generate performance metrics.

60
src/pipecat/services/azure/tts.py
View File

@@ -141,7 +141,6 @@ class AzureBaseTTSService:
api_key: str, api_key: str,
region: str, region: str,
voice: str = "en-US-SaraNeural", voice: str = "en-US-SaraNeural",
params: Optional[InputParams] = None,
): ):
"""Initialize Azure-specific configuration. """Initialize Azure-specific configuration.
@@ -151,25 +150,7 @@ class AzureBaseTTSService:
api_key: Azure Cognitive Services subscription key. api_key: Azure Cognitive Services subscription key.
region: Azure region identifier (e.g., "eastus", "westus2"). region: Azure region identifier (e.g., "eastus", "westus2").
voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural". voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
params: Voice and synthesis parameters configuration.
""" """
params = params or AzureBaseTTSService.InputParams()
self._settings = AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
)
self._api_key = api_key self._api_key = api_key
self._region = region self._region = region
self._speech_synthesizer = None self._speech_synthesizer = None
@@ -289,6 +270,8 @@ class AzureTTSService(TTSService, AzureBaseTTSService):
aggregate_sentences: Whether to aggregate sentences before synthesis. aggregate_sentences: Whether to aggregate sentences before synthesis.
**kwargs: Additional arguments passed to the parent TTSService. **kwargs: Additional arguments passed to the parent TTSService.
""" """
params = params or AzureBaseTTSService.InputParams()
super().__init__( super().__init__(
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
push_text_frames=False, # We'll push text frames based on word timestamps push_text_frames=False, # We'll push text frames based on word timestamps
@@ -296,11 +279,25 @@ class AzureTTSService(TTSService, AzureBaseTTSService):
pause_frame_processing=True, pause_frame_processing=True,
supports_word_timestamps=True, supports_word_timestamps=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs, **kwargs,
) )
# Initialize Azure-specific functionality from mixin # Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params) self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None self._speech_config = None
self._speech_synthesizer = None self._speech_synthesizer = None
@@ -734,10 +731,29 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
params: Voice and synthesis parameters configuration. params: Voice and synthesis parameters configuration.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs) params = params or AzureBaseTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs,
)
# Initialize Azure-specific functionality from mixin # Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params) self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None self._speech_config = None
self._speech_synthesizer = None self._speech_synthesizer = None

28
src/pipecat/services/camb/tts.py
View File

@@ -213,11 +213,6 @@ class CambTTSService(TTSService):
params: Additional voice parameters. If None, uses defaults. params: Additional voice parameters. If None, uses defaults.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._timeout = timeout
params = params or CambTTSService.InputParams() params = params or CambTTSService.InputParams()
# Warn if sample rate doesn't match model's supported rate # Warn if sample rate doesn't match model's supported rate
@@ -227,16 +222,23 @@ class CambTTSService(TTSService):
f"sample rate. Current rate of {sample_rate}Hz may cause issues." f"sample rate. Current rate of {sample_rate}Hz may cause issues."
) )
# Build settings super().__init__(
self._settings = CambTTSSettings( sample_rate=sample_rate,
model=model, settings=CambTTSSettings(
voice=voice_id, model=model,
language=( voice=voice_id,
self.language_to_service_language(params.language) if params.language else "en-us" language=(
self.language_to_service_language(params.language)
if params.language
else "en-us"
),
user_instructions=params.user_instructions,
), ),
user_instructions=params.user_instructions, **kwargs,
) )
self._sync_model_name_to_metrics()
self._api_key = api_key
self._timeout = timeout
self._client = None self._client = None

24
src/pipecat/services/cartesia/stt.py
View File

@@ -173,13 +173,6 @@ class CartesiaSTTService(WebsocketSTTService):
**kwargs: Additional arguments passed to parent STTService. **kwargs: Additional arguments passed to parent STTService.
""" """
sample_rate = sample_rate or (live_options.sample_rate if live_options else None) sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=120,
keepalive_interval=30,
**kwargs,
)
default_options = CartesiaLiveOptions( default_options = CartesiaLiveOptions(
model="ink-whisper", model="ink-whisper",
@@ -196,12 +189,19 @@ class CartesiaSTTService(WebsocketSTTService):
k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None" k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None"
} }
self._settings = CartesiaSTTSettings( super().__init__(
model=merged_options["model"], sample_rate=sample_rate,
language=merged_options.get("language"), ttfs_p99_latency=ttfs_p99_latency,
encoding=merged_options.get("encoding", "pcm_s16le"), keepalive_timeout=120,
keepalive_interval=30,
settings=CartesiaSTTSettings(
model=merged_options["model"],
language=merged_options.get("language"),
encoding=merged_options.get("encoding", "pcm_s16le"),
),
**kwargs,
) )
self._sync_model_name_to_metrics()
self._api_key = api_key self._api_key = api_key
self._base_url = base_url or "api.cartesia.ai" self._base_url = base_url or "api.cartesia.ai"
self._receive_task = None self._receive_task = None

69
src/pipecat/services/cartesia/tts.py
View File

@@ -305,6 +305,8 @@ class CartesiaTTSService(AudioContextTTSService):
# if we're interrupted. Cartesia gives us word-by-word timestamps. We # if we're interrupted. Cartesia gives us word-by-word timestamps. We
# can use those to generate text frames ourselves aligned with the # can use those to generate text frames ourselves aligned with the
# playout timing of the audio! # playout timing of the audio!
params = params or CartesiaTTSService.InputParams()
super().__init__( super().__init__(
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
push_text_frames=False, push_text_frames=False,
@@ -312,6 +314,20 @@ class CartesiaTTSService(AudioContextTTSService):
supports_word_timestamps=True, supports_word_timestamps=True,
sample_rate=sample_rate, sample_rate=sample_rate,
text_aggregator=text_aggregator, text_aggregator=text_aggregator,
settings=CartesiaTTSSettings(
model=model,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
voice=voice_id,
),
**kwargs, **kwargs,
) )
@@ -323,26 +339,9 @@ class CartesiaTTSService(AudioContextTTSService):
# and insert these tags for the purpose of the TTS service alone. # and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("<spell>", "</spell>")]) self._text_aggregator = SkipTagsAggregator([("<spell>", "</spell>")])
params = params or CartesiaTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._cartesia_version = cartesia_version self._cartesia_version = cartesia_version
self._url = url self._url = url
self._settings = CartesiaTTSSettings(
model=model,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
voice=voice_id,
)
self._sync_model_name_to_metrics()
self._receive_task = None self._receive_task = None
@@ -727,28 +726,30 @@ class CartesiaHttpTTSService(TTSService):
params: Additional input parameters for voice customization. params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService. **kwargs: Additional arguments passed to the parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or CartesiaHttpTTSService.InputParams() params = params or CartesiaHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=CartesiaTTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._base_url = base_url self._base_url = base_url
self._cartesia_version = cartesia_version self._cartesia_version = cartesia_version
self._settings = CartesiaTTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
)
self._sync_model_name_to_metrics()
self._client = AsyncCartesia( self._client = AsyncCartesia(
api_key=api_key, api_key=api_key,

28
src/pipecat/services/deepgram/flux/stt.py
View File

@@ -207,26 +207,24 @@ class DeepgramFluxSTTService(WebsocketSTTService):
# was never destroyed. # was never destroyed.
# So we can keep it here as false, because inside the method send_with_retry, it will # So we can keep it here as false, because inside the method send_with_retry, it will
# already try to reconnect if needed. # already try to reconnect if needed.
params = params or DeepgramFluxSTTService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
reconnect_on_error=False, reconnect_on_error=False,
settings=DeepgramFluxSTTSettings(
model=model,
language=Language.EN,
encoding=flux_encoding,
eager_eot_threshold=params.eager_eot_threshold,
eot_threshold=params.eot_threshold,
eot_timeout_ms=params.eot_timeout_ms,
keyterm=params.keyterm or [],
mip_opt_out=params.mip_opt_out,
tag=params.tag or [],
min_confidence=params.min_confidence,
),
**kwargs, **kwargs,
) )
params = params or DeepgramFluxSTTService.InputParams()
self._settings = DeepgramFluxSTTSettings(
model=model,
language=Language.EN,
encoding=flux_encoding,
eager_eot_threshold=params.eager_eot_threshold,
eot_threshold=params.eot_threshold,
eot_timeout_ms=params.eot_timeout_ms,
keyterm=params.keyterm or [],
mip_opt_out=params.mip_opt_out,
tag=params.tag or [],
min_confidence=params.min_confidence,
)
self._sync_model_name_to_metrics()
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._should_interrupt = should_interrupt self._should_interrupt = should_interrupt

15
src/pipecat/services/deepgram/stt.py
View File

@@ -117,7 +117,6 @@ class DeepgramSTTService(STTService):
The `vad_events` option in LiveOptions is deprecated as of version 0.0.99 and will be removed in a future version. Please use the Silero VAD instead. The `vad_events` option in LiveOptions is deprecated as of version 0.0.99 and will be removed in a future version. Please use the Silero VAD instead.
""" """
sample_rate = sample_rate or (live_options.sample_rate if live_options else None) sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if url: if url:
import warnings import warnings
@@ -155,12 +154,16 @@ class DeepgramSTTService(STTService):
merged_options["language"] = merged_options["language"].value merged_options["language"] = merged_options["language"].value
merged_live_options = LiveOptions(**merged_options) merged_live_options = LiveOptions(**merged_options)
self._settings = DeepgramSTTSettings( super().__init__(
model=merged_options.get("model"), sample_rate=sample_rate,
language=merged_options.get("language"), ttfs_p99_latency=ttfs_p99_latency,
live_options=merged_live_options, settings=DeepgramSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
),
**kwargs,
) )
self._sync_model_name_to_metrics()
self._addons = addons self._addons = addons
self._should_interrupt = should_interrupt self._should_interrupt = should_interrupt

21
src/pipecat/services/deepgram/stt_sagemaker.py
View File

@@ -115,10 +115,6 @@ class DeepgramSageMakerSTTService(STTService):
**kwargs: Additional arguments passed to the parent STTService. **kwargs: Additional arguments passed to the parent STTService.
""" """
sample_rate = sample_rate or (live_options.sample_rate if live_options else None) sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._endpoint_name = endpoint_name
self._region = region
# Create default options similar to DeepgramSTTService # Create default options similar to DeepgramSTTService
default_options = LiveOptions( default_options = LiveOptions(
@@ -144,12 +140,19 @@ class DeepgramSageMakerSTTService(STTService):
merged_options["language"] = merged_options["language"].value merged_options["language"] = merged_options["language"].value
merged_live_options = LiveOptions(**merged_options) merged_live_options = LiveOptions(**merged_options)
self._settings = DeepgramSageMakerSTTSettings( super().__init__(
model=merged_options.get("model"), sample_rate=sample_rate,
language=merged_options.get("language"), ttfs_p99_latency=ttfs_p99_latency,
live_options=merged_live_options, settings=DeepgramSageMakerSTTSettings(
model=merged_options.get("model"),
language=merged_options.get("language"),
live_options=merged_live_options,
),
**kwargs,
) )
self._sync_model_name_to_metrics()
self._endpoint_name = endpoint_name
self._region = region
self._client: Optional[SageMakerBidiClient] = None self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None self._response_task: Optional[asyncio.Task] = None

31
src/pipecat/services/deepgram/tts.py
View File

@@ -101,18 +101,17 @@ class DeepgramTTSService(WebsocketTTSService):
pause_frame_processing=True, pause_frame_processing=True,
push_stop_frames=True, push_stop_frames=True,
append_trailing_space=True, append_trailing_space=True,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs, **kwargs,
) )
self._api_key = api_key self._api_key = api_key
self._base_url = base_url self._base_url = base_url
self._settings = DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
)
self._sync_model_name_to_metrics()
self._receive_task = None self._receive_task = None
self._context_id: Optional[str] = None self._context_id: Optional[str] = None
@@ -394,18 +393,20 @@ class DeepgramHttpTTSService(TTSService):
encoding: Audio encoding format. Defaults to "linear16". encoding: Audio encoding format. Defaults to "linear16".
**kwargs: Additional arguments passed to parent TTSService class. **kwargs: Additional arguments passed to parent TTSService class.
""" """
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(
sample_rate=sample_rate,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._session = aiohttp_session self._session = aiohttp_session
self._base_url = base_url self._base_url = base_url
self._settings = DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics. """Check if the service can generate metrics.

13
src/pipecat/services/deepgram/tts_sagemaker.py
View File

@@ -99,18 +99,17 @@ class DeepgramSageMakerTTSService(TTSService):
push_stop_frames=True, push_stop_frames=True,
pause_frame_processing=True, pause_frame_processing=True,
append_trailing_space=True, append_trailing_space=True,
settings=DeepgramSageMakerTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs, **kwargs,
) )
self._endpoint_name = endpoint_name self._endpoint_name = endpoint_name
self._region = region self._region = region
self._settings = DeepgramSageMakerTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
)
self._sync_model_name_to_metrics()
self._client: Optional[SageMakerBidiClient] = None self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None self._response_task: Optional[asyncio.Task] = None

50
src/pipecat/services/elevenlabs/stt.py
View File

@@ -261,28 +261,26 @@ class ElevenLabsSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService. **kwargs: Additional arguments passed to SegmentedSTTService.
""" """
params = params or ElevenLabsSTTService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency, ttfs_p99_latency=ttfs_p99_latency,
settings=ElevenLabsSTTSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
tag_audio_events=params.tag_audio_events,
),
**kwargs, **kwargs,
) )
params = params or ElevenLabsSTTService.InputParams()
self._api_key = api_key self._api_key = api_key
self._base_url = base_url self._base_url = base_url
self._session = aiohttp_session self._session = aiohttp_session
self._model_id = model self._model_id = model
self._settings = ElevenLabsSTTSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
tag_audio_events=params.tag_audio_events,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics. """Check if the service can generate processing metrics.
@@ -500,16 +498,28 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to WebsocketSTTService. **kwargs: Additional arguments passed to WebsocketSTTService.
""" """
params = params or ElevenLabsRealtimeSTTService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency, ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=10, keepalive_timeout=10,
keepalive_interval=5, keepalive_interval=5,
settings=ElevenLabsRealtimeSTTSettings(
model=model,
language=params.language_code,
commit_strategy=params.commit_strategy,
vad_silence_threshold_secs=params.vad_silence_threshold_secs,
vad_threshold=params.vad_threshold,
min_speech_duration_ms=params.min_speech_duration_ms,
min_silence_duration_ms=params.min_silence_duration_ms,
include_timestamps=params.include_timestamps,
enable_logging=params.enable_logging,
include_language_detection=params.include_language_detection,
),
**kwargs, **kwargs,
) )
params = params or ElevenLabsRealtimeSTTService.InputParams()
self._api_key = api_key self._api_key = api_key
self._base_url = base_url self._base_url = base_url
self._model_id = model self._model_id = model
@@ -519,20 +529,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
self._connected_event = asyncio.Event() self._connected_event = asyncio.Event()
self._connected_event.set() self._connected_event.set()
self._settings = ElevenLabsRealtimeSTTSettings(
model=model,
language=params.language_code,
commit_strategy=params.commit_strategy,
vad_silence_threshold_secs=params.vad_silence_threshold_secs,
vad_threshold=params.vad_threshold,
min_speech_duration_ms=params.min_speech_duration_ms,
min_silence_duration_ms=params.min_silence_duration_ms,
include_timestamps=params.include_timestamps,
enable_logging=params.enable_logging,
include_language_detection=params.include_language_detection,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics. """Check if the service can generate processing metrics.

71
src/pipecat/services/elevenlabs/tts.py
View File

@@ -394,6 +394,8 @@ class ElevenLabsTTSService(AudioContextTTSService):
# Finally, ElevenLabs doesn't provide information on when the bot stops # Finally, ElevenLabs doesn't provide information on when the bot stops
# speaking for a while, so we want the parent class to send TTSStopFrame # speaking for a while, so we want the parent class to send TTSStopFrame
# after a short period not receiving any audio. # after a short period not receiving any audio.
params = params or ElevenLabsTTSService.InputParams()
super().__init__( super().__init__(
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
push_text_frames=False, push_text_frames=False,
@@ -401,30 +403,27 @@ class ElevenLabsTTSService(AudioContextTTSService):
pause_frame_processing=True, pause_frame_processing=True,
supports_word_timestamps=True, supports_word_timestamps=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=ElevenLabsTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else None
),
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
auto_mode=str(params.auto_mode).lower(),
enable_ssml_parsing=params.enable_ssml_parsing,
enable_logging=params.enable_logging,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs, **kwargs,
) )
params = params or ElevenLabsTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._settings = ElevenLabsTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else None
),
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
auto_mode=str(params.auto_mode).lower(),
enable_ssml_parsing=params.enable_ssml_parsing,
enable_logging=params.enable_logging,
apply_text_normalization=params.apply_text_normalization,
)
self._sync_model_name_to_metrics()
self._output_format = "" # initialized in start() self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings() self._voice_settings = self._set_voice_settings()
@@ -910,37 +909,35 @@ class ElevenLabsHttpTTSService(TTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService. aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to the parent service. **kwargs: Additional arguments passed to the parent service.
""" """
params = params or ElevenLabsHttpTTSService.InputParams()
super().__init__( super().__init__(
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
push_text_frames=False, push_text_frames=False,
push_stop_frames=True, push_stop_frames=True,
supports_word_timestamps=True, supports_word_timestamps=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=ElevenLabsHttpTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(params.language)
if params.language
else None,
optimize_streaming_latency=params.optimize_streaming_latency,
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs, **kwargs,
) )
params = params or ElevenLabsHttpTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._base_url = base_url self._base_url = base_url
self._params = params
self._session = aiohttp_session self._session = aiohttp_session
self._settings = ElevenLabsHttpTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(params.language)
if params.language
else None,
optimize_streaming_latency=params.optimize_streaming_latency,
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
apply_text_normalization=params.apply_text_normalization,
)
self._sync_model_name_to_metrics()
self._output_format = "" # initialized in start() self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings() self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators

15
src/pipecat/services/fal/image.py
View File

@@ -13,6 +13,7 @@ for creating images from text prompts using various AI models.
import asyncio import asyncio
import io import io
import os import os
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Optional, Union from typing import AsyncGenerator, Dict, Optional, Union
import aiohttp import aiohttp
@@ -22,6 +23,7 @@ from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try: try:
import fal_client import fal_client
@@ -31,6 +33,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}") raise Exception(f"Missing module: {e}")
@dataclass
class FalImageGenSettings(ImageGenSettings):
"""Settings for the Fal image generation service.
Parameters:
model: Fal.ai model identifier.
"""
class FalImageGenService(ImageGenService): class FalImageGenService(ImageGenService):
"""Fal's image generation service. """Fal's image generation service.
@@ -77,9 +88,7 @@ class FalImageGenService(ImageGenService):
key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable. key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable.
**kwargs: Additional arguments passed to parent ImageGenService. **kwargs: Additional arguments passed to parent ImageGenService.
""" """
super().__init__(**kwargs) super().__init__(settings=FalImageGenSettings(model=model), **kwargs)
self._settings.model = model
self._sync_model_name_to_metrics()
self._params = params self._params = params
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
if key: if key:

22
src/pipecat/services/fal/stt.py
View File

@@ -207,14 +207,23 @@ class FalSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService. **kwargs: Additional arguments passed to SegmentedSTTService.
""" """
params = params or FalSTTService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency, ttfs_p99_latency=ttfs_p99_latency,
settings=FalSTTSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en",
task=params.task,
chunk_level=params.chunk_level,
version=params.version,
),
**kwargs, **kwargs,
) )
params = params or FalSTTService.InputParams()
if api_key: if api_key:
os.environ["FAL_KEY"] = api_key os.environ["FAL_KEY"] = api_key
elif "FAL_KEY" not in os.environ: elif "FAL_KEY" not in os.environ:
@@ -223,15 +232,6 @@ class FalSTTService(SegmentedSTTService):
) )
self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY")) self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY"))
self._settings = FalSTTSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en",
task=params.task,
chunk_level=params.chunk_level,
version=params.version,
)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics. """Check if the service can generate processing metrics.

38
src/pipecat/services/fish/tts.py
View File

@@ -138,13 +138,6 @@ class FishAudioTTSService(InterruptibleTTSService):
params: Additional input parameters for voice customization. params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent service. **kwargs: Additional arguments passed to the parent service.
""" """
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or FishAudioTTSService.InputParams() params = params or FishAudioTTSService.InputParams()
# Validation for model and reference_id parameters # Validation for model and reference_id parameters
@@ -169,25 +162,30 @@ class FishAudioTTSService(InterruptibleTTSService):
) )
reference_id = model reference_id = model
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
settings=FishAudioTTSSettings(
model=model_id,
voice=reference_id,
fish_sample_rate=0,
latency=params.latency,
format=output_format,
normalize=params.normalize,
prosody_speed=params.prosody_speed,
prosody_volume=params.prosody_volume,
reference_id=reference_id,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._base_url = "wss://api.fish.audio/v1/tts/live" self._base_url = "wss://api.fish.audio/v1/tts/live"
self._websocket = None self._websocket = None
self._receive_task = None self._receive_task = None
self._request_id = None self._request_id = None
self._settings = FishAudioTTSSettings(
model=model_id,
voice=reference_id,
fish_sample_rate=0,
latency=params.latency,
format=output_format,
normalize=params.normalize,
prosody_speed=params.prosody_speed,
prosody_volume=params.prosody_volume,
reference_id=reference_id,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

54
src/pipecat/services/gladia/stt.py
View File

@@ -278,14 +278,6 @@ class GladiaSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService parent class. **kwargs: Additional arguments passed to the STTService parent class.
""" """
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=20,
keepalive_interval=5,
**kwargs,
)
params = params or GladiaInputParams() params = params or GladiaInputParams()
if params.language is not None: if params.language is not None:
@@ -308,11 +300,6 @@ class GladiaSTTService(WebsocketSTTService):
stacklevel=2, stacklevel=2,
) )
self._api_key = api_key
self._region = region
self._url = url
self._receive_task = None
# Resolve deprecated language → language_config at init time # Resolve deprecated language → language_config at init time
language_config = params.language_config language_config = params.language_config
if not language_config and params.language: if not language_config and params.language:
@@ -320,22 +307,33 @@ class GladiaSTTService(WebsocketSTTService):
if language_code: if language_code:
language_config = LanguageConfig(languages=[language_code], code_switching=False) language_config = LanguageConfig(languages=[language_code], code_switching=False)
self._settings = GladiaSTTSettings( super().__init__(
model=model, sample_rate=sample_rate,
language=None, ttfs_p99_latency=ttfs_p99_latency,
encoding=params.encoding, keepalive_timeout=20,
bit_depth=params.bit_depth, keepalive_interval=5,
channels=params.channels, settings=GladiaSTTSettings(
custom_metadata=params.custom_metadata, model=model,
endpointing=params.endpointing, language=None,
maximum_duration_without_endpointing=params.maximum_duration_without_endpointing, encoding=params.encoding,
language_config=language_config, bit_depth=params.bit_depth,
pre_processing=params.pre_processing, channels=params.channels,
realtime_processing=params.realtime_processing, custom_metadata=params.custom_metadata,
messages_config=params.messages_config, endpointing=params.endpointing,
enable_vad=params.enable_vad, maximum_duration_without_endpointing=params.maximum_duration_without_endpointing,
language_config=language_config,
pre_processing=params.pre_processing,
realtime_processing=params.realtime_processing,
messages_config=params.messages_config,
enable_vad=params.enable_vad,
),
**kwargs,
) )
self._sync_model_name_to_metrics()
self._api_key = api_key
self._region = region
self._url = url
self._receive_task = None
# Session management # Session management
self._session_url = None self._session_url = None

57
src/pipecat/services/google/gemini_live/llm.py
View File

@@ -695,10 +695,38 @@ class GeminiLiveLLMService(LLMService):
stacklevel=2, stacklevel=2,
) )
super().__init__(base_url=base_url, **kwargs)
params = params or InputParams() params = params or InputParams()
super().__init__(
base_url=base_url,
settings=GeminiLiveLLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
max_tokens=params.max_tokens,
presence_penalty=params.presence_penalty,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
modalities=params.modalities,
language=language_to_gemini_language(params.language)
if params.language
else "en-US",
media_resolution=params.media_resolution,
vad=params.vad,
context_window_compression=params.context_window_compression.model_dump()
if params.context_window_compression
else {},
thinking=params.thinking or {},
enable_affective_dialog=params.enable_affective_dialog or False,
proactivity=params.proactivity or {},
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._last_sent_time = 0 self._last_sent_time = 0
self._base_url = base_url self._base_url = base_url
self._voice_id = voice_id self._voice_id = voice_id
@@ -742,31 +770,6 @@ class GeminiLiveLLMService(LLMService):
self._consecutive_failures = 0 self._consecutive_failures = 0
self._connection_start_time = None self._connection_start_time = None
self._settings = GeminiLiveLLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
max_tokens=params.max_tokens,
presence_penalty=params.presence_penalty,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
modalities=params.modalities,
language=self._language_code,
media_resolution=params.media_resolution,
vad=params.vad,
context_window_compression=params.context_window_compression.model_dump()
if params.context_window_compression
else {},
thinking=params.thinking or {},
enable_affective_dialog=params.enable_affective_dialog or False,
proactivity=params.proactivity or {},
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
self._file_api_base_url = file_api_base_url self._file_api_base_url = file_api_base_url
self._file_api: Optional[GeminiFileAPI] = None self._file_api: Optional[GeminiFileAPI] = None

19
src/pipecat/services/google/image.py
View File

@@ -16,6 +16,7 @@ import os
# Suppress gRPC fork warnings # Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Optional from typing import Any, AsyncGenerator, Optional
from loguru import logger from loguru import logger
@@ -25,6 +26,7 @@ from pydantic import BaseModel, Field
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.google.utils import update_google_client_http_options from pipecat.services.google.utils import update_google_client_http_options
from pipecat.services.image_service import ImageGenService from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try: try:
from google import genai from google import genai
@@ -35,6 +37,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}") raise Exception(f"Missing module: {e}")
@dataclass
class GoogleImageGenSettings(ImageGenSettings):
"""Settings for the Google image generation service.
Parameters:
model: Google Imagen model identifier.
"""
class GoogleImageGenService(ImageGenService): class GoogleImageGenService(ImageGenService):
"""Google AI image generation service using Imagen models. """Google AI image generation service using Imagen models.
@@ -72,17 +83,15 @@ class GoogleImageGenService(ImageGenService):
http_options: HTTP options for the client. http_options: HTTP options for the client.
**kwargs: Additional arguments passed to the parent ImageGenService. **kwargs: Additional arguments passed to the parent ImageGenService.
""" """
super().__init__(**kwargs) params = params or GoogleImageGenService.InputParams()
self._params = params or GoogleImageGenService.InputParams() super().__init__(settings=GoogleImageGenSettings(model=params.model), **kwargs)
self._params = params
# Add client header # Add client header
http_options = update_google_client_http_options(http_options) http_options = update_google_client_http_options(http_options)
self._client = genai.Client(api_key=api_key, http_options=http_options) self._client = genai.Client(api_key=api_key, http_options=http_options)
self._settings.model = self._params.model
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

36
src/pipecat/services/google/llm.py
View File

@@ -793,29 +793,29 @@ class GoogleLLMService(LLMService):
http_options: HTTP options for the client. http_options: HTTP options for the client.
**kwargs: Additional arguments passed to parent class. **kwargs: Additional arguments passed to parent class.
""" """
super().__init__(**kwargs)
params = params or GoogleLLMService.InputParams() params = params or GoogleLLMService.InputParams()
super().__init__(
settings=GoogleLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._system_instruction = system_instruction self._system_instruction = system_instruction
self._http_options = update_google_client_http_options(http_options) self._http_options = update_google_client_http_options(http_options)
self._settings = GoogleLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
)
self._sync_model_name_to_metrics()
self._tools = tools self._tools = tools
self._tool_config = tool_config self._tool_config = tool_config

39
src/pipecat/services/google/stt.py
View File

@@ -499,10 +499,29 @@ class GoogleSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService. **kwargs: Additional arguments passed to STTService.
""" """
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or GoogleSTTService.InputParams() params = params or GoogleSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=GoogleSTTSettings(
language=None,
languages=list(params.language_list),
language_codes=None,
model=params.model,
use_separate_recognition_per_channel=params.use_separate_recognition_per_channel,
enable_automatic_punctuation=params.enable_automatic_punctuation,
enable_spoken_punctuation=params.enable_spoken_punctuation,
enable_spoken_emojis=params.enable_spoken_emojis,
profanity_filter=params.profanity_filter,
enable_word_time_offsets=params.enable_word_time_offsets,
enable_word_confidence=params.enable_word_confidence,
enable_interim_results=params.enable_interim_results,
enable_voice_activity_events=params.enable_voice_activity_events,
),
**kwargs,
)
self._location = location self._location = location
self._stream = None self._stream = None
self._config = None self._config = None
@@ -553,22 +572,6 @@ class GoogleSTTService(STTService):
self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options) self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options)
self._settings = GoogleSTTSettings(
language=None,
languages=list(params.language_list),
language_codes=None,
model=params.model,
use_separate_recognition_per_channel=params.use_separate_recognition_per_channel,
enable_automatic_punctuation=params.enable_automatic_punctuation,
enable_spoken_punctuation=params.enable_spoken_punctuation,
enable_spoken_emojis=params.enable_spoken_emojis,
profanity_filter=params.profanity_filter,
enable_word_time_offsets=params.enable_word_time_offsets,
enable_word_confidence=params.enable_word_confidence,
enable_interim_results=params.enable_interim_results,
enable_voice_activity_events=params.enable_voice_activity_events,
)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics. """Check if the service can generate metrics.

86
src/pipecat/services/google/tts.py
View File

@@ -602,25 +602,28 @@ class GoogleHttpTTSService(TTSService):
params: Voice customization parameters including pitch, rate, volume, etc. params: Voice customization parameters including pitch, rate, volume, etc.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleHttpTTSService.InputParams() params = params or GoogleHttpTTSService.InputParams()
self._location = location super().__init__(
self._settings = GoogleHttpTTSSettings( sample_rate=sample_rate,
model=None, settings=GoogleHttpTTSSettings(
pitch=params.pitch, model=None,
rate=params.rate, pitch=params.pitch,
speaking_rate=params.speaking_rate, rate=params.rate,
volume=params.volume, speaking_rate=params.speaking_rate,
emphasis=params.emphasis, volume=params.volume,
language=self.language_to_service_language(params.language) emphasis=params.emphasis,
if params.language language=self.language_to_service_language(params.language)
else "en-US", if params.language
gender=params.gender, else "en-US",
google_style=params.google_style, gender=params.gender,
voice=voice_id, google_style=params.google_style,
voice=voice_id,
),
**kwargs,
) )
self._location = location
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path credentials, credentials_path
) )
@@ -1016,19 +1019,22 @@ class GoogleTTSService(GoogleBaseTTSService):
params: Language configuration parameters. params: Language configuration parameters.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleTTSService.InputParams() params = params or GoogleTTSService.InputParams()
self._location = location super().__init__(
self._settings = GoogleStreamTTSSettings( sample_rate=sample_rate,
model=None, settings=GoogleStreamTTSSettings(
language=self.language_to_service_language(params.language) model=None,
if params.language language=self.language_to_service_language(params.language)
else "en-US", if params.language
speaking_rate=params.speaking_rate, else "en-US",
voice=voice_id, speaking_rate=params.speaking_rate,
voice=voice_id,
),
**kwargs,
) )
self._location = location
self._voice_cloning_key = voice_cloning_key self._voice_cloning_key = voice_cloning_key
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path credentials, credentials_path
@@ -1222,26 +1228,28 @@ class GeminiTTSService(GoogleBaseTTSService):
f"Google TTS only supports {self.GOOGLE_SAMPLE_RATE}Hz sample rate. " f"Google TTS only supports {self.GOOGLE_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues." f"Current rate of {sample_rate}Hz may cause issues."
) )
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GeminiTTSService.InputParams() params = params or GeminiTTSService.InputParams()
if voice_id not in self.AVAILABLE_VOICES: if voice_id not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.") logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
self._location = location super().__init__(
self._model = model sample_rate=sample_rate,
self._settings = GeminiTTSSettings( settings=GeminiTTSSettings(
model=None, model=None,
language=self.language_to_service_language(params.language) language=self.language_to_service_language(params.language)
if params.language if params.language
else "en-US", else "en-US",
prompt=params.prompt, prompt=params.prompt,
multi_speaker=params.multi_speaker, multi_speaker=params.multi_speaker,
speaker_configs=params.speaker_configs, speaker_configs=params.speaker_configs,
voice=voice_id, voice=voice_id,
),
**kwargs,
) )
self._location = location
self._model = model
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path credentials, credentials_path
) )

23
src/pipecat/services/gradium/stt.py
View File

@@ -129,8 +129,6 @@ class GradiumSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class. **kwargs: Additional arguments passed to parent STTService class.
""" """
super().__init__(sample_rate=SAMPLE_RATE, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if json_config is not None: if json_config is not None:
import warnings import warnings
@@ -140,19 +138,24 @@ class GradiumSTTService(WebsocketSTTService):
stacklevel=2, stacklevel=2,
) )
params = params or GradiumSTTService.InputParams()
super().__init__(
sample_rate=SAMPLE_RATE,
ttfs_p99_latency=ttfs_p99_latency,
settings=GradiumSTTSettings(
model=None,
language=params.language,
delay_in_frames=params.delay_in_frames or None,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._api_endpoint_base_url = api_endpoint_base_url self._api_endpoint_base_url = api_endpoint_base_url
self._websocket = None self._websocket = None
self._json_config = json_config self._json_config = json_config
params = params or GradiumSTTService.InputParams()
self._settings = GradiumSTTSettings(
model=None,
language=params.language,
delay_in_frames=params.delay_in_frames or None,
)
self._receive_task = None self._receive_task = None
self._audio_buffer = bytearray() self._audio_buffer = bytearray()

16
src/pipecat/services/gradium/tts.py
View File

@@ -85,27 +85,27 @@ class GradiumTTSService(AudioContextTTSService):
params: Additional configuration parameters. params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent class. **kwargs: Additional arguments passed to parent class.
""" """
params = params or GradiumTTSService.InputParams()
super().__init__( super().__init__(
push_stop_frames=True, push_stop_frames=True,
push_text_frames=False, push_text_frames=False,
pause_frame_processing=True, pause_frame_processing=True,
supports_word_timestamps=True, supports_word_timestamps=True,
sample_rate=SAMPLE_RATE, sample_rate=SAMPLE_RATE,
settings=GradiumTTSSettings(
model=model,
voice=voice_id,
language=None,
output_format="pcm",
),
**kwargs, **kwargs,
) )
params = params or GradiumTTSService.InputParams()
# Store service configuration # Store service configuration
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._json_config = json_config self._json_config = json_config
self._settings = GradiumTTSSettings(
model=model,
voice=voice_id,
language=None,
output_format="pcm",
)
# State tracking # State tracking
self._receive_task = None self._receive_task = None

32
src/pipecat/services/grok/realtime/llm.py
View File

@@ -145,25 +145,27 @@ class GrokRealtimeLLMService(LLMService):
start_audio_paused: Whether to start with audio input paused. Defaults to False. start_audio_paused: Whether to start with audio input paused. Defaults to False.
**kwargs: Additional arguments passed to parent LLMService. **kwargs: Additional arguments passed to parent LLMService.
""" """
super().__init__(base_url=base_url, **kwargs) super().__init__(
base_url=base_url,
settings=GrokRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key self.api_key = api_key
self.base_url = base_url self.base_url = base_url
self._settings = GrokRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
)
self._audio_input_paused = start_audio_paused self._audio_input_paused = start_audio_paused
self._websocket = None self._websocket = None
self._receive_task = None self._receive_task = None

23
src/pipecat/services/groq/tts.py
View File

@@ -99,27 +99,24 @@ class GroqTTSService(TTSService):
if sample_rate != self.GROQ_SAMPLE_RATE: if sample_rate != self.GROQ_SAMPLE_RATE:
logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ") logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ")
params = params or GroqTTSService.InputParams()
super().__init__( super().__init__(
pause_frame_processing=True, pause_frame_processing=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=GroqTTSSettings(
model=model_name,
voice=voice_id,
language=str(params.language) if params.language else "en",
output_format=output_format,
speed=params.speed,
groq_sample_rate=sample_rate,
),
**kwargs, **kwargs,
) )
params = params or GroqTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._output_format = output_format self._output_format = output_format
self._params = params
self._settings = GroqTTSSettings(
model=model_name,
voice=voice_id,
language=str(params.language) if params.language else "en",
output_format=output_format,
speed=params.speed,
groq_sample_rate=sample_rate,
)
self._sync_model_name_to_metrics()
self._client = AsyncGroq(api_key=self._api_key) self._client = AsyncGroq(api_key=self._api_key)

17
src/pipecat/services/hathora/stt.py
View File

@@ -89,24 +89,23 @@ class HathoraSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the parent class. **kwargs: Additional arguments passed to the parent class.
""" """
params = params or HathoraSTTService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency, ttfs_p99_latency=ttfs_p99_latency,
settings=HathoraSTTSettings(
model=model,
language=params.language,
config=params.config,
),
**kwargs, **kwargs,
) )
self._model = model self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY") self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url self._base_url = base_url
params = params or HathoraSTTService.InputParams()
self._settings = HathoraSTTSettings(
model=model,
language=params.language,
config=params.config,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

18
src/pipecat/services/hathora/tts.py
View File

@@ -107,24 +107,22 @@ class HathoraTTSService(TTSService):
params: Configuration parameters. params: Configuration parameters.
**kwargs: Additional arguments passed to the parent class. **kwargs: Additional arguments passed to the parent class.
""" """
params = params or HathoraTTSService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
settings=HathoraTTSSettings(
model=model,
voice=voice_id,
speed=params.speed,
config=params.config,
),
**kwargs, **kwargs,
) )
self._model = model self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY") self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url self._base_url = base_url
params = params or HathoraTTSService.InputParams()
self._settings = HathoraTTSSettings(
model=model,
voice=voice_id,
speed=params.speed,
config=params.config,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

19
src/pipecat/services/hume/tts.py
View File

@@ -121,11 +121,21 @@ class HumeTTSService(TTSService):
f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}" f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}"
) )
params = params or HumeTTSService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
push_text_frames=False, push_text_frames=False,
push_stop_frames=True, push_stop_frames=True,
supports_word_timestamps=True, supports_word_timestamps=True,
settings=HumeTTSSettings(
model=None,
voice=voice_id,
language=None, # Not applicable here
description=params.description,
speed=params.speed,
trailing_silence=params.trailing_silence,
),
**kwargs, **kwargs,
) )
@@ -135,15 +145,6 @@ class HumeTTSService(TTSService):
self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client) self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client)
params = params or HumeTTSService.InputParams()
self._settings = HumeTTSSettings(
model=None,
voice=voice_id,
description=params.description,
speed=params.speed,
trailing_silence=params.trailing_silence,
)
self._audio_bytes = b"" self._audio_bytes = b""
# Track cumulative time for word timestamps across utterances # Track cumulative time for word timestamps across utterances

14
src/pipecat/services/image_service.py
View File

@@ -11,11 +11,12 @@ text prompts into images.
""" """
from abc import abstractmethod from abc import abstractmethod
from typing import AsyncGenerator from typing import AsyncGenerator, Optional
from pipecat.frames.frames import Frame, TextFrame from pipecat.frames.frames import Frame, TextFrame
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService from pipecat.services.ai_service import AIService
from pipecat.services.settings import ImageGenSettings
class ImageGenService(AIService): class ImageGenService(AIService):
@@ -26,13 +27,20 @@ class ImageGenService(AIService):
generation functionality using their specific AI service. generation functionality using their specific AI service.
""" """
def __init__(self, **kwargs): def __init__(self, *, settings: Optional[ImageGenSettings] = None, **kwargs):
"""Initialize the image generation service. """Initialize the image generation service.
Args: Args:
settings: The runtime-updatable settings for the image generation service.
**kwargs: Additional arguments passed to the parent AIService. **kwargs: Additional arguments passed to the parent AIService.
""" """
super().__init__(**kwargs) super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ImageGenSettings(),
**kwargs,
)
# Renders the image. Returns an Image object. # Renders the image. Returns an Image object.
@abstractmethod @abstractmethod

61
src/pipecat/services/inworld/tts.py
View File

@@ -150,16 +150,28 @@ class InworldHttpTTSService(TTSService):
params: Input parameters for Inworld TTS configuration. params: Input parameters for Inworld TTS configuration.
**kwargs: Additional arguments passed to the parent class. **kwargs: Additional arguments passed to the parent class.
""" """
params = params or InworldHttpTTSService.InputParams()
super().__init__( super().__init__(
push_text_frames=False, push_text_frames=False,
push_stop_frames=True, push_stop_frames=True,
supports_word_timestamps=True, supports_word_timestamps=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=None, # Not applicable for HTTP TTS
apply_text_normalization=None, # Not applicable for HTTP TTS
),
**kwargs, **kwargs,
) )
params = params or InworldHttpTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._session = aiohttp_session self._session = aiohttp_session
self._streaming = streaming self._streaming = streaming
@@ -170,23 +182,8 @@ class InworldHttpTTSService(TTSService):
else: else:
self._base_url = "https://api.inworld.ai/tts/v1/voice" self._base_url = "https://api.inworld.ai/tts/v1/voice"
self._settings = InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=None, # Not applicable for HTTP TTS
apply_text_normalization=None, # Not applicable for HTTP TTS
)
self._cumulative_time = 0.0 self._cumulative_time = 0.0
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.
@@ -530,6 +527,8 @@ class InworldTTSService(AudioContextTTSService):
append_trailing_space: Whether to append a trailing space to text before sending to TTS. append_trailing_space: Whether to append a trailing space to text before sending to TTS.
**kwargs: Additional arguments passed to the parent class. **kwargs: Additional arguments passed to the parent class.
""" """
params = params or InworldTTSService.InputParams()
super().__init__( super().__init__(
push_text_frames=False, push_text_frames=False,
push_stop_frames=True, push_stop_frames=True,
@@ -538,25 +537,23 @@ class InworldTTSService(AudioContextTTSService):
sample_rate=sample_rate, sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
append_trailing_space=append_trailing_space, append_trailing_space=append_trailing_space,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
apply_text_normalization=params.apply_text_normalization,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=params.auto_mode if params.auto_mode is not None else aggregate_sentences,
),
**kwargs, **kwargs,
) )
params = params or InworldTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._settings = InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
apply_text_normalization=params.apply_text_normalization,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=params.auto_mode if params.auto_mode is not None else aggregate_sentences,
)
self._timestamp_type = "WORD" self._timestamp_type = "WORD"
self._buffer_settings = { self._buffer_settings = {
@@ -575,8 +572,6 @@ class InworldTTSService(AudioContextTTSService):
# Track the end time of the last word in the current generation # Track the end time of the last word in the current generation
self._generation_end_time = 0.0 self._generation_end_time = 0.0
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

19
src/pipecat/services/kokoro/tts.py
View File

@@ -137,19 +137,20 @@ class KokoroTTSService(TTSService):
**kwargs: Additional arguments passed to parent `TTSService`. **kwargs: Additional arguments passed to parent `TTSService`.
""" """
super().__init__(**kwargs)
params = params or KokoroTTSService.InputParams() params = params or KokoroTTSService.InputParams()
self._lang_code = language_to_kokoro_language(params.language) super().__init__(
settings=KokoroTTSSettings(
self._settings = KokoroTTSSettings( model=None,
model=None, voice=voice_id,
voice=voice_id, language=language_to_kokoro_language(params.language),
language=language_to_kokoro_language(params.language), lang_code=language_to_kokoro_language(params.language),
lang_code=language_to_kokoro_language(params.language), ),
**kwargs,
) )
self._lang_code = language_to_kokoro_language(params.language)
model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx" model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx"
voices = Path(voices_path) if voices_path else KOKORO_CACHE_DIR / "voices-v1.0.bin" voices = Path(voices_path) if voices_path else KOKORO_CACHE_DIR / "voices-v1.0.bin"

16
src/pipecat/services/llm_service.py
View File

@@ -181,7 +181,11 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
def __init__( def __init__(
self, run_in_parallel: bool = True, function_call_timeout_secs: float = 10.0, **kwargs self,
run_in_parallel: bool = True,
function_call_timeout_secs: float = 10.0,
settings: Optional[LLMSettings] = None,
**kwargs,
): ):
"""Initialize the LLM service. """Initialize the LLM service.
@@ -190,10 +194,17 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
Defaults to True. Defaults to True.
function_call_timeout_secs: Timeout in seconds for deferred function calls. function_call_timeout_secs: Timeout in seconds for deferred function calls.
Defaults to 10.0 seconds. Defaults to 10.0 seconds.
settings: The runtime-updatable settings for the LLM service.
**kwargs: Additional arguments passed to the parent AIService. **kwargs: Additional arguments passed to the parent AIService.
""" """
super().__init__(**kwargs) super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or LLMSettings(),
**kwargs,
)
self._run_in_parallel = run_in_parallel self._run_in_parallel = run_in_parallel
self._function_call_timeout_secs = function_call_timeout_secs self._function_call_timeout_secs = function_call_timeout_secs
self._filter_incomplete_user_turns: bool = False self._filter_incomplete_user_turns: bool = False
@@ -204,7 +215,6 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
self._sequential_runner_task: Optional[asyncio.Task] = None self._sequential_runner_task: Optional[asyncio.Task] = None
self._skip_tts: Optional[bool] = None self._skip_tts: Optional[bool] = None
self._summary_task: Optional[asyncio.Task] = None self._summary_task: Optional[asyncio.Task] = None
self._settings = LLMSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._register_event_handler("on_function_calls_started") self._register_event_handler("on_function_calls_started")
self._register_event_handler("on_completion_timeout") self._register_event_handler("on_completion_timeout")

13
src/pipecat/services/lmnt/tts.py
View File

@@ -118,17 +118,16 @@ class LmntTTSService(InterruptibleTTSService):
push_stop_frames=True, push_stop_frames=True,
pause_frame_processing=True, pause_frame_processing=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=LmntTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(language),
format="raw",
),
**kwargs, **kwargs,
) )
self._api_key = api_key self._api_key = api_key
self._settings = LmntTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(language),
format="raw",
)
self._sync_model_name_to_metrics()
self._receive_task = None self._receive_task = None
self._context_id: Optional[str] = None self._context_id: Optional[str] = None

44
src/pipecat/services/minimax/tts.py
View File

@@ -227,35 +227,35 @@ class MiniMaxHttpTTSService(TTSService):
params: Additional configuration parameters. params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or MiniMaxHttpTTSService.InputParams() params = params or MiniMaxHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=MiniMaxTTSSettings(
model=model,
voice=voice_id,
language=None,
stream=True,
speed=params.speed,
volume=params.volume,
pitch=params.pitch,
language_boost=None,
emotion=None,
text_normalization=None,
latex_read=None,
audio_bitrate=128000,
audio_format="pcm",
audio_channel=1,
audio_sample_rate=0,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._group_id = group_id self._group_id = group_id
self._base_url = f"{base_url}?GroupId={group_id}" self._base_url = f"{base_url}?GroupId={group_id}"
self._session = aiohttp_session self._session = aiohttp_session
# Create voice settings
self._settings = MiniMaxTTSSettings(
model=model,
voice=voice_id,
language=None,
stream=True,
speed=params.speed,
volume=params.volume,
pitch=params.pitch,
language_boost=None,
emotion=None,
text_normalization=None,
latex_read=None,
audio_bitrate=128000,
audio_format="pcm",
audio_channel=1,
audio_sample_rate=0,
)
self._sync_model_name_to_metrics()
# Add language boost if provided # Add language boost if provided
if params.language: if params.language:
service_lang = self.language_to_service_language(params.language) service_lang = self.language_to_service_language(params.language)

16
src/pipecat/services/moondream/vision.py
View File

@@ -11,6 +11,7 @@ for image analysis and description generation.
""" """
import asyncio import asyncio
from dataclasses import dataclass
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
from loguru import logger from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
VisionFullResponseStartFrame, VisionFullResponseStartFrame,
VisionTextFrame, VisionTextFrame,
) )
from pipecat.services.settings import VisionSettings
from pipecat.services.vision_service import VisionService from pipecat.services.vision_service import VisionService
try: try:
@@ -60,6 +62,15 @@ def detect_device():
return torch.device("cpu"), torch.float32 return torch.device("cpu"), torch.float32
@dataclass
class MoondreamSettings(VisionSettings):
"""Settings for the Moondream vision service.
Parameters:
model: Moondream model identifier.
"""
class MoondreamService(VisionService): class MoondreamService(VisionService):
"""Moondream vision-language model service. """Moondream vision-language model service.
@@ -79,10 +90,7 @@ class MoondreamService(VisionService):
use_cpu: Whether to force CPU usage instead of hardware acceleration. use_cpu: Whether to force CPU usage instead of hardware acceleration.
**kwargs: Additional arguments passed to the parent VisionService. **kwargs: Additional arguments passed to the parent VisionService.
""" """
super().__init__(**kwargs) super().__init__(settings=MoondreamSettings(model=model), **kwargs)
self._settings.model = model
self._sync_model_name_to_metrics()
if not use_cpu: if not use_cpu:
device, dtype = detect_device() device, dtype = detect_device()

43
src/pipecat/services/neuphonic/tts.py
View File

@@ -134,26 +134,26 @@ class NeuphonicTTSService(InterruptibleTTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService. aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to parent InterruptibleTTSService. **kwargs: Additional arguments passed to parent InterruptibleTTSService.
""" """
params = params or NeuphonicTTSService.InputParams()
super().__init__( super().__init__(
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
push_stop_frames=True, push_stop_frames=True,
stop_frame_timeout_s=2.0, stop_frame_timeout_s=2.0,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
language=self.language_to_service_language(params.language),
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
voice=voice_id,
),
**kwargs, **kwargs,
) )
params = params or NeuphonicTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._settings = NeuphonicTTSSettings(
model=None,
language=self.language_to_service_language(params.language),
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
voice=voice_id,
)
self._cumulative_time = 0 self._cumulative_time = 0
@@ -443,21 +443,24 @@ class NeuphonicHttpTTSService(TTSService):
params: Additional input parameters for TTS configuration. params: Additional input parameters for TTS configuration.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NeuphonicHttpTTSService.InputParams() params = params or NeuphonicHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(params.language) or "en",
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._session = aiohttp_session self._session = aiohttp_session
self._base_url = url.rstrip("/") self._base_url = url.rstrip("/")
self._settings = NeuphonicTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(params.language) or "en",
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

47
src/pipecat/services/nvidia/stt.py
View File

@@ -164,10 +164,18 @@ class NvidiaSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService. **kwargs: Additional arguments passed to STTService.
""" """
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSTTService.InputParams() params = params or NvidiaSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSTTSettings(
model=model_function_map.get("model_name"),
language=params.language,
),
**kwargs,
)
self._server = server self._server = server
self._api_key = api_key self._api_key = api_key
self._use_ssl = use_ssl self._use_ssl = use_ssl
@@ -180,12 +188,6 @@ class NvidiaSTTService(STTService):
self._custom_configuration = "" self._custom_configuration = ""
self._function_id = model_function_map.get("function_id") self._function_id = model_function_map.get("function_id")
self._settings = NvidiaSTTSettings(
model=model_function_map.get("model_name"),
language=params.language,
)
self._sync_model_name_to_metrics()
self._asr_service = None self._asr_service = None
self._queue = None self._queue = None
self._config = None self._config = None
@@ -463,10 +465,24 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService **kwargs: Additional arguments passed to SegmentedSTTService
""" """
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSegmentedSTTService.InputParams() params = params or NvidiaSegmentedSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSegmentedSTTSettings(
model=model_function_map.get("model_name"),
language=self.language_to_service_language(params.language or Language.EN_US)
or "en-US",
profanity_filter=params.profanity_filter,
automatic_punctuation=params.automatic_punctuation,
verbatim_transcripts=params.verbatim_transcripts,
boosted_lm_words=params.boosted_lm_words,
boosted_lm_score=params.boosted_lm_score,
),
**kwargs,
)
# Initialize NVIDIA Riva settings # Initialize NVIDIA Riva settings
self._api_key = api_key self._api_key = api_key
self._server = server self._server = server
@@ -484,17 +500,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
self._config = None self._config = None
self._asr_service = None self._asr_service = None
self._settings = NvidiaSegmentedSTTSettings(
model=model_function_map.get("model_name"),
language=self.language_to_service_language(params.language or Language.EN_US)
or "en-US",
profanity_filter=params.profanity_filter,
automatic_punctuation=params.automatic_punctuation,
verbatim_transcripts=params.verbatim_transcripts,
boosted_lm_words=params.boosted_lm_words,
boosted_lm_score=params.boosted_lm_score,
)
self._sync_model_name_to_metrics()
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to NVIDIA Riva's language code. """Convert pipecat Language enum to NVIDIA Riva's language code.

20
src/pipecat/services/nvidia/tts.py
View File

@@ -103,21 +103,23 @@ class NvidiaTTSService(TTSService):
use_ssl: Whether to use SSL for the NVIDIA Riva server. Defaults to True. use_ssl: Whether to use SSL for the NVIDIA Riva server. Defaults to True.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NvidiaTTSService.InputParams() params = params or NvidiaTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NvidiaTTSSettings(
model=model_function_map.get("model_name"),
voice=voice_id,
language=params.language,
quality=params.quality,
),
**kwargs,
)
self._server = server self._server = server
self._api_key = api_key self._api_key = api_key
self._function_id = model_function_map.get("function_id") self._function_id = model_function_map.get("function_id")
self._use_ssl = use_ssl self._use_ssl = use_ssl
self._settings = NvidiaTTSSettings(
model=model_function_map.get("model_name"),
voice=voice_id,
language=params.language,
quality=params.quality,
)
self._sync_model_name_to_metrics()
self._service = None self._service = None
self._config = None self._config = None

34
src/pipecat/services/openai/base_llm.py
View File

@@ -133,28 +133,28 @@ class BaseOpenAILLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out. retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to the parent LLMService. **kwargs: Additional arguments passed to the parent LLMService.
""" """
super().__init__(**kwargs)
params = params or BaseOpenAILLMService.InputParams() params = params or BaseOpenAILLMService.InputParams()
self._settings = OpenAILLMSettings( super().__init__(
model=model, settings=OpenAILLMSettings(
frequency_penalty=params.frequency_penalty, model=model,
presence_penalty=params.presence_penalty, frequency_penalty=params.frequency_penalty,
seed=params.seed, presence_penalty=params.presence_penalty,
temperature=params.temperature, seed=params.seed,
top_p=params.top_p, temperature=params.temperature,
top_k=None, top_p=params.top_p,
max_tokens=params.max_tokens, top_k=None,
max_completion_tokens=params.max_completion_tokens, max_tokens=params.max_tokens,
service_tier=params.service_tier, max_completion_tokens=params.max_completion_tokens,
filter_incomplete_user_turns=False, service_tier=params.service_tier,
user_turn_completion_config=None, filter_incomplete_user_turns=False,
extra=params.extra if isinstance(params.extra, dict) else {}, user_turn_completion_config=None,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
) )
self._retry_timeout_secs = retry_timeout_secs self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout self._retry_on_timeout = retry_on_timeout
self._sync_model_name_to_metrics()
self._full_model_name: str = "" self._full_model_name: str = ""
self._client = self.create_client( self._client = self.create_client(
api_key=api_key, api_key=api_key,

15
src/pipecat/services/openai/image.py
View File

@@ -11,6 +11,7 @@ for creating images from text prompts.
""" """
import io import io
from dataclasses import dataclass
from typing import AsyncGenerator, Literal, Optional from typing import AsyncGenerator, Literal, Optional
import aiohttp import aiohttp
@@ -24,6 +25,16 @@ from pipecat.frames.frames import (
URLImageRawFrame, URLImageRawFrame,
) )
from pipecat.services.image_service import ImageGenService from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class OpenAIImageGenSettings(ImageGenSettings):
"""Settings for the OpenAI image generation service.
Parameters:
model: DALL-E model identifier.
"""
class OpenAIImageGenService(ImageGenService): class OpenAIImageGenService(ImageGenService):
@@ -52,9 +63,7 @@ class OpenAIImageGenService(ImageGenService):
image_size: Target size for generated images. image_size: Target size for generated images.
model: DALL-E model to use for generation. Defaults to "dall-e-3". model: DALL-E model to use for generation. Defaults to "dall-e-3".
""" """
super().__init__() super().__init__(settings=OpenAIImageGenSettings(model=model))
self._settings.model = model
self._sync_model_name_to_metrics()
self._image_size = image_size self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session

33
src/pipecat/services/openai/realtime/llm.py
View File

@@ -171,25 +171,26 @@ class OpenAIRealtimeLLMService(LLMService):
# Build WebSocket URL with model query parameter # Build WebSocket URL with model query parameter
# Source: https://platform.openai.com/docs/guides/realtime-websocket # Source: https://platform.openai.com/docs/guides/realtime-websocket
full_url = f"{base_url}?model={model}" full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs) super().__init__(
base_url=full_url,
settings=OpenAIRealtimeLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key self.api_key = api_key
self.base_url = full_url self.base_url = full_url
self._settings = OpenAIRealtimeLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
)
self._sync_model_name_to_metrics()
self._audio_input_paused = start_audio_paused self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused self._video_input_paused = start_video_paused
self._video_frame_detail = video_frame_detail self._video_frame_detail = video_frame_detail

12
src/pipecat/services/openai/stt.py
View File

@@ -221,6 +221,11 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
super().__init__( super().__init__(
ttfs_p99_latency=ttfs_p99_latency, ttfs_p99_latency=ttfs_p99_latency,
settings=OpenAIRealtimeSTTSettings(
model=model,
language=language,
prompt=prompt,
),
**kwargs, **kwargs,
) )
@@ -232,13 +237,6 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
self._noise_reduction = noise_reduction self._noise_reduction = noise_reduction
self._should_interrupt = should_interrupt self._should_interrupt = should_interrupt
self._settings = OpenAIRealtimeSTTSettings(
model=model,
language=language,
prompt=prompt,
)
self._sync_model_name_to_metrics()
self._receive_task = None self._receive_task = None
self._session_ready = False self._session_ready = False
self._resampler = create_stream_resampler() self._resampler = create_stream_resampler()

21
src/pipecat/services/openai/tts.py
View File

@@ -132,10 +132,6 @@ class OpenAITTSService(TTSService):
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. " f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues." f"Current rate of {sample_rate}Hz may cause issues."
) )
super().__init__(sample_rate=sample_rate, **kwargs)
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
if instructions or speed: if instructions or speed:
import warnings import warnings
@@ -147,13 +143,18 @@ class OpenAITTSService(TTSService):
stacklevel=2, stacklevel=2,
) )
self._settings = OpenAITTSSettings( super().__init__(
model=model, sample_rate=sample_rate,
voice=voice, settings=OpenAITTSSettings(
instructions=params.instructions if params else instructions, model=model,
speed=params.speed if params else speed, voice=voice,
instructions=params.instructions if params else instructions,
speed=params.speed if params else speed,
),
**kwargs,
) )
self._sync_model_name_to_metrics()
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

33
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -156,25 +156,26 @@ class OpenAIRealtimeBetaLLMService(LLMService):
) )
full_url = f"{base_url}?model={model}" full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs) super().__init__(
base_url=full_url,
settings=OpenAIRealtimeBetaLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key self.api_key = api_key
self.base_url = full_url self.base_url = full_url
self._settings = OpenAIRealtimeBetaLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
)
self._sync_model_name_to_metrics()
self._audio_input_paused = start_audio_paused self._audio_input_paused = start_audio_paused
self._send_transcription_frames = send_transcription_frames self._send_transcription_frames = send_transcription_frames
self._websocket = None self._websocket = None

13
src/pipecat/services/piper/tts.py
View File

@@ -69,9 +69,10 @@ class PiperTTSService(TTSService):
use_cuda: Use CUDA for GPU-accelerated inference. use_cuda: Use CUDA for GPU-accelerated inference.
**kwargs: Additional arguments passed to the parent `TTSService`. **kwargs: Additional arguments passed to the parent `TTSService`.
""" """
super().__init__(**kwargs) super().__init__(
settings=PiperTTSSettings(model=None, voice=voice_id, language=None),
self._settings = PiperTTSSettings(model=None, voice=voice_id, language=None) **kwargs,
)
download_dir = download_dir or Path.cwd() download_dir = download_dir or Path.cwd()
@@ -199,7 +200,10 @@ class PiperHttpTTSService(TTSService):
voice_id: Piper voice model identifier (e.g. `en_US-ryan-high`). voice_id: Piper voice model identifier (e.g. `en_US-ryan-high`).
**kwargs: Additional arguments passed to the parent TTSService. **kwargs: Additional arguments passed to the parent TTSService.
""" """
super().__init__(**kwargs) super().__init__(
settings=PiperHttpTTSSettings(model=None, voice=voice_id, language=None),
**kwargs,
)
if base_url.endswith("/"): if base_url.endswith("/"):
logger.warning("Base URL ends with a slash, this is not allowed.") logger.warning("Base URL ends with a slash, this is not allowed.")
@@ -207,7 +211,6 @@ class PiperHttpTTSService(TTSService):
self._base_url = base_url self._base_url = base_url
self._session = aiohttp_session self._session = aiohttp_session
self._settings = PiperHttpTTSSettings(model=None, voice=voice_id, language=None)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

16
src/pipecat/services/resembleai/tts.py
View File

@@ -92,19 +92,19 @@ class ResembleAITTSService(AudioContextTTSService):
sample_rate=sample_rate, sample_rate=sample_rate,
reuse_context_id_within_turn=False, reuse_context_id_within_turn=False,
supports_word_timestamps=True, supports_word_timestamps=True,
settings=ResembleAITTSSettings(
model=None,
voice=voice_id,
language=None,
precision=precision,
output_format=output_format,
resemble_sample_rate=sample_rate,
),
**kwargs, **kwargs,
) )
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._settings = ResembleAITTSSettings(
model=None,
voice=voice_id,
language=None,
precision=precision,
output_format=output_format,
resemble_sample_rate=sample_rate,
)
self._websocket = None self._websocket = None
self._request_id_counter = 0 self._request_id_counter = 0

126
src/pipecat/services/rime/tts.py
View File

@@ -202,6 +202,8 @@ class RimeTTSService(AudioContextTTSService):
**kwargs: Additional arguments passed to parent class. **kwargs: Additional arguments passed to parent class.
""" """
# Initialize with parent class settings for proper frame handling # Initialize with parent class settings for proper frame handling
params = params or RimeTTSService.InputParams()
super().__init__( super().__init__(
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
push_text_frames=False, push_text_frames=False,
@@ -210,6 +212,28 @@ class RimeTTSService(AudioContextTTSService):
supports_word_timestamps=True, supports_word_timestamps=True,
append_trailing_space=True, append_trailing_space=True,
sample_rate=sample_rate, sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
voice=voice_id,
audioFormat="pcm",
samplingRate=0, # updated in start()
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
inlineSpeedAlpha=None, # Not applicable here
# Arcana params
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
# Mistv2 params
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=params.no_text_normalization,
saveOovs=params.save_oovs,
),
**kwargs, **kwargs,
) )
@@ -221,34 +245,9 @@ class RimeTTSService(AudioContextTTSService):
# and insert these tags for the purpose of the TTS service alone. # and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("spell(", ")")]) self._text_aggregator = SkipTagsAggregator([("spell(", ")")])
params = params or RimeTTSService.InputParams()
# Store service configuration # Store service configuration
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._settings = RimeTTSSettings(
model=model,
voice=voice_id,
audioFormat="pcm",
samplingRate=0, # updated in start()
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
inlineSpeedAlpha=None, # Not applicable here
# Arcana params
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
# Mistv2 params
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=params.no_text_normalization,
saveOovs=params.save_oovs,
)
self._sync_model_name_to_metrics()
# State tracking # State tracking
self._receive_task = None self._receive_task = None
@@ -657,34 +656,36 @@ class RimeHttpTTSService(TTSService):
params: Additional configuration parameters. params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RimeHttpTTSService.InputParams() params = params or RimeHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
audioFormat="pcm",
samplingRate=0,
segment=None,
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=None,
saveOovs=None,
inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else None,
repetition_penalty=None,
temperature=None,
top_p=None,
voice=voice_id,
),
**kwargs,
)
self._api_key = api_key self._api_key = api_key
self._session = aiohttp_session self._session = aiohttp_session
self._base_url = "https://users.rime.ai/v1/rime-tts" self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = RimeTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
audioFormat="pcm",
samplingRate=0,
segment=None,
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=None,
saveOovs=None,
inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else None,
repetition_penalty=None,
temperature=None,
top_p=None,
voice=voice_id,
)
self._sync_model_name_to_metrics()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.
@@ -841,31 +842,30 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
aggregate_sentences: Whether to aggregate sentences within the TTSService. aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to parent class. **kwargs: Additional arguments passed to parent class.
""" """
params = params or RimeNonJsonTTSService.InputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences, aggregate_sentences=aggregate_sentences,
push_stop_frames=True, push_stop_frames=True,
pause_frame_processing=True, pause_frame_processing=True,
append_trailing_space=True, append_trailing_space=True,
settings=RimeNonJsonTTSSettings(
voice=voice_id,
model=model,
audioFormat=audio_format,
samplingRate=sample_rate,
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
),
**kwargs, **kwargs,
) )
params = params or RimeNonJsonTTSService.InputParams()
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._settings = RimeNonJsonTTSSettings(
voice=voice_id,
model=model,
audioFormat=audio_format,
samplingRate=sample_rate,
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
)
self._sync_model_name_to_metrics()
# Add any extra parameters for future compatibility # Add any extra parameters for future compatibility
if params.extra: if params.extra:
self._settings.extra.update(params.extra) self._settings.extra.update(params.extra)

24
src/pipecat/services/sarvam/stt.py
View File

@@ -240,11 +240,22 @@ class SarvamSTTService(STTService):
f"Model '{model}' does not support language parameter (auto-detects language)." f"Model '{model}' does not support language parameter (auto-detects language)."
) )
# Resolve mode default from model config
mode = params.mode if params.mode is not None else self._config.default_mode
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency, ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=keepalive_timeout, keepalive_timeout=keepalive_timeout,
keepalive_interval=keepalive_interval, keepalive_interval=keepalive_interval,
settings=SarvamSTTSettings(
model=model,
language=params.language,
prompt=params.prompt,
mode=mode,
vad_signals=params.vad_signals,
high_vad_sensitivity=params.high_vad_sensitivity,
),
**kwargs, **kwargs,
) )
@@ -268,19 +279,6 @@ class SarvamSTTService(STTService):
self._socket_client = None self._socket_client = None
self._receive_task = None self._receive_task = None
# Resolve mode default from model config
mode = params.mode if params.mode is not None else self._config.default_mode
self._settings = SarvamSTTSettings(
model=model,
language=params.language,
prompt=params.prompt,
mode=mode,
vad_signals=params.vad_signals,
high_vad_sensitivity=params.high_vad_sensitivity,
)
self._sync_model_name_to_metrics()
if params.vad_signals: if params.vad_signals:
self._register_event_handler("on_speech_started") self._register_event_handler("on_speech_started")
self._register_event_handler("on_speech_stopped") self._register_event_handler("on_speech_stopped")

110
src/pipecat/services/sarvam/tts.py
View File

@@ -466,12 +466,6 @@ class SarvamHttpTTSService(TTSService):
if voice_id is None: if voice_id is None:
voice_id = self._config.default_speaker voice_id = self._config.default_speaker
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Validate and clamp pace to model's valid range # Validate and clamp pace to model's valid range
pace = params.pace pace = params.pace
pace_min, pace_max = self._config.pace_range pace_min, pace_max = self._config.pace_range
@@ -479,22 +473,32 @@ class SarvamHttpTTSService(TTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.") logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace)) pace = max(pace_min, min(pace_max, pace))
# Build base settings super().__init__(
self._settings = SarvamHttpTTSSettings( sample_rate=sample_rate,
language=( settings=SarvamHttpTTSSettings(
self.language_to_service_language(params.language) if params.language else "en-IN" language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
), ),
enable_preprocessing=( **kwargs,
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
) )
self._sync_model_name_to_metrics()
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Add parameters based on model support # Add parameters based on model support
if self._config.supports_pitch: if self._config.supports_pitch:
@@ -818,21 +822,8 @@ class SarvamTTSService(InterruptibleTTSService):
if voice_id is None: if voice_id is None:
voice_id = self._config.default_speaker voice_id = self._config.default_speaker
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or SarvamTTSService.InputParams() params = params or SarvamTTSService.InputParams()
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
# Validate and clamp pace to model's valid range # Validate and clamp pace to model's valid range
pace = params.pace pace = params.pace
pace_min, pace_max = self._config.pace_range pace_min, pace_max = self._config.pace_range
@@ -840,27 +831,42 @@ class SarvamTTSService(InterruptibleTTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.") logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace)) pace = max(pace_min, min(pace_max, pace))
# Build base settings # Initialize parent class first
self._settings = SarvamTTSSettings( super().__init__(
language=( aggregate_sentences=aggregate_sentences,
self.language_to_service_language(params.language) if params.language else "en-IN" push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
settings=SarvamTTSSettings(
language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
speech_sample_rate=str(sample_rate),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
min_buffer_size=params.min_buffer_size,
max_chunk_length=params.max_chunk_length,
output_audio_codec=params.output_audio_codec,
output_audio_bitrate=params.output_audio_bitrate,
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
), ),
speech_sample_rate=str(sample_rate), **kwargs,
enable_preprocessing=(
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
min_buffer_size=params.min_buffer_size,
max_chunk_length=params.max_chunk_length,
output_audio_codec=params.output_audio_codec,
output_audio_bitrate=params.output_audio_bitrate,
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
) )
self._sync_model_name_to_metrics()
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
# Add parameters based on model support # Add parameters based on model support
if self._config.supports_pitch: if self._config.supports_pitch:

22
src/pipecat/services/settings.py
View File

@@ -319,6 +319,28 @@ class ServiceSettings:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@dataclass
class ImageGenSettings(ServiceSettings):
"""Runtime-updatable settings for image generation services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Image generation model identifier.
"""
@dataclass
class VisionSettings(ServiceSettings):
"""Runtime-updatable settings for vision services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Vision model identifier.
"""
@dataclass @dataclass
class LLMSettings(ServiceSettings): class LLMSettings(ServiceSettings):
"""Runtime-updatable settings for LLM services. """Runtime-updatable settings for LLM services.

29
src/pipecat/services/soniox/stt.py
View File

@@ -202,33 +202,32 @@ class SonioxSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService. **kwargs: Additional arguments passed to the STTService.
""" """
params = params or SonioxInputParams()
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency, ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=1, keepalive_timeout=1,
keepalive_interval=5, keepalive_interval=5,
settings=SonioxSTTSettings(
model=params.model,
language=None,
audio_format=params.audio_format,
num_channels=params.num_channels,
language_hints=params.language_hints,
language_hints_strict=params.language_hints_strict,
context=params.context,
enable_speaker_diarization=params.enable_speaker_diarization,
enable_language_identification=params.enable_language_identification,
client_reference_id=params.client_reference_id,
),
**kwargs, **kwargs,
) )
params = params or SonioxInputParams()
self._api_key = api_key self._api_key = api_key
self._url = url self._url = url
self._vad_force_turn_endpoint = vad_force_turn_endpoint self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._settings = SonioxSTTSettings(
model=params.model,
language=None,
audio_format=params.audio_format,
num_channels=params.num_channels,
language_hints=params.language_hints,
language_hints_strict=params.language_hints_strict,
context=params.context,
enable_speaker_diarization=params.enable_speaker_diarization,
enable_language_identification=params.enable_language_identification,
client_reference_id=params.client_reference_id,
)
self._sync_model_name_to_metrics()
self._final_transcription_buffer = [] self._final_transcription_buffer = []
self._last_tokens_received: Optional[float] = None self._last_tokens_received: Optional[float] = None

38
src/pipecat/services/speechmatics/stt.py
View File

@@ -398,8 +398,6 @@ class SpeechmaticsSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService. **kwargs: Additional arguments passed to STTService.
""" """
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
# Service parameters # Service parameters
self._api_key: str = api_key or os.getenv("SPEECHMATICS_API_KEY") self._api_key: str = api_key or os.getenv("SPEECHMATICS_API_KEY")
self._base_url: str = ( self._base_url: str = (
@@ -428,8 +426,8 @@ class SpeechmaticsSTTService(STTService):
speaker_passive_format = params.speaker_passive_format or speaker_active_format speaker_passive_format = params.speaker_passive_format or speaker_active_format
# Settings — seeded from InputParams # Settings — seeded from InputParams
self._settings = SpeechmaticsSTTSettings( settings = SpeechmaticsSTTSettings(
model=None, model=None, # Will be resolved from operating_point after config is built
language=params.language, language=params.language,
domain=params.domain, domain=params.domain,
turn_detection_mode=params.turn_detection_mode, turn_detection_mode=params.turn_detection_mode,
@@ -455,9 +453,17 @@ class SpeechmaticsSTTService(STTService):
extra_params=params.extra_params, extra_params=params.extra_params,
) )
# Build SDK config from settings # Build SDK config from settings, then resolve model from operating_point
self._client: VoiceAgentClient | None = None self._client: VoiceAgentClient | None = None
self._config: VoiceAgentConfig = self._build_config() self._config: VoiceAgentConfig = self._build_config(settings)
settings.model = self._config.operating_point.value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
# Outbound frame queue # Outbound frame queue
self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue() self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue()
@@ -468,10 +474,6 @@ class SpeechmaticsSTTService(STTService):
EndOfUtteranceMode.EXTERNAL, EndOfUtteranceMode.EXTERNAL,
] ]
# Model + metrics (operating_point comes from the SDK config/preset)
self._settings.model = self._config.operating_point.value
self._sync_model_name_to_metrics()
# Message queue # Message queue
self._stt_msg_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue() self._stt_msg_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
self._stt_msg_task: asyncio.Task | None = None self._stt_msg_task: asyncio.Task | None = None
@@ -524,7 +526,7 @@ class SpeechmaticsSTTService(STTService):
logger.debug(f"{self} settings update requires reconnect: {changed.keys()}") logger.debug(f"{self} settings update requires reconnect: {changed.keys()}")
# Connection-level fields changed — rebuild the SDK config # Connection-level fields changed — rebuild the SDK config
# from the now-updated self._settings, then reconnect. # from the now-updated self._settings, then reconnect.
self._config = self._build_config() self._config = self._build_config(self._settings)
await self._disconnect() await self._disconnect()
await self._connect() await self._connect()
elif changed.keys() & SpeechmaticsSTTSettings.HOT_FIELDS: elif changed.keys() & SpeechmaticsSTTSettings.HOT_FIELDS:
@@ -661,13 +663,17 @@ class SpeechmaticsSTTService(STTService):
# CONFIGURATION # CONFIGURATION
# ============================================================================ # ============================================================================
def _build_config(self) -> VoiceAgentConfig: def _build_config(self, settings: SpeechmaticsSTTSettings) -> VoiceAgentConfig:
"""Build a ``VoiceAgentConfig`` from the current ``self._settings``. """Build a ``VoiceAgentConfig`` from the given settings.
Used both at init time and before reconnecting so the connection Used both at init time (with explicit settings, before
always reflects the latest settings. ``super().__init__`` has run) and before reconnecting so the
connection always reflects the latest settings.
Args:
settings: Settings to build from.
""" """
s = self._settings s = settings
# Preset from turn detection mode # Preset from turn detection mode
config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value) config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value)

21
src/pipecat/services/speechmatics/tts.py
View File

@@ -95,7 +95,18 @@ class SpeechmaticsTTSService(TTSService):
f"Speechmatics TTS only supports {self.SPEECHMATICS_SAMPLE_RATE}Hz sample rate. " f"Speechmatics TTS only supports {self.SPEECHMATICS_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues." f"Current rate of {sample_rate}Hz may cause issues."
) )
super().__init__(sample_rate=sample_rate, **kwargs) params = params or SpeechmaticsTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=SpeechmaticsTTSSettings(
model=None,
voice=voice_id,
language=None,
max_retries=params.max_retries,
),
**kwargs,
)
# Service parameters # Service parameters
self._api_key: str = api_key self._api_key: str = api_key
@@ -106,14 +117,6 @@ class SpeechmaticsTTSService(TTSService):
if not self._api_key: if not self._api_key:
raise ValueError("Missing Speechmatics API key") raise ValueError("Missing Speechmatics API key")
params = params or SpeechmaticsTTSService.InputParams()
self._settings = SpeechmaticsTTSSettings(
model=None,
voice=voice_id,
language=None,
max_retries=params.max_retries,
)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics. """Check if this service can generate processing metrics.

11
src/pipecat/services/stt_service.py
View File

@@ -86,6 +86,7 @@ class STTService(AIService):
ttfs_p99_latency: Optional[float] = None, ttfs_p99_latency: Optional[float] = None,
keepalive_timeout: Optional[float] = None, keepalive_timeout: Optional[float] = None,
keepalive_interval: float = 5.0, keepalive_interval: float = 5.0,
settings: Optional[STTSettings] = None,
**kwargs, **kwargs,
): ):
"""Initialize the STT service. """Initialize the STT service.
@@ -109,14 +110,20 @@ class STTService(AIService):
connection alive. None disables keepalive. Useful for services that connection alive. None disables keepalive. Useful for services that
close idle connections (e.g. behind a ServiceSwitcher). close idle connections (e.g. behind a ServiceSwitcher).
keepalive_interval: Seconds between idle checks when keepalive is enabled. keepalive_interval: Seconds between idle checks when keepalive is enabled.
settings: The runtime-updatable settings for the STT service.
**kwargs: Additional arguments passed to the parent AIService. **kwargs: Additional arguments passed to the parent AIService.
""" """
super().__init__(**kwargs) super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or STTSettings(),
**kwargs,
)
self._audio_passthrough = audio_passthrough self._audio_passthrough = audio_passthrough
self._init_sample_rate = sample_rate self._init_sample_rate = sample_rate
self._sample_rate = 0 self._sample_rate = 0
self._settings = STTSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._muted: bool = False self._muted: bool = False
self._user_id: str = "" self._user_id: str = ""
self._ttfs_p99_latency = ttfs_p99_latency self._ttfs_p99_latency = ttfs_p99_latency

11
src/pipecat/services/tts_service.py
View File

@@ -147,6 +147,7 @@ class TTSService(AIService):
text_filter: Optional[BaseTextFilter] = None, text_filter: Optional[BaseTextFilter] = None,
# Audio transport destination of the generated frames. # Audio transport destination of the generated frames.
transport_destination: Optional[str] = None, transport_destination: Optional[str] = None,
settings: Optional[TTSSettings] = None,
**kwargs, **kwargs,
): ):
"""Initialize the TTS service. """Initialize the TTS service.
@@ -183,9 +184,16 @@ class TTSService(AIService):
Use `text_filters` instead, which allows multiple filters. Use `text_filters` instead, which allows multiple filters.
transport_destination: Destination for generated audio frames. transport_destination: Destination for generated audio frames.
settings: The runtime-updatable settings for the TTS service.
**kwargs: Additional arguments passed to the parent AIService. **kwargs: Additional arguments passed to the parent AIService.
""" """
super().__init__(**kwargs) super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or TTSSettings(),
**kwargs,
)
self._aggregate_sentences: bool = aggregate_sentences self._aggregate_sentences: bool = aggregate_sentences
self._push_text_frames: bool = push_text_frames self._push_text_frames: bool = push_text_frames
self._push_stop_frames: bool = push_stop_frames self._push_stop_frames: bool = push_stop_frames
@@ -196,7 +204,6 @@ class TTSService(AIService):
self._append_trailing_space: bool = append_trailing_space self._append_trailing_space: bool = append_trailing_space
self._init_sample_rate = sample_rate self._init_sample_rate = sample_rate
self._sample_rate = 0 self._sample_rate = 0
self._settings = TTSSettings() # Here in case subclass doesn't implement more specific settings (hopefully shouldn't happen)
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
if text_aggregator: if text_aggregator:
import warnings import warnings

28
src/pipecat/services/ultravox/llm.py
View File

@@ -176,19 +176,21 @@ class UltravoxRealtimeLLMService(LLMService):
May only be set with OneShotInputParams. May only be set with OneShotInputParams.
**kwargs: Additional arguments passed to parent LLMService. **kwargs: Additional arguments passed to parent LLMService.
""" """
super().__init__(**kwargs) super().__init__(
self._settings = UltravoxRealtimeLLMSettings( settings=UltravoxRealtimeLLMSettings(
model=None, model=None,
temperature=None, temperature=None,
max_tokens=None, max_tokens=None,
top_p=None, top_p=None,
top_k=None, top_k=None,
frequency_penalty=None, frequency_penalty=None,
presence_penalty=None, presence_penalty=None,
seed=None, seed=None,
filter_incomplete_user_turns=False, filter_incomplete_user_turns=False,
user_turn_completion_config=None, user_turn_completion_config=None,
output_medium=None, output_medium=None,
),
**kwargs,
) )
self._params = params self._params = params
if one_shot_selected_tools: if one_shot_selected_tools:

14
src/pipecat/services/vision_service.py
View File

@@ -12,11 +12,12 @@ visual content.
""" """
from abc import abstractmethod from abc import abstractmethod
from typing import AsyncGenerator from typing import AsyncGenerator, Optional
from pipecat.frames.frames import Frame, UserImageRawFrame from pipecat.frames.frames import Frame, UserImageRawFrame
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService from pipecat.services.ai_service import AIService
from pipecat.services.settings import VisionSettings
class VisionService(AIService): class VisionService(AIService):
@@ -27,13 +28,20 @@ class VisionService(AIService):
with the AI service infrastructure for metrics and lifecycle management. with the AI service infrastructure for metrics and lifecycle management.
""" """
def __init__(self, **kwargs): def __init__(self, *, settings: Optional[VisionSettings] = None, **kwargs):
"""Initialize the vision service. """Initialize the vision service.
Args: Args:
settings: The runtime-updatable settings for the vision service.
**kwargs: Additional arguments passed to the parent AIService. **kwargs: Additional arguments passed to the parent AIService.
""" """
super().__init__(**kwargs) super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or VisionSettings(),
**kwargs,
)
self._describe_text = None self._describe_text = None
@abstractmethod @abstractmethod

23
src/pipecat/services/whisper/base_stt.py
View File

@@ -155,22 +155,23 @@ class BaseWhisperSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService. **kwargs: Additional arguments passed to SegmentedSTTService.
""" """
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs) super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=BaseWhisperSTTSettings(
model=model,
language=self.language_to_service_language(language or Language.EN),
base_url=base_url,
prompt=prompt,
temperature=temperature,
),
**kwargs,
)
self._client = self._create_client(api_key, base_url) self._client = self._create_client(api_key, base_url)
self._language = self.language_to_service_language(language or Language.EN) self._language = self._settings.language
self._prompt = prompt self._prompt = prompt
self._temperature = temperature self._temperature = temperature
self._include_prob_metrics = include_prob_metrics self._include_prob_metrics = include_prob_metrics
self._settings = BaseWhisperSTTSettings(
model=model,
language=self._language,
base_url=base_url,
prompt=self._prompt,
temperature=self._temperature,
)
self._sync_model_name_to_metrics()
def _create_client(self, api_key: Optional[str], base_url: Optional[str]): def _create_client(self, api_key: Optional[str], base_url: Optional[str]):
return AsyncOpenAI(api_key=api_key, base_url=base_url) return AsyncOpenAI(api_key=api_key, base_url=base_url)

41
src/pipecat/services/whisper/stt.py
View File

@@ -233,21 +233,21 @@ class WhisperSTTService(SegmentedSTTService):
language: The default language for transcription. language: The default language for transcription.
**kwargs: Additional arguments passed to SegmentedSTTService. **kwargs: Additional arguments passed to SegmentedSTTService.
""" """
super().__init__(**kwargs) super().__init__(
settings=WhisperSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
device=device,
compute_type=compute_type,
no_speech_prob=no_speech_prob,
),
**kwargs,
)
self._device: str = device self._device: str = device
self._compute_type = compute_type self._compute_type = compute_type
self._no_speech_prob = no_speech_prob self._no_speech_prob = no_speech_prob
self._model: Optional[WhisperModel] = None self._model: Optional[WhisperModel] = None
self._settings = WhisperSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
device=self._device,
compute_type=self._compute_type,
no_speech_prob=self._no_speech_prob,
)
self._sync_model_name_to_metrics()
self._load() self._load()
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
@@ -368,20 +368,21 @@ class WhisperSTTServiceMLX(WhisperSTTService):
**kwargs: Additional arguments passed to SegmentedSTTService. **kwargs: Additional arguments passed to SegmentedSTTService.
""" """
# Skip WhisperSTTService.__init__ and call its parent directly # Skip WhisperSTTService.__init__ and call its parent directly
SegmentedSTTService.__init__(self, **kwargs) SegmentedSTTService.__init__(
self,
settings=WhisperMLXSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
no_speech_prob=no_speech_prob,
temperature=temperature,
engine="mlx",
),
**kwargs,
)
self._no_speech_prob = no_speech_prob self._no_speech_prob = no_speech_prob
self._temperature = temperature self._temperature = temperature
self._settings = WhisperMLXSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
no_speech_prob=self._no_speech_prob,
temperature=self._temperature,
engine="mlx",
)
self._sync_model_name_to_metrics()
# No need to call _load() as MLX Whisper loads models on demand # No need to call _load() as MLX Whisper loads models on demand
@override @override

16
src/pipecat/services/xtts/tts.py
View File

@@ -111,13 +111,15 @@ class XTTSService(TTSService):
sample_rate: Audio sample rate. If None, uses default. sample_rate: Audio sample rate. If None, uses default.
**kwargs: Additional arguments passed to parent TTSService. **kwargs: Additional arguments passed to parent TTSService.
""" """
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(
sample_rate=sample_rate,
self._settings = XTTSTTSSettings( settings=XTTSTTSSettings(
model=None, model=None,
voice=voice_id, voice=voice_id,
language=self.language_to_service_language(language), language=self.language_to_service_language(language),
base_url=base_url, base_url=base_url,
),
**kwargs,
) )
self._studio_speakers: Optional[Dict[str, Any]] = None self._studio_speakers: Optional[Dict[str, Any]] = None
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session