wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into filipi/lemonslice

Browse Source 
# Conflicts:
#	README.md
#	uv.lock
This commit is contained in:
filipi87
2026-03-02 19:24:52 -03:00
parent f07e55a4ed a4796a2373
commit 49c73bb0a3
351 changed files with 22154 additions and 4596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/pipecat/audio/filters/krisp_viva_filter.py
View File

@@ -39,7 +39,11 @@ class KrispVivaFilter(BaseAudioFilter):
"""
def __init__(
self, model_path: str = None, frame_duration: int = 10, noise_suppression_level: int = 100
self,
model_path: str = None,
frame_duration: int = 10,
noise_suppression_level: int = 100,
api_key: str = "",
) -> None:
"""Initialize the Krisp noise reduction filter.
@@ -48,6 +52,8 @@ class KrispVivaFilter(BaseAudioFilter):
If None, uses KRISP_VIVA_FILTER_MODEL_PATH environment variable.
frame_duration: Frame duration in milliseconds.
noise_suppression_level: Noise suppression level.
api_key: Krisp SDK API key. If empty, falls back to
the KRISP_VIVA_API_KEY environment variable.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_FILTER_MODEL_PATH is not set.
@@ -57,6 +63,8 @@ class KrispVivaFilter(BaseAudioFilter):
"""
super().__init__()
self._api_key = api_key
try:
# Set model path, checking environment if not specified
if model_path:
@@ -132,7 +140,7 @@ class KrispVivaFilter(BaseAudioFilter):
"""
try:
# Acquire SDK reference (will initialize on first call)
KrispVivaSDKManager.acquire()
KrispVivaSDKManager.acquire(api_key=self._api_key)
self._session = self._create_session(sample_rate, self._frame_duration_ms)
except Exception as e:
logger.error(f"Failed to start Krisp session: {e}", exc_info=True)

26
src/pipecat/audio/krisp_instance.py
View File

@@ -7,6 +7,7 @@
"""Krisp Instance manager for pipecat audio."""
import atexit
import os
from threading import Lock
from loguru import logger
@@ -88,17 +89,26 @@ class KrispVivaSDKManager:
_lock = Lock()
_reference_count = 0
@staticmethod
def _license_callback(error, error_message):
"""Callback for Krisp SDK licensing errors."""
logger.error(f"Krisp licensing error: {error} - {error_message}")
@staticmethod
def _log_callback(log_message, log_level):
"""Thread-safe callback for Krisp SDK logging."""
logger.info(f"[{log_level}] {log_message}")
@classmethod
def acquire(cls):
def acquire(cls, api_key: str = ""):
"""Acquire a reference to the SDK (initializes if needed).
Call this when creating a filter instance.
Args:
api_key: Krisp SDK API key. If empty, falls back to the
KRISP_VIVA_API_KEY environment variable.
Raises:
Exception: If SDK initialization fails (propagated from krisp_audio)
"""
@@ -106,7 +116,19 @@ class KrispVivaSDKManager:
# Initialize SDK on first acquire
if cls._reference_count == 0:
try:
krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
key = api_key or os.environ.get("KRISP_VIVA_API_KEY", "")
try:
# New SDK signature (requires license key)
krisp_audio.globalInit(
"",
key,
cls._license_callback,
cls._log_callback,
krisp_audio.LogLevel.Off,
)
except TypeError:
# Old SDK signature (no license key)
krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
cls._initialized = True

39
src/pipecat/audio/turn/krisp_viva_turn.py
View File

@@ -15,6 +15,7 @@ passed directly to the constructor.
"""
import os
import time
from typing import Optional, Tuple
import numpy as np
@@ -26,7 +27,7 @@ from pipecat.audio.krisp_instance import (
int_to_krisp_sample_rate,
)
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData
from pipecat.metrics.metrics import MetricsData, TurnMetricsData
try:
import krisp_audio
@@ -63,6 +64,7 @@ class KrispVivaTurn(BaseTurnAnalyzer):
model_path: Optional[str] = None,
sample_rate: Optional[int] = None,
params: Optional[KrispTurnParams] = None,
api_key: str = "",
) -> None:
"""Initialize the Krisp turn analyzer.
@@ -72,6 +74,8 @@ class KrispVivaTurn(BaseTurnAnalyzer):
sample_rate: Optional initial sample rate for audio processing.
If provided, this will be used as the fixed sample rate.
params: Configuration parameters for turn analysis behavior.
api_key: Krisp SDK API key. If empty, falls back to
the KRISP_VIVA_API_KEY environment variable.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_TURN_MODEL_PATH is not set.
@@ -83,7 +87,7 @@ class KrispVivaTurn(BaseTurnAnalyzer):
# Acquire SDK reference (will initialize on first call)
try:
KrispVivaSDKManager.acquire()
KrispVivaSDKManager.acquire(api_key=api_key)
self._sdk_acquired = True
except Exception as e:
self._sdk_acquired = False
@@ -115,6 +119,9 @@ class KrispVivaTurn(BaseTurnAnalyzer):
self._last_probability = None
self._frame_probabilities = []
self._last_state = EndOfTurnState.INCOMPLETE
self._speech_stopped_time: Optional[float] = None
self._e2e_processing_time_ms: Optional[float] = None
self._last_metrics: Optional[TurnMetricsData] = None
# Create session with provided sample rate or default to 16000 Hz
# This preloads the model to improve latency when set_sample_rate is called later
@@ -288,7 +295,14 @@ class KrispVivaTurn(BaseTurnAnalyzer):
# Track speech start time
if not self._speech_triggered:
logger.trace("Speech detected, turn analysis started")
self._e2e_processing_time_ms = None
self._speech_triggered = True
# Reset speech stopped time when speech resumes
self._speech_stopped_time = None
else:
# Record the moment speech transitions to non-speech
if self._speech_triggered and self._speech_stopped_time is None:
self._speech_stopped_time = time.perf_counter()
# Note: We don't immediately mark as complete on silence detection.
# Instead, we wait for the model's probability check below to confirm
# end-of-turn based on the threshold.
@@ -308,6 +322,18 @@ class KrispVivaTurn(BaseTurnAnalyzer):
# Only mark as complete if we've detected speech and the model
# confirms with sufficient confidence
if self._speech_triggered and prob >= self._params.threshold:
# Calculate e2e processing time: time from speech stop to threshold crossing
if self._speech_stopped_time is not None:
self._e2e_processing_time_ms = (
time.perf_counter() - self._speech_stopped_time
) * 1000
self._last_metrics = TurnMetricsData(
processor="KrispVivaTurn",
is_complete=True,
probability=prob,
e2e_processing_time_ms=self._e2e_processing_time_ms,
)
logger.debug(f"Krisp turn complete")
state = EndOfTurnState.COMPLETE
self.clear()
break
@@ -329,12 +355,15 @@ class KrispVivaTurn(BaseTurnAnalyzer):
Tuple containing the end-of-turn state and optional metrics data.
Returns the last state determined by append_audio().
"""
# For real-time processing, the state is determined in append_audio
# Return the last state that was computed
return self._last_state, None
# For real-time processing, the state is determined in append_audio.
# Consume metrics so they aren't pushed twice.
metrics = self._last_metrics
self._last_metrics = None
return self._last_state, metrics
def clear(self):
"""Reset the turn analyzer to its initial state."""
self._speech_triggered = False
self._audio_buffer.clear()
self._last_state = EndOfTurnState.INCOMPLETE
self._speech_stopped_time = None

13
src/pipecat/audio/turn/smart_turn/base_smart_turn.py
View File

@@ -21,7 +21,7 @@ import numpy as np
from loguru import logger
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData
from pipecat.metrics.metrics import MetricsData, TurnMetricsData
# Default timing parameters
STOP_SECS = 3
@@ -222,18 +222,11 @@ class BaseSmartTurn(BaseTurnAnalyzer):
# Calculate processing time
e2e_processing_time_ms = (end_time - start_time) * 1000
# Extract metrics from the nested structure
metrics = result.get("metrics", {})
inference_time = metrics.get("inference_time", 0)
total_time = metrics.get("total_time", 0)
# Prepare the result data
result_data = SmartTurnMetricsData(
result_data = TurnMetricsData(
processor="BaseSmartTurn",
is_complete=result["prediction"] == 1,
probability=result["probability"],
inference_time_ms=inference_time * 1000,
server_total_time_ms=total_time * 1000,
e2e_processing_time_ms=e2e_processing_time_ms,
)
@@ -241,8 +234,6 @@ class BaseSmartTurn(BaseTurnAnalyzer):
f"Prediction: {'Complete' if result_data.is_complete else 'Incomplete'}"
)
logger.trace(f"Probability of complete: {result_data.probability:.4f}")
logger.trace(f"Inference time: {result_data.inference_time_ms:.2f}ms")
logger.trace(f"Server total time: {result_data.server_total_time_ms:.2f}ms")
logger.trace(f"E2E processing time: {result_data.e2e_processing_time_ms:.2f}ms")
except SmartTurnTimeoutException:
logger.debug(

45
src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py
View File

@@ -13,20 +13,16 @@ local end-of-turn detection without requiring network connectivity.
from typing import Any, Dict, Optional
import numpy as np
import onnxruntime as ort
import soxr
from loguru import logger
from transformers import WhisperFeatureExtractor
from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn
from pipecat.utils.env import env_truthy
try:
import onnxruntime as ort
from transformers import WhisperFeatureExtractor
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use LocalSmartTurnAnalyzerV3, you need to `pip install pipecat-ai[local-smart-turn-v3]`."
)
raise Exception(f"Missing module: {e}")
# The Whisper-based ONNX model expects 16 kHz audio input.
_MODEL_SAMPLE_RATE = 16000
class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
@@ -85,7 +81,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
logger.debug("Loaded Local Smart Turn v3.x")
def _write_audio_to_wav(
self, audio_array: np.ndarray, sample_rate: int = 16000, suffix: str = ""
self, audio_array: np.ndarray, sample_rate: int = _MODEL_SAMPLE_RATE, suffix: str = ""
) -> None:
"""Write audio data to a WAV file in a background thread.
@@ -127,10 +123,27 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
thread = threading.Thread(target=write_wav, daemon=True)
thread.start()
def _resample_to_model_rate(self, audio_array: np.ndarray) -> np.ndarray:
"""Resample audio to the model's expected sample rate (16 kHz).
Args:
audio_array: Audio data as a float32 numpy array.
Returns:
Resampled audio array at 16 kHz.
"""
actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
if actual_rate == _MODEL_SAMPLE_RATE:
return audio_array
return soxr.resample(audio_array, actual_rate, _MODEL_SAMPLE_RATE, quality="VHQ")
def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
"""Predict end-of-turn using local ONNX model."""
def truncate_audio_to_last_n_seconds(audio_array, n_seconds=8, sample_rate=16000):
def truncate_audio_to_last_n_seconds(
audio_array, n_seconds=8, sample_rate=_MODEL_SAMPLE_RATE
):
"""Truncate audio to last n seconds or pad with zeros to meet n seconds."""
max_samples = n_seconds * sample_rate
if len(audio_array) > max_samples:
@@ -142,6 +155,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
return audio_array
audio_for_logging = audio_array
actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
# Resample to 16 kHz if the pipeline uses a different sample rate
audio_array = self._resample_to_model_rate(audio_array)
# Truncate to 8 seconds (keeping the end) or pad to 8 seconds
audio_array = truncate_audio_to_last_n_seconds(audio_array, n_seconds=8)
@@ -149,10 +166,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
# Process audio using Whisper's feature extractor
inputs = self._feature_extractor(
audio_array,
sampling_rate=16000,
sampling_rate=_MODEL_SAMPLE_RATE,
return_tensors="np",
padding="max_length",
max_length=8 * 16000,
max_length=8 * _MODEL_SAMPLE_RATE,
truncation=True,
do_normalize=True,
)
@@ -172,7 +189,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
if self._log_data:
suffix = "_complete" if prediction == 1 else "_incomplete"
self._write_audio_to_wav(audio_for_logging, sample_rate=16000, suffix=suffix)
self._write_audio_to_wav(audio_for_logging, sample_rate=actual_rate, suffix=suffix)
return {
"prediction": prediction,

2
src/pipecat/extensions/voicemail/voicemail_detector.py
View File

@@ -368,7 +368,7 @@ class ClassificationProcessor(FrameProcessor):
await self._voicemail_notifier.notify() # Clear buffered TTS frames
# Interrupt the current pipeline to stop any ongoing processing
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
# Set the voicemail event to trigger the voicemail handler
self._voicemail_event.clear()

109
src/pipecat/frames/frames.py
View File

@@ -11,10 +11,8 @@ including data frames, system frames, and control frames for audio, video, text,
and LLM processing.
"""
import asyncio
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import (
TYPE_CHECKING,
Any,
@@ -36,12 +34,15 @@ from pipecat.audio.turn.base_turn_analyzer import BaseTurnParams
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import AggregationType
from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
if TYPE_CHECKING:
from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.services.settings import ServiceSettings
from pipecat.utils.context.llm_context_summarization import LLMContextSummaryConfig
from pipecat.utils.tracing.tracing_context import TracingContext
@@ -392,16 +393,6 @@ class LLMTextFrame(TextFrame):
self.includes_inter_frame_spaces = True
class AggregationType(str, Enum):
"""Built-in aggregation strings."""
SENTENCE = "sentence"
WORD = "word"
def __str__(self):
return self.value
@dataclass
class AggregatedTextFrame(TextFrame):
"""Text frame representing an aggregation of TextFrames.
@@ -1149,24 +1140,9 @@ class InterruptionFrame(SystemFrame):
This frame is used to interrupt the pipeline. For example, when a user
starts speaking to cancel any in-progress bot output. It can also be pushed
by any processor.
Parameters:
event: Optional event set when the frame has fully traversed the
pipeline.
"""
event: Optional[asyncio.Event] = None
def complete(self):
"""Signal that this interruption has been fully processed.
Called automatically when the frame reaches the pipeline sink, or
manually when the frame is consumed before reaching it (e.g. when
the user is muted).
"""
if self.event:
self.event.set()
pass
@dataclass
@@ -1833,16 +1809,11 @@ class InterruptionTaskFrame(TaskFrame):
"""Frame indicating the pipeline should be interrupted.
This frame should be pushed upstream to indicate the pipeline should be
interrupted. The pipeline task converts this into an `InterruptionFrame` and
sends it downstream. The `event` is passed to the `InterruptionFrame` so it
can signal when the interruption has fully traversed the pipeline.
Parameters:
event: Optional event passed to the corresponding `InterruptionFrame`.
interrupted. The pipeline task converts this into an `InterruptionFrame`
and sends it downstream.
"""
event: Optional[asyncio.Event] = None
pass
@dataclass
@@ -1918,6 +1889,29 @@ class StopFrame(ControlFrame, UninterruptibleFrame):
pass
@dataclass
class BotConnectedFrame(SystemFrame):
"""Frame indicating the bot has connected to the transport service.
Pushed downstream by SFU transports (Daily, LiveKit, HeyGen, Tavus)
when the bot successfully joins the room. Non-SFU transports do not
emit this frame.
"""
pass
@dataclass
class ClientConnectedFrame(SystemFrame):
"""Frame indicating that a client has connected to the transport.
Pushed downstream by the input transport when a client (participant)
connects. Used by observers to measure transport readiness timing.
"""
pass
@dataclass
class OutputTransportReadyFrame(ControlFrame):
"""Frame indicating that the output transport is ready.
@@ -1999,6 +1993,32 @@ class LLMFullResponseEndFrame(ControlFrame):
self.skip_tts = None
@dataclass
class LLMAssistantPushAggregationFrame(ControlFrame):
"""Frame that forces the LLM assistant aggregator to push its current aggregation to context.
When received by ``LLMAssistantAggregator``, any text that has been accumulated
in the aggregation buffer is immediately committed to the conversation context as
an assistant message, without waiting for an ``LLMFullResponseEndFrame``.
"""
@dataclass
class LLMSummarizeContextFrame(ControlFrame):
"""Frame requesting on-demand context summarization.
Push this frame into the pipeline to trigger a manual context summarization.
Parameters:
config: Optional per-request override for summary generation settings
(prompt, token budget, messages to keep). If ``None``, the
summarizer's default :class:`~pipecat.utils.context.llm_context_summarization.LLMContextSummaryConfig`
is used.
"""
config: Optional["LLMContextSummaryConfig"] = None
@dataclass
class LLMContextSummaryRequestFrame(ControlFrame):
"""Frame requesting context summarization from an LLM service.
@@ -2018,6 +2038,8 @@ class LLMContextSummaryRequestFrame(ControlFrame):
the summary text.
summarization_prompt: System prompt instructing the LLM how to generate
the summary.
summarization_timeout: Maximum time in seconds for the LLM to generate a
summary. When None, a default timeout of 120s is applied.
"""
request_id: str
@@ -2025,6 +2047,7 @@ class LLMContextSummaryRequestFrame(ControlFrame):
min_messages_to_keep: int
target_context_tokens: int
summarization_prompt: str
summarization_timeout: Optional[float] = None
@dataclass
@@ -2117,16 +2140,24 @@ class TTSStoppedFrame(ControlFrame):
@dataclass
class ServiceUpdateSettingsFrame(ControlFrame):
class ServiceUpdateSettingsFrame(ControlFrame, UninterruptibleFrame):
"""Base frame for updating service settings.
A control frame containing a request to update service settings.
Supports both a ``settings`` dict (for backward compatibility) and a
``delta`` object. When both are provided, ``delta`` takes precedence.
Parameters:
settings: Dictionary of setting name to value mappings.
.. deprecated:: 0.0.104
Use ``delta`` with a typed settings object instead.
delta: :class:`~pipecat.services.settings.ServiceSettings` delta-mode
object describing the fields to change.
"""
settings: Mapping[str, Any]
settings: Mapping[str, Any] = field(default_factory=dict)
delta: Optional["ServiceSettings"] = None
@dataclass

39
src/pipecat/metrics/metrics.py
View File

@@ -87,19 +87,44 @@ class TTSUsageMetricsData(MetricsData):
value: int
class SmartTurnMetricsData(MetricsData):
"""Metrics data for smart turn predictions.
class TextAggregationMetricsData(MetricsData):
"""Text aggregation time metrics data.
Measures the time from the first LLM token to the first complete sentence,
representing the latency cost of sentence aggregation in the TTS pipeline.
Parameters:
value: Aggregation time in seconds.
"""
value: float
class TurnMetricsData(MetricsData):
"""Metrics data for turn detection predictions.
Parameters:
is_complete: Whether the turn is predicted to be complete.
probability: Confidence probability of the turn completion prediction.
inference_time_ms: Time taken for inference in milliseconds.
server_total_time_ms: Total server processing time in milliseconds.
e2e_processing_time_ms: End-to-end processing time in milliseconds.
e2e_processing_time_ms: End-to-end processing time in milliseconds,
measured from VAD speech-to-silence transition to turn completion.
"""
is_complete: bool
probability: float
inference_time_ms: float
server_total_time_ms: float
e2e_processing_time_ms: float
class SmartTurnMetricsData(TurnMetricsData):
"""Metrics data for smart turn predictions.
.. deprecated:: 0.0.104
Use :class:`TurnMetricsData` instead. This class will be removed in a future version.
Parameters:
inference_time_ms: Time taken for inference in milliseconds.
server_total_time_ms: Total server processing time in milliseconds.
"""
inference_time_ms: float = 0.0
server_total_time_ms: float = 0.0

8
src/pipecat/observers/base_observer.py
View File

@@ -100,3 +100,11 @@ class BaseObserver(BaseObject):
data: The event data containing details about the frame transfer.
"""
pass
async def on_pipeline_started(self):
"""Called when the pipeline has fully started.
Fired after the ``StartFrame`` has been processed by all processors
in the pipeline, including nested ``ParallelPipeline`` branches.
"""
pass

33
src/pipecat/observers/loggers/metrics_log_observer.py
View File

@@ -24,6 +24,7 @@ from pipecat.metrics.metrics import (
SmartTurnMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
TurnMetricsData,
)
from pipecat.observers.base_observer import BaseObserver, FramePushed
@@ -37,7 +38,7 @@ class MetricsLogObserver(BaseObserver):
- ProcessingMetricsData (General processing time)
- LLMUsageMetricsData (Token usage statistics)
- TTSUsageMetricsData (Text-to-Speech character counts)
- SmartTurnMetricsData (Turn prediction metrics)
- TurnMetricsData (Turn prediction metrics)
This allows developers to track performance metrics, token usage,
and other statistics throughout the pipeline.
@@ -70,6 +71,17 @@ class MetricsLogObserver(BaseObserver):
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
# Normalize deprecated types in include_metrics
if include_metrics and SmartTurnMetricsData in include_metrics:
import warnings
warnings.warn(
"SmartTurnMetricsData is deprecated in include_metrics, "
"use TurnMetricsData instead.",
DeprecationWarning,
stacklevel=2,
)
include_metrics = (include_metrics - {SmartTurnMetricsData}) | {TurnMetricsData}
self._include_metrics = include_metrics
self._frames_seen = set()
@@ -144,8 +156,8 @@ class MetricsLogObserver(BaseObserver):
logger.debug(
f"📊 {processor_info} TTS USAGE{model_info}: {metrics_data.value} characters at {time_sec:.3f}s"
)
elif isinstance(metrics_data, SmartTurnMetricsData):
self._log_smart_turn(metrics_data, processor_info, model_info, time_sec)
elif isinstance(metrics_data, TurnMetricsData):
self._log_turn(metrics_data, processor_info, model_info, time_sec)
else:
# Generic fallback for unknown metrics types
logger.debug(
@@ -191,28 +203,27 @@ class MetricsLogObserver(BaseObserver):
f"📊 {processor_info} LLM TOKEN USAGE{model_info}: {usage_str} at {time_sec:.2f}s"
)
def _log_smart_turn(
def _log_turn(
self,
metrics_data: SmartTurnMetricsData,
metrics_data: TurnMetricsData,
processor_info: str,
model_info: str,
time_sec: float,
):
"""Log smart turn prediction metrics.
"""Log turn prediction metrics.
Args:
metrics_data: The smart turn metrics data.
metrics_data: The turn metrics data.
processor_info: Formatted processor name string.
model_info: Formatted model name string.
time_sec: Timestamp in seconds.
"""
complete_str = "COMPLETE" if metrics_data.is_complete else "INCOMPLETE"
e2e_str = f"{metrics_data.e2e_processing_time_ms:.1f}ms"
logger.debug(
f"📊 {processor_info} SMART TURN{model_info}: {complete_str} "
f"📊 {processor_info} TURN{model_info}: {complete_str} "
f"(probability: {metrics_data.probability:.2%}, "
f"inference: {metrics_data.inference_time_ms:.1f}ms, "
f"server: {metrics_data.server_total_time_ms:.1f}ms, "
f"e2e: {metrics_data.e2e_processing_time_ms:.1f}ms) "
f"e2e: {e2e_str}) "
f"at {time_sec:.2f}s"
)

328
src/pipecat/observers/startup_timing_observer.py Normal file
View File

@@ -0,0 +1,328 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Observer for tracking pipeline startup timing.
This module provides an observer that measures how long each processor's
``start()`` method takes during pipeline startup. It works by tracking
when a ``StartFrame`` arrives at a processor (``on_process_frame``) versus
when it leaves (``on_push_frame``), giving the exact ``start()`` duration
for each processor in the pipeline.
It also measures transport timing — the time from ``StartFrame`` to the
first ``BotConnectedFrame`` (SFU transports only) and ``ClientConnectedFrame``
— via a separate ``on_transport_timing_report`` event.
Example::
observer = StartupTimingObserver()
@observer.event_handler("on_startup_timing_report")
async def on_report(observer, report):
for t in report.processor_timings:
print(f"{t.processor_name}: {t.duration_secs:.3f}s")
@observer.event_handler("on_transport_timing_report")
async def on_transport(observer, report):
if report.bot_connected_secs is not None:
print(f"Bot connected in {report.bot_connected_secs:.3f}s")
print(f"Client connected in {report.client_connected_secs:.3f}s")
task = PipelineTask(pipeline, observers=[observer])
"""
import time
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Type
from pydantic import BaseModel, Field
from pipecat.frames.frames import BotConnectedFrame, ClientConnectedFrame, StartFrame
from pipecat.observers.base_observer import BaseObserver, FrameProcessed, FramePushed
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.pipeline import PipelineSource
from pipecat.processors.frame_processor import FrameProcessor
# Internal pipeline types excluded from tracking by default.
_INTERNAL_TYPES = (PipelineSource, BasePipeline)
@dataclass
class _ArrivalInfo:
"""Internal record of when a StartFrame arrived at a processor."""
processor: FrameProcessor
arrival_ts_ns: int
class ProcessorStartupTiming(BaseModel):
"""Startup timing for a single processor.
Parameters:
processor_name: The name of the processor.
start_offset_secs: Offset in seconds from the StartFrame to when this
processor's start() began.
duration_secs: How long the processor's start() took, in seconds.
"""
processor_name: str
start_offset_secs: float
duration_secs: float
class StartupTimingReport(BaseModel):
"""Report of startup timings for all measured processors.
Parameters:
start_time: Unix timestamp when the first processor began starting.
total_duration_secs: Total wall-clock time from first to last processor start.
processor_timings: Per-processor timing data, in pipeline order.
"""
start_time: float
total_duration_secs: float
processor_timings: List[ProcessorStartupTiming] = Field(default_factory=list)
class TransportTimingReport(BaseModel):
"""Time from pipeline start to transport connection milestones.
Parameters:
start_time: Unix timestamp of the StartFrame (pipeline start).
bot_connected_secs: Seconds from StartFrame to first BotConnectedFrame
(only set for SFU transports).
client_connected_secs: Seconds from StartFrame to first ClientConnectedFrame.
"""
start_time: float
bot_connected_secs: Optional[float] = None
client_connected_secs: Optional[float] = None
class StartupTimingObserver(BaseObserver):
"""Observer that measures processor startup times during pipeline initialization.
Tracks how long each processor's ``start()`` method takes by measuring the
time between when a ``StartFrame`` arrives at a processor and when it is
pushed downstream. This captures WebSocket connections, API authentication,
model loading, and other initialization work.
Also measures transport timing, the time from ``StartFrame`` to connection
milestones:
- ``bot_connected_secs``: When the bot joins the transport room
(SFU transports only, triggered by ``BotConnectedFrame``).
- ``client_connected_secs``: When a remote participant connects
(triggered by ``ClientConnectedFrame``).
By default, internal pipeline processors (``PipelineSource``, ``Pipeline``)
are excluded from the report. Pass ``processor_types`` to measure only
specific types.
Event handlers available:
- on_startup_timing_report: Called once after startup completes with the full
timing report.
- on_transport_timing_report: Called once when the first client connects with a
TransportTimingReport containing client_connected_secs and bot_connected_secs
(if available).
Example::
observer = StartupTimingObserver(
processor_types=(STTService, TTSService)
)
@observer.event_handler("on_startup_timing_report")
async def on_report(observer, report):
for t in report.processor_timings:
logger.info(f"{t.processor_name}: {t.duration_secs:.3f}s")
@observer.event_handler("on_transport_timing_report")
async def on_transport(observer, report):
if report.bot_connected_secs is not None:
logger.info(f"Bot connected in {report.bot_connected_secs:.3f}s")
logger.info(f"Client connected in {report.client_connected_secs:.3f}s")
task = PipelineTask(pipeline, observers=[observer])
Args:
processor_types: Optional tuple of processor types to measure. If None,
all non-internal processors are measured.
"""
def __init__(
self,
*,
processor_types: Optional[Tuple[Type[FrameProcessor], ...]] = None,
**kwargs,
):
"""Initialize the startup timing observer.
Args:
processor_types: Optional tuple of processor types to measure.
If None, all non-internal processors are measured.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._processor_types = processor_types
# Map processor ID -> arrival info.
self._arrivals: Dict[int, _ArrivalInfo] = {}
# Collected timings in pipeline order.
self._timings: List[ProcessorStartupTiming] = []
# Lock onto the first StartFrame we see (by frame ID).
self._start_frame_id: Optional[str] = None
# Whether we've already emitted the startup timing report.
self._startup_timing_reported = False
# Whether we've already measured transport timing.
self._transport_timing_reported = False
# Timestamp (ns) when we first see a StartFrame arrive at a processor.
self._start_frame_arrival_ns: Optional[int] = None
# Bot connected timing (stored for inclusion in the transport report).
self._bot_connected_secs: Optional[float] = None
# Wall clock time when the StartFrame was first seen.
self._start_wall_clock: Optional[float] = None
self._register_event_handler("on_startup_timing_report")
self._register_event_handler("on_transport_timing_report")
def _should_track(self, processor: FrameProcessor) -> bool:
"""Check if a processor should be tracked for timing.
Args:
processor: The processor to check.
Returns:
True if the processor matches the filter or no filter is set.
"""
if self._processor_types is not None:
return isinstance(processor, self._processor_types)
# Default: exclude internal pipeline plumbing.
return not isinstance(processor, _INTERNAL_TYPES)
async def on_pipeline_started(self):
"""Emit the startup timing report when the pipeline has fully started.
Called by the ``PipelineTask`` after the ``StartFrame`` has been
processed by all processors, including nested ``ParallelPipeline``
branches.
"""
if self._timings:
await self._emit_report()
async def on_process_frame(self, data: FrameProcessed):
"""Record when a StartFrame arrives at a processor.
Args:
data: The frame processing event data.
"""
if self._startup_timing_reported:
return
if not isinstance(data.frame, StartFrame):
return
# Lock onto the first StartFrame.
if self._start_frame_id is None:
self._start_frame_id = data.frame.id
self._start_frame_arrival_ns = data.timestamp
self._start_wall_clock = time.time()
elif data.frame.id != self._start_frame_id:
return
if self._should_track(data.processor):
self._arrivals[data.processor.id] = _ArrivalInfo(
processor=data.processor, arrival_ts_ns=data.timestamp
)
async def on_push_frame(self, data: FramePushed):
"""Record when a StartFrame leaves a processor and compute the delta.
Also handles ``BotConnectedFrame`` and ``ClientConnectedFrame`` to
measure transport timing.
Args:
data: The frame push event data.
"""
if isinstance(data.frame, BotConnectedFrame):
self._handle_bot_connected(data)
return
if isinstance(data.frame, ClientConnectedFrame):
await self._handle_client_connected(data)
return
if self._startup_timing_reported:
return
if not isinstance(data.frame, StartFrame):
return
if self._start_frame_id is not None and data.frame.id != self._start_frame_id:
return
arrival = self._arrivals.pop(data.source.id, None)
if arrival is None:
return
duration_ns = data.timestamp - arrival.arrival_ts_ns
duration_secs = duration_ns / 1e9
start_offset_secs = (arrival.arrival_ts_ns - self._start_frame_arrival_ns) / 1e9
self._timings.append(
ProcessorStartupTiming(
processor_name=arrival.processor.name,
start_offset_secs=start_offset_secs,
duration_secs=duration_secs,
)
)
def _handle_bot_connected(self, data: FramePushed):
"""Record bot connected timing on first BotConnectedFrame."""
if self._bot_connected_secs is not None or self._start_frame_arrival_ns is None:
return
delta_ns = data.timestamp - self._start_frame_arrival_ns
self._bot_connected_secs = delta_ns / 1e9
async def _handle_client_connected(self, data: FramePushed):
"""Emit transport timing report on first ClientConnectedFrame."""
if self._transport_timing_reported or self._start_frame_arrival_ns is None:
return
self._transport_timing_reported = True
delta_ns = data.timestamp - self._start_frame_arrival_ns
client_connected_secs = delta_ns / 1e9
report = TransportTimingReport(
start_time=self._start_wall_clock or 0.0,
bot_connected_secs=self._bot_connected_secs,
client_connected_secs=client_connected_secs,
)
await self._call_event_handler("on_transport_timing_report", report)
async def _emit_report(self):
"""Build and emit the startup timing report."""
if self._startup_timing_reported:
return
self._startup_timing_reported = True
total = sum(t.duration_secs for t in self._timings)
report = StartupTimingReport(
start_time=self._start_wall_clock or 0.0,
total_duration_secs=total,
processor_timings=self._timings,
)
await self._call_event_handler("on_startup_timing_report", report)

47
src/pipecat/pipeline/task.py
View File

@@ -330,6 +330,7 @@ class PipelineTask(BasePipelineTask):
# RTVI support
self._rtvi = None
prepend_rtvi = False
external_rtvi = self._find_processor(pipeline, RTVIProcessor)
external_observer_found = any(isinstance(o, RTVIObserver) for o in observers)
@@ -352,6 +353,7 @@ class PipelineTask(BasePipelineTask):
elif enable_rtvi:
self._rtvi = rtvi_processor or RTVIProcessor()
observers.append(self._rtvi.create_rtvi_observer(params=rtvi_observer_params))
prepend_rtvi = True
if self._rtvi:
# Automatically call RTVIProcessor.set_bot_ready()
@@ -387,9 +389,12 @@ class PipelineTask(BasePipelineTask):
# source allows us to receive and react to upstream frames, and the sink
# allows us to receive and react to downstream frames.
source = PipelineSource(self._source_push_frame, name=f"{self}::Source")
sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
processors = [self._rtvi, pipeline] if self._rtvi else [pipeline]
self._pipeline = Pipeline(processors, source=source, sink=sink)
self._sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
# Only prepend the RTVIProcessor if we created it ourselves. When the
# user already placed it inside their pipeline we must not insert it
# again or it will appear twice in the frame chain.
processors = [self._rtvi, pipeline] if prepend_rtvi else [pipeline]
self._pipeline = Pipeline(processors, source=source, sink=self._sink)
# The task observer acts as a proxy to the provided observers. This way,
# we only need to pass a single observer (using the StartFrame) which
@@ -620,26 +625,43 @@ class PipelineTask(BasePipelineTask):
self._finished = True
logger.debug(f"Pipeline task {self} has finished")
async def queue_frame(self, frame: Frame):
"""Queue a single frame to be pushed down the pipeline.
async def queue_frame(
self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
):
"""Queue a single frame to be pushed through the pipeline.
Downstream frames are pushed from the beginning of the pipeline.
Upstream frames are pushed from the end of the pipeline.
Args:
frame: The frame to be processed.
direction: The direction to push the frame. Defaults to downstream.
"""
await self._push_queue.put(frame)
if direction == FrameDirection.DOWNSTREAM:
await self._push_queue.put(frame)
else:
await self._sink.queue_frame(frame, direction)
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
"""Queues multiple frames to be pushed down the pipeline.
async def queue_frames(
self,
frames: Iterable[Frame] | AsyncIterable[Frame],
direction: FrameDirection = FrameDirection.DOWNSTREAM,
):
"""Queue multiple frames to be pushed through the pipeline.
Downstream frames are pushed from the beginning of the pipeline.
Upstream frames are pushed from the end of the pipeline.
Args:
frames: An iterable or async iterable of frames to be processed.
direction: The direction to push the frames. Defaults to downstream.
"""
if isinstance(frames, AsyncIterable):
async for frame in frames:
await self.queue_frame(frame)
await self.queue_frame(frame, direction)
elif isinstance(frames, Iterable):
for frame in frames:
await self.queue_frame(frame)
await self.queue_frame(frame, direction)
async def _cancel(self, *, reason: Optional[str] = None):
"""Internal cancellation logic for the pipeline task.
@@ -870,7 +892,7 @@ class PipelineTask(BasePipelineTask):
# pipeline. This is in case the push task is blocked waiting for a
# pipeline-ending frame to finish traversing the pipeline.
logger.debug(f"{self}: received interruption task frame {frame}")
await self._pipeline.queue_frame(InterruptionFrame(event=frame.event))
await self._pipeline.queue_frame(InterruptionFrame())
elif isinstance(frame, ErrorFrame):
await self._call_event_handler("on_pipeline_error", frame)
if frame.fatal:
@@ -893,6 +915,7 @@ class PipelineTask(BasePipelineTask):
if isinstance(frame, StartFrame):
await self._call_event_handler("on_pipeline_started", frame)
await self._observer.on_pipeline_started()
# Start heartbeat tasks now that StartFrame has been processed
# by all processors in the pipeline
@@ -909,8 +932,6 @@ class PipelineTask(BasePipelineTask):
self._pipeline_end_event.set()
elif isinstance(frame, CancelFrame):
self._pipeline_end_event.set()
elif isinstance(frame, InterruptionFrame):
frame.complete()
elif isinstance(frame, HeartbeatFrame):
await self._heartbeat_queue.put(frame)

14
src/pipecat/pipeline/task_observer.py
View File

@@ -39,6 +39,12 @@ class Proxy:
observer: BaseObserver
class _PipelineStartedSignal:
"""Internal sentinel queued to observers when the pipeline has started."""
pass
class TaskObserver(BaseObserver):
"""Proxy observer that manages multiple observers without blocking the pipeline.
@@ -129,6 +135,10 @@ class TaskObserver(BaseObserver):
for proxy in self._proxies:
await proxy.cleanup()
async def on_pipeline_started(self):
"""Forward pipeline started signal to all managed observers."""
await self._send_to_proxy(_PipelineStartedSignal())
async def on_process_frame(self, data: FrameProcessed):
"""Queue frame data for all managed observers.
@@ -186,7 +196,9 @@ class TaskObserver(BaseObserver):
while True:
data = await queue.get()
if isinstance(data, FramePushed):
if isinstance(data, _PipelineStartedSignal):
await observer.on_pipeline_started()
elif isinstance(data, FramePushed):
if on_push_frame_deprecated:
await observer.on_push_frame(
data.source, data.destination, data.frame, data.direction, data.timestamp

2
src/pipecat/processors/aggregators/dtmf_aggregator.py
View File

@@ -104,7 +104,7 @@ class DTMFAggregator(FrameProcessor):
# For first digit, schedule interruption.
if is_first_digit:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
# Check for immediate flush conditions
if frame.button == self._termination_digit:

232
src/pipecat/processors/aggregators/llm_context_summarizer.py
View File

@@ -6,8 +6,10 @@
"""This module defines a summarizer for managing LLM context summarization."""
import asyncio
import uuid
from typing import Optional
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from loguru import logger
@@ -17,28 +19,68 @@ from pipecat.frames.frames import (
LLMContextSummaryRequestFrame,
LLMContextSummaryResultFrame,
LLMFullResponseStartFrame,
LLMSummarizeContextFrame,
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject
from pipecat.utils.context.llm_context_summarization import (
LLMContextSummarizationConfig,
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMAutoContextSummarizationConfig,
LLMContextSummarizationUtil,
LLMContextSummaryConfig,
)
if TYPE_CHECKING:
from pipecat.services.llm_service import LLMService
@dataclass
class SummaryAppliedEvent:
"""Event data emitted when context summarization completes successfully.
Parameters:
original_message_count: Number of messages before summarization.
new_message_count: Number of messages after summarization.
summarized_message_count: Number of messages that were compressed
into the summary.
preserved_message_count: Number of recent messages preserved
uncompressed.
"""
original_message_count: int
new_message_count: int
summarized_message_count: int
preserved_message_count: int
class LLMContextSummarizer(BaseObject):
"""Summarizer for managing LLM context summarization.
This class manages automatic context summarization when token or message
limits are reached. It monitors the LLM context size, triggers
summarization requests, and applies the results to compress conversation history.
This class manages context summarization, either automatically when token or
message limits are reached, or on-demand when an ``LLMSummarizeContextFrame``
is received. It monitors the LLM context size, triggers summarization requests,
and applies the results to compress conversation history.
When ``auto_trigger=True`` (the default), summarization is triggered
automatically based on the configured thresholds in
``LLMAutoContextSummarizationConfig``. When ``auto_trigger=False``,
threshold checks are skipped and summarization only happens when an
``LLMSummarizeContextFrame`` is explicitly pushed into the pipeline.
Both modes can coexist: set ``auto_trigger=True`` and also push
``LLMSummarizeContextFrame`` at any time to force an immediate summarization
(subject to the ``_summarization_in_progress`` guard).
Event handlers available:
- on_request_summarization: Emitted when summarization should be triggered.
The aggregator should broadcast this frame to the LLM service.
- on_summary_applied: Emitted after a summary has been successfully applied
to the context. Receives a SummaryAppliedEvent with metrics about the
compression.
Example::
@summarizer.event_handler("on_request_summarization")
@@ -49,24 +91,36 @@ class LLMContextSummarizer(BaseObject):
context=frame.context,
...
)
@summarizer.event_handler("on_summary_applied")
async def on_summary_applied(summarizer, event: SummaryAppliedEvent):
logger.info(f"Compressed {event.original_message_count} -> {event.new_message_count} messages")
"""
def __init__(
self,
*,
context: LLMContext,
config: Optional[LLMContextSummarizationConfig] = None,
config: Optional[LLMAutoContextSummarizationConfig] = None,
auto_trigger: bool = True,
):
"""Initialize the context summarizer.
Args:
context: The LLM context to monitor and summarize.
config: Configuration for summarization behavior. If None, uses default config.
config: Auto-summarization configuration controlling both trigger
thresholds and default summary generation parameters. If None,
uses default ``LLMAutoContextSummarizationConfig`` values.
auto_trigger: Whether to automatically trigger summarization when
thresholds are reached. When False, summarization only happens
when an ``LLMSummarizeContextFrame`` is pushed into the pipeline.
Defaults to True.
"""
super().__init__()
self._context = context
self._config = config or LLMContextSummarizationConfig()
self._auto_config = config or LLMAutoContextSummarizationConfig()
self._auto_trigger = auto_trigger
self._task_manager: Optional[BaseTaskManager] = None
@@ -74,6 +128,7 @@ class LLMContextSummarizer(BaseObject):
self._pending_summary_request_id: Optional[str] = None
self._register_event_handler("on_request_summarization", sync=True)
self._register_event_handler("on_summary_applied")
@property
def task_manager(self) -> BaseTaskManager:
@@ -103,6 +158,8 @@ class LLMContextSummarizer(BaseObject):
"""
if isinstance(frame, LLMFullResponseStartFrame):
await self._handle_llm_response_start(frame)
elif isinstance(frame, LLMSummarizeContextFrame):
await self._handle_manual_summarization_request(frame)
elif isinstance(frame, LLMContextSummaryResultFrame):
await self._handle_summary_result(frame)
elif isinstance(frame, InterruptionFrame):
@@ -117,12 +174,24 @@ class LLMContextSummarizer(BaseObject):
if self._should_summarize():
await self._request_summarization()
async def _handle_interruption(self):
"""Handle interruption by canceling summarization in progress.
async def _handle_manual_summarization_request(self, frame: LLMSummarizeContextFrame):
"""Handle an explicit on-demand summarization request.
Reuses the same ``_request_summarization()`` code path as auto mode,
so bookkeeping (``_summarization_in_progress``,
``_pending_summary_request_id``) is always updated correctly.
Args:
frame: The interruption frame.
frame: The manual summarization request frame, optionally carrying
a per-request :class:`~pipecat.utils.context.llm_context_summarization.LLMContextSummaryConfig`.
"""
if self._summarization_in_progress:
logger.debug(f"{self}: Summarization already in progress, ignoring manual request")
return
await self._request_summarization(config_override=frame.config)
async def _handle_interruption(self):
"""Handle interruption by canceling summarization in progress."""
# Reset summarization state to allow new requests. This is necessary because
# the request frame (LLMContextSummaryRequestFrame) may have been cancelled
# during interruption. We preserve _pending_summary_request_id to handle the
@@ -145,13 +214,17 @@ class LLMContextSummarizer(BaseObject):
Returns:
True if all conditions are met:
- ``auto_trigger`` is enabled
- No summarization currently in progress
- AND either:
- Token count exceeds max_context_tokens
- OR message count exceeds max_unsummarized_messages since last summary
- Token count exceeds ``max_context_tokens``
- OR message count exceeds ``max_unsummarized_messages`` since last summary
"""
logger.trace(f"{self}: Checking if context summarization is needed")
if not self._auto_trigger:
return False
if self._summarization_in_progress:
logger.debug(f"{self}: Summarization already in progress")
return False
@@ -161,20 +234,20 @@ class LLMContextSummarizer(BaseObject):
num_messages = len(self._context.messages)
# Check if we've reached the token limit
token_limit = self._config.max_context_tokens
token_limit = self._auto_config.max_context_tokens
token_limit_exceeded = total_tokens >= token_limit
# Check if we've exceeded max unsummarized messages
messages_since_summary = len(self._context.messages) - 1
message_threshold_exceeded = (
messages_since_summary >= self._config.max_unsummarized_messages
messages_since_summary >= self._auto_config.max_unsummarized_messages
)
logger.trace(
f"{self}: Context has {num_messages} messages, "
f"~{total_tokens} tokens (limit: {token_limit}), "
f"{messages_since_summary} messages since last summary "
f"(message threshold: {self._config.max_unsummarized_messages})"
f"(message threshold: {self._auto_config.max_unsummarized_messages})"
)
# Trigger if either limit is exceeded
@@ -189,21 +262,30 @@ class LLMContextSummarizer(BaseObject):
reason.append(f"~{total_tokens} tokens (>={token_limit} limit)")
if message_threshold_exceeded:
reason.append(
f"{messages_since_summary} messages (>={self._config.max_unsummarized_messages} threshold)"
f"{messages_since_summary} messages (>={self._auto_config.max_unsummarized_messages} threshold)"
)
logger.debug(f"{self}: ✓ Summarization needed - {', '.join(reason)}")
return True
async def _request_summarization(self):
async def _request_summarization(
self, config_override: Optional[LLMContextSummaryConfig] = None
):
"""Request context summarization from LLM service.
Creates a summarization request frame and emits it via event handler.
Creates a summarization request frame and either handles it directly
using a dedicated LLM (if configured) or emits it via event handler
for the pipeline's primary LLM.
Tracks the request ID to match async responses and prevent race conditions.
Args:
config_override: Optional per-request summary configuration. If provided,
overrides the default summary generation settings from
``self._auto_config.summary_config``.
"""
# Generate unique request ID
request_id = str(uuid.uuid4())
min_keep = self._config.min_messages_after_summary
summary_config = config_override or self._auto_config.summary_config
# Mark summarization in progress
self._summarization_in_progress = True
@@ -215,13 +297,66 @@ class LLMContextSummarizer(BaseObject):
request_frame = LLMContextSummaryRequestFrame(
request_id=request_id,
context=self._context,
min_messages_to_keep=min_keep,
target_context_tokens=self._config.target_context_tokens,
summarization_prompt=self._config.summary_prompt,
min_messages_to_keep=summary_config.min_messages_after_summary,
target_context_tokens=summary_config.target_context_tokens,
summarization_prompt=summary_config.summary_prompt,
summarization_timeout=summary_config.summarization_timeout,
)
# Emit event for aggregator to broadcast
await self._call_event_handler("on_request_summarization", request_frame)
if summary_config.llm:
# Use dedicated LLM directly — no need to involve the pipeline
self.task_manager.create_task(
self._generate_summary_with_dedicated_llm(summary_config.llm, request_frame),
f"{self}-dedicated-llm-summary",
)
else:
# Emit event for aggregator to broadcast to the pipeline LLM
await self._call_event_handler("on_request_summarization", request_frame)
async def _generate_summary_with_dedicated_llm(
self, llm: "LLMService", frame: LLMContextSummaryRequestFrame
):
"""Generate summary using a dedicated LLM service.
Calls the dedicated LLM's _generate_summary directly and feeds the
result back through _handle_summary_result, bypassing the pipeline.
Args:
llm: The dedicated LLM service to use for summarization.
frame: The summarization request frame.
"""
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try:
summary, last_index = await asyncio.wait_for(
llm._generate_summary(frame),
timeout=timeout,
)
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary=summary,
last_summarized_index=last_index,
)
except asyncio.TimeoutError:
error = f"Context summarization timed out after {timeout}s"
logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary="",
last_summarized_index=-1,
error=error,
)
except Exception as e:
error = f"Error generating context summary: {e}"
logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary="",
last_summarized_index=-1,
error=error,
)
await self._handle_summary_result(result_frame)
async def _handle_summary_result(self, frame: LLMContextSummaryResultFrame):
"""Handle context summarization result from LLM service.
@@ -234,7 +369,9 @@ class LLMContextSummarizer(BaseObject):
"""
logger.debug(f"{self}: Received summary result (request_id={frame.request_id})")
# Check if this is the result we're waiting for
# Check if this is the result we're waiting for. Both auto and manual
# summarization set _pending_summary_request_id via _request_summarization(),
# so this check always applies.
if frame.request_id != self._pending_summary_request_id:
logger.debug(f"{self}: Ignoring stale summary result (request_id={frame.request_id})")
return
@@ -271,7 +408,7 @@ class LLMContextSummarizer(BaseObject):
if last_summarized_index >= len(self._context.messages):
return False
min_keep = self._config.min_messages_after_summary
min_keep = self._auto_config.summary_config.min_messages_after_summary
remaining = len(self._context.messages) - 1 - last_summarized_index
if remaining < min_keep:
return False
@@ -288,16 +425,29 @@ class LLMContextSummarizer(BaseObject):
summary: The generated summary text.
last_summarized_index: Index of the last message that was summarized.
"""
config = self._auto_config.summary_config
messages = self._context.messages
# Find the first system message to preserve
first_system_msg = next((msg for msg in messages if msg.get("role") == "system"), None)
# Find the first system message to preserve. LLMSpecificMessage instances are excluded
# because they are not dict-like and never represent a system message; they hold
# service-specific metadata (e.g. thinking blocks) that is always paired with a
# standard message.
first_system_msg = next(
(
msg
for msg in messages
if not isinstance(msg, LLMSpecificMessage) and msg.get("role") == "system"
),
None,
)
# Get recent messages to keep
recent_messages = messages[last_summarized_index + 1 :]
# Create summary message as an assistant message
summary_message = {"role": "assistant", "content": f"Conversation summary: {summary}"}
# Create summary message as a user message (the summary is context
# provided *to* the assistant, not something the assistant said)
summary_content = config.summary_message_template.format(summary=summary)
summary_message = {"role": "user", "content": summary_content}
# Reconstruct context
new_messages = []
@@ -307,9 +457,23 @@ class LLMContextSummarizer(BaseObject):
new_messages.extend(recent_messages)
# Update context
original_message_count = len(messages)
num_system_preserved = 1 if first_system_msg else 0
self._context.set_messages(new_messages)
# Messages actually summarized = index range minus the preserved system message
summarized_count = last_summarized_index + 1 - num_system_preserved
logger.info(
f"{self}: Applied context summary, compressed {last_summarized_index + 1} messages "
f"into summary. Context now has {len(new_messages)} messages (was {len(messages)})"
f"{self}: Applied context summary, compressed {summarized_count} messages "
f"into summary. Context now has {len(new_messages)} messages (was {original_message_count})"
)
# Emit event for observability
event = SummaryAppliedEvent(
original_message_count=original_message_count,
new_message_count=len(new_messages),
summarized_message_count=summarized_count,
preserved_message_count=len(recent_messages) + num_system_preserved,
)
await self._call_event_handler("on_summary_applied", event)

2
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -581,7 +581,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
logger.debug(
"Interruption conditions met - pushing interruption and aggregation"
)
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
await self._process_aggregation()
else:
logger.debug("Interruption conditions not met - not pushing aggregation")

95
src/pipecat/processors/aggregators/llm_response_universal.py
View File

@@ -35,6 +35,7 @@ from pipecat.frames.frames import (
InputAudioRawFrame,
InterimTranscriptionFrame,
InterruptionFrame,
LLMAssistantPushAggregationFrame,
LLMContextAssistantTimestampFrame,
LLMContextFrame,
LLMContextSummaryRequestFrame,
@@ -78,7 +79,10 @@ from pipecat.turns.user_stop import BaseUserTurnStopStrategy, UserTurnStoppedPar
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionConfig
from pipecat.turns.user_turn_controller import UserTurnController
from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies, UserTurnStrategies
from pipecat.utils.context.llm_context_summarization import LLMContextSummarizationConfig
from pipecat.utils.context.llm_context_summarization import (
LLMAutoContextSummarizationConfig,
LLMContextSummarizationConfig,
)
from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text
from pipecat.utils.time import time_now_iso8601
@@ -124,18 +128,54 @@ class LLMAssistantAggregatorParams:
in text frames by adding spaces between tokens. This parameter is
ignored when used with the newer LLMAssistantAggregator, which
handles word spacing automatically.
enable_context_summarization: Enable automatic context summarization when token
limits are reached (disabled by default). When enabled, older conversation
messages are automatically compressed into summaries to manage context size.
context_summarization_config: Configuration for context summarization behavior.
Controls thresholds, message preservation, and summarization prompts. If None
and summarization is enabled, uses default configuration values.
enable_auto_context_summarization: Enable automatic context summarization when token
or message-count limits are reached (disabled by default). When enabled,
older conversation messages are automatically compressed into summaries to
manage context size.
auto_context_summarization_config: Configuration for automatic context
summarization. Controls trigger thresholds, message preservation, and
summarization prompts. If None, uses default
``LLMAutoContextSummarizationConfig`` values.
"""
expect_stripped_words: bool = True
enable_context_summarization: bool = False
enable_auto_context_summarization: bool = False
auto_context_summarization_config: Optional[LLMAutoContextSummarizationConfig] = None
# ---------------------------------------------------------------------------
# Deprecated field names — kept for backward compatibility.
# Use enable_auto_context_summarization and auto_context_summarization_config instead.
# ---------------------------------------------------------------------------
enable_context_summarization: Optional[bool] = None
context_summarization_config: Optional[LLMContextSummarizationConfig] = None
def __post_init__(self):
if self.enable_context_summarization is not None:
warnings.warn(
"LLMAssistantAggregatorParams.enable_context_summarization is deprecated. "
"Use enable_auto_context_summarization instead.",
DeprecationWarning,
stacklevel=2,
)
self.enable_auto_context_summarization = self.enable_context_summarization
self.enable_context_summarization = None
if self.context_summarization_config is not None:
warnings.warn(
"LLMAssistantAggregatorParams.context_summarization_config is deprecated. "
"Use auto_context_summarization_config (LLMAutoContextSummarizationConfig) instead.",
DeprecationWarning,
stacklevel=2,
)
if isinstance(self.context_summarization_config, LLMContextSummarizationConfig):
self.auto_context_summarization_config = (
self.context_summarization_config.to_auto_config()
)
else:
# Accept LLMAutoContextSummarizationConfig passed to the deprecated field
self.auto_context_summarization_config = self.context_summarization_config # type: ignore[assignment]
self.context_summarization_config = None
@dataclass
class UserTurnStoppedMessage:
@@ -461,6 +501,10 @@ class LLMUserAggregator(LLMContextAggregator):
await self.push_frame(frame, direction)
elif isinstance(frame, TranscriptionFrame):
await self._handle_transcription(frame)
elif isinstance(frame, (InterimTranscriptionFrame, TranslationFrame)):
# Interim transcriptions and translations are consumed here
# and not pushed downstream, same as final TranscriptionFrame.
pass
elif isinstance(frame, LLMRunFrame):
await self._handle_llm_run(frame)
elif isinstance(frame, LLMMessagesAppendFrame):
@@ -564,12 +608,6 @@ class LLMUserAggregator(LLMContextAggregator):
if should_mute_frame:
logger.trace(f"{frame.name} suppressed - user currently muted")
# When muted, the InterruptionFrame won't propagate further and
# will never reach the pipeline sink. Complete it here so
# push_interruption_task_frame_and_wait() doesn't hang.
if should_mute_frame and isinstance(frame, InterruptionFrame):
frame.complete()
should_mute_next_time = False
for s in self._params.user_mute_strategies:
should_mute_next_time |= await s.process_frame(frame)
@@ -598,6 +636,9 @@ class LLMUserAggregator(LLMContextAggregator):
async def _handle_llm_messages_update(self, frame: LLMMessagesUpdateFrame):
self.set_messages(frame.messages)
if self._params.filter_incomplete_user_turns:
config = self._params.user_turn_completion_config or UserTurnCompletionConfig()
self._context.add_message({"role": "system", "content": config.completion_instructions})
if frame.run_llm:
await self.push_context_frame()
@@ -690,7 +731,7 @@ class LLMUserAggregator(LLMContextAggregator):
await self._user_idle_controller.process_frame(UserStartedSpeakingFrame())
if params.enable_interruptions and self._allow_interruptions:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
await self._call_event_handler("on_user_turn_started", strategy)
@@ -820,16 +861,18 @@ class LLMAssistantAggregator(LLMContextAggregator):
self._thought_aggregation: List[TextPartForConcatenation] = []
self._thought_start_time: str = ""
# Context summarization
self._summarizer: Optional[LLMContextSummarizer] = None
if self._params.enable_context_summarization:
self._summarizer = LLMContextSummarizer(
context=self._context,
config=self._params.context_summarization_config,
)
self._summarizer.add_event_handler(
"on_request_summarization", self._on_request_summarization
)
# Context summarization — always create the summarizer so that manually
# pushed LLMSummarizeContextFrame frames are always handled.
# Auto-triggering based on thresholds is only enabled when
# enable_auto_context_summarization is True.
self._summarizer: Optional[LLMContextSummarizer] = LLMContextSummarizer(
context=self._context,
config=self._params.auto_context_summarization_config,
auto_trigger=self._params.enable_auto_context_summarization,
)
self._summarizer.add_event_handler(
"on_request_summarization", self._on_request_summarization
)
self._register_event_handler("on_assistant_turn_started")
self._register_event_handler("on_assistant_turn_stopped")
@@ -875,6 +918,8 @@ class LLMAssistantAggregator(LLMContextAggregator):
elif isinstance(frame, (EndFrame, CancelFrame)):
await self._handle_end_or_cancel(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, LLMAssistantPushAggregationFrame):
await self.push_aggregation()
elif isinstance(frame, LLMFullResponseStartFrame):
await self._handle_llm_start(frame)
elif isinstance(frame, LLMFullResponseEndFrame):

6
src/pipecat/processors/filters/stt_mute_filter.py
View File

@@ -234,12 +234,6 @@ class STTMuteFilter(FrameProcessor):
await self.push_frame(frame, direction)
else:
logger.trace(f"{frame.__class__.__name__} suppressed - STT currently muted")
# When muted, the InterruptionFrame won't propagate further
# and will never reach the pipeline sink. Complete it here so
# push_interruption_task_frame_and_wait() doesn't hang.
if isinstance(frame, InterruptionFrame):
frame.complete()
else:
# Pass all other frames through
await self.push_frame(frame, direction)

92
src/pipecat/processors/frame_processor.py
View File

@@ -41,7 +41,6 @@ from pipecat.frames.frames import (
FrameProcessorResumeFrame,
FrameProcessorResumeUrgentFrame,
InterruptionFrame,
InterruptionTaskFrame,
StartFrame,
SystemFrame,
UninterruptibleFrame,
@@ -240,10 +239,6 @@ class FrameProcessor(BaseObject):
self.__process_frame_task: Optional[asyncio.Task] = None
self.__process_current_frame: Optional[Frame] = None
# Set while awaiting push_interruption_task_frame_and_wait() so that
# _start_interruption() knows not to cancel the process task.
self._wait_for_interruption = False
# Frame processor events.
self._register_event_handler("on_before_process_frame", sync=True)
self._register_event_handler("on_after_process_frame", sync=True)
@@ -329,7 +324,7 @@ class FrameProcessor(BaseObject):
warnings.simplefilter("always")
warnings.warn(
"`FrameProcessor.interruptions_allowed` is deprecated. "
"Use `LLMUserAggregator`'s new `user_mute_strategies` parameter instead.",
"Use `LLMUserAggregator`'s new `user_mute_strategies` parameter instead.",
DeprecationWarning,
stacklevel=2,
)
@@ -485,10 +480,23 @@ class FrameProcessor(BaseObject):
if frame:
await self.push_frame(frame)
async def start_text_aggregation_metrics(self):
"""Start text aggregation time metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_text_aggregation_metrics()
async def stop_text_aggregation_metrics(self):
"""Stop text aggregation time metrics collection and push results."""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_text_aggregation_metrics()
if frame:
await self.push_frame(frame)
async def stop_all_metrics(self):
"""Stop all active metrics collection."""
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
await self.stop_text_aggregation_metrics()
def create_task(self, coroutine: Coroutine, name: Optional[str] = None) -> asyncio.Task:
"""Create a new task managed by this processor.
@@ -618,15 +626,6 @@ class FrameProcessor(BaseObject):
if self._cancelling:
return
# If we are waiting for an interruption, bypass all queued system frames
# and process the frame right away. This is because a previous system
# frame might be waiting for the interruption frame blocking the input
# task, so this InterruptionFrame would never be dequeued and we'd
# deadlock.
if self._wait_for_interruption and isinstance(frame, InterruptionFrame):
await self.__process_frame(frame, direction, callback)
return
if self._enable_direct_mode:
await self.__process_frame(frame, direction, callback)
else:
@@ -761,43 +760,32 @@ class FrameProcessor(BaseObject):
await self._call_event_handler("on_after_push_frame", frame)
async def broadcast_interruption(self):
"""Broadcast an `InterruptionFrame` both upstream and downstream."""
logger.debug(f"{self}: broadcasting interruption")
self.__reset_process_task()
await self.stop_all_metrics()
await self.broadcast_frame(InterruptionFrame)
async def push_interruption_task_frame_and_wait(self, *, timeout: float = 5.0):
"""Push an interruption task frame upstream and wait for the interruption.
This function sends an `InterruptionTaskFrame` upstream to the
pipeline task. The task creates a corresponding `InterruptionFrame`
and sends it downstream through the pipeline. An `asyncio.Event` is
attached to both frames so the caller can wait until the interruption
has fully traversed the pipeline. The event is set when the
`InterruptionFrame` reaches the pipeline sink. If the frame does
not complete within the given timeout, a warning is logged and the
event is forcibly set so the caller is unblocked.
Args:
timeout: Maximum seconds to wait for the interruption to complete.
.. deprecated:: 0.0.104
Use :meth:`broadcast_interruption` instead. This method now
delegates to ``broadcast_interruption()`` and ignores *timeout*.
"""
self._wait_for_interruption = True
import warnings
event = asyncio.Event()
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"`FrameProcessor.push_interruption_task_frame_and_wait()` is deprecated. "
"Use `FrameProcessor.broadcast_interruption()` instead.",
DeprecationWarning,
stacklevel=2,
)
await self.push_frame(InterruptionTaskFrame(event=event), FrameDirection.UPSTREAM)
# Wait for the `InterruptionFrame` to complete and log a warning if it
# takes too long. If it does take too long make sure we unblock it,
# otherwise we will hang here forever.
while not event.is_set():
try:
await asyncio.wait_for(event.wait(), timeout=timeout)
except asyncio.TimeoutError:
logger.warning(
f"{self}: InterruptionFrame has not completed after"
f" {timeout}s. Make sure InterruptionFrame.complete()"
" is being called (e.g. if the frame is being blocked"
" or consumed before reaching the pipeline sink)."
)
event.set()
self._wait_for_interruption = False
await self.broadcast_interruption()
async def broadcast_frame(self, frame_cls: Type[Frame], **kwargs):
"""Broadcasts a frame of the specified class upstream and downstream.
@@ -904,15 +892,7 @@ class FrameProcessor(BaseObject):
async def _start_interruption(self):
"""Start handling an interruption by cancelling current tasks."""
try:
if self._wait_for_interruption:
# If we get here we know the process task was just waiting for
# an interruption (push_interruption_task_frame_and_wait()), so
# we can't cancel the task because it might still need to do
# more things (e.g. pushing a frame after the
# interruption). Instead we just drain the queue because this is
# an interruption.
self.__reset_process_task()
elif isinstance(self.__process_current_frame, UninterruptibleFrame):
if isinstance(self.__process_current_frame, UninterruptibleFrame):
# We don't want to cancel UninterruptibleFrame, so we simply
# cleanup the queue.
self.__reset_process_queue()
@@ -936,7 +916,7 @@ class FrameProcessor(BaseObject):
try:
timestamp = self._clock.get_time() if self._clock else 0
if direction == FrameDirection.DOWNSTREAM and self._next:
logger.trace(f"Pushing {frame} from {self} to {self._next}")
logger.trace(f"Pushing {frame} downstream from {self} to {self._next}")
if self._observer:
data = FramePushed(

2
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -1702,7 +1702,7 @@ class RTVIProcessor(FrameProcessor):
async def interrupt_bot(self):
"""Send a bot interruption frame upstream."""
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
async def send_server_message(self, data: Any):
"""Send a server message to the client."""

23
src/pipecat/processors/metrics/frame_processor_metrics.py
View File

@@ -17,6 +17,7 @@ from pipecat.metrics.metrics import (
LLMUsageMetricsData,
MetricsData,
ProcessingMetricsData,
TextAggregationMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
)
@@ -43,6 +44,7 @@ class FrameProcessorMetrics(BaseObject):
self._task_manager = None
self._start_ttfb_time = 0
self._start_processing_time = 0
self._start_text_aggregation_time = 0
self._last_ttfb_time = 0
self._should_report_ttfb = True
@@ -211,3 +213,24 @@ class FrameProcessorMetrics(BaseObject):
)
logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
return MetricsFrame(data=[characters])
async def start_text_aggregation_metrics(self):
"""Start measuring text aggregation time (first token to first sentence)."""
self._start_text_aggregation_time = time.time()
async def stop_text_aggregation_metrics(self):
"""Stop text aggregation measurement and generate metrics frame.
Returns:
MetricsFrame containing text aggregation time, or None if not measuring.
"""
if self._start_text_aggregation_time == 0:
return None
value = time.time() - self._start_text_aggregation_time
logger.debug(f"{self._processor_name()} text aggregation time: {value}")
aggregation = TextAggregationMetricsData(
processor=self._processor_name(), value=value, model=self._model_name()
)
self._start_text_aggregation_time = 0
return MetricsFrame(data=[aggregation])

31
src/pipecat/processors/metrics/sentry.py
View File

@@ -7,6 +7,7 @@
"""Sentry integration for frame processor metrics."""
import asyncio
from typing import Optional
from loguru import logger
@@ -70,13 +71,18 @@ class SentryMetrics(FrameProcessorMetrics):
logger.trace(f"{self} Flushing Sentry metrics")
sentry_sdk.flush(timeout=5.0)
async def start_ttfb_metrics(self, report_only_initial_ttfb):
async def start_ttfb_metrics(
self, *, start_time: Optional[float] = None, report_only_initial_ttfb: bool
):
"""Start tracking time-to-first-byte metrics.
Args:
start_time: Optional start timestamp override.
report_only_initial_ttfb: Whether to report only the initial TTFB measurement.
"""
await super().start_ttfb_metrics(report_only_initial_ttfb)
await super().start_ttfb_metrics(
start_time=start_time, report_only_initial_ttfb=report_only_initial_ttfb
)
if self._should_report_ttfb and self._sentry_available:
self._ttfb_metrics_tx = sentry_sdk.start_transaction(
@@ -87,23 +93,25 @@ class SentryMetrics(FrameProcessorMetrics):
f"{self} Sentry transaction started (ID: {self._ttfb_metrics_tx.span_id} Name: {self._ttfb_metrics_tx.name})"
)
async def stop_ttfb_metrics(self):
async def stop_ttfb_metrics(self, *, end_time: Optional[float] = None):
"""Stop tracking time-to-first-byte metrics.
Queues the TTFB transaction for completion and transmission to Sentry.
Args:
end_time: Optional end timestamp override.
"""
await super().stop_ttfb_metrics()
await super().stop_ttfb_metrics(end_time=end_time)
if self._sentry_available and self._ttfb_metrics_tx:
await self._sentry_queue.put(self._ttfb_metrics_tx)
self._ttfb_metrics_tx = None
async def start_processing_metrics(self):
async def start_processing_metrics(self, *, start_time: Optional[float] = None):
"""Start tracking frame processing metrics.
Creates a new Sentry transaction to track processing performance.
Args:
start_time: Optional start timestamp override.
"""
await super().start_processing_metrics()
await super().start_processing_metrics(start_time=start_time)
if self._sentry_available:
self._processing_metrics_tx = sentry_sdk.start_transaction(
@@ -114,12 +122,13 @@ class SentryMetrics(FrameProcessorMetrics):
f"{self} Sentry transaction started (ID: {self._processing_metrics_tx.span_id} Name: {self._processing_metrics_tx.name})"
)
async def stop_processing_metrics(self):
async def stop_processing_metrics(self, *, end_time: Optional[float] = None):
"""Stop tracking frame processing metrics.
Queues the processing transaction for completion and transmission to Sentry.
Args:
end_time: Optional end timestamp override.
"""
await super().stop_processing_metrics()
await super().stop_processing_metrics(end_time=end_time)
if self._sentry_available and self._processing_metrics_tx:
await self._sentry_queue.put(self._processing_metrics_tx)

1
src/pipecat/serializers/genesys.py
View File

@@ -642,7 +642,6 @@ class GenesysAudioHookSerializer(FrameSerializer):
"""
# Binary data = audio
if isinstance(data, bytes):
logger.debug(f"[AUDIO IN] Received {len(data)} bytes from Genesys")
return await self._deserialize_audio(data)
# Text data = JSON control message

101
src/pipecat/services/ai_service.py
View File

@@ -10,7 +10,7 @@ Provides the foundation for all AI services in the Pipecat framework, including
model management, settings handling, and frame processing lifecycle methods.
"""
from typing import Any, AsyncGenerator, Dict, Mapping
from typing import Any, AsyncGenerator, Dict
from loguru import logger
@@ -23,6 +23,7 @@ from pipecat.frames.frames import (
)
from pipecat.metrics.metrics import MetricsData
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.settings import ServiceSettings
class AIService(FrameProcessor):
@@ -34,36 +35,38 @@ class AIService(FrameProcessor):
this base infrastructure.
"""
def __init__(self, **kwargs):
def __init__(self, settings: ServiceSettings | None = None, **kwargs):
"""Initialize the AI service.
Args:
settings: The runtime-updatable settings for the AI service.
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
self._model_name: str = ""
self._settings: Dict[str, Any] = {}
self._settings: ServiceSettings = (
settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ServiceSettings()
)
self._sync_model_name_to_metrics()
self._session_properties: Dict[str, Any] = {}
self._tracing_enabled: bool = False
self._tracing_context = None
@property
def model_name(self) -> str:
"""Get the current model name.
def _sync_model_name_to_metrics(self):
"""Sync the current AI model name (in `self._settings.model`) for usage in metrics.
Returns:
The name of the AI model being used.
"""
return self._model_name
def set_model_name(self, model: str):
"""Set the AI model name and update metrics.
We don't store model name here because there's already a single source
of truth for it in `self._settings.model`. This method is just for
syncing the model name to the metrics data.
Args:
model: The name of the AI model to use.
"""
self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
self.set_core_metrics_data(
MetricsData(processor=self.name, model=self._settings.model or "")
)
async def start(self, frame: StartFrame):
"""Start the AI service.
@@ -74,6 +77,7 @@ class AIService(FrameProcessor):
Args:
frame: The start frame containing initialization parameters.
"""
self._settings.validate_complete()
self._tracing_enabled = frame.enable_tracing
self._tracing_context = frame.tracing_context
@@ -99,44 +103,45 @@ class AIService(FrameProcessor):
"""
pass
async def _update_settings(self, settings: Mapping[str, Any]):
from pipecat.services.openai.realtime.events import SessionProperties
async def _update_settings(self, delta: ServiceSettings) -> Dict[str, Any]:
"""Apply a settings delta and return the changed fields.
for key, value in settings.items():
logger.debug("Update request for:", key, value)
The delta is applied to ``_settings`` and a dict mapping each changed
field name to its **pre-update** value is returned. The ``model``
field is handled specially: when it changes, ``set_model_name`` is
called.
if key in self._settings:
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._settings[key] = value
elif key in SessionProperties.model_fields:
logger.debug("Attempting to update", key, value)
Concrete services should override this method (calling ``super()``)
to react to specific changed fields (e.g. reconnect on voice change).
try:
from pipecat.services.openai.realtime.events import TurnDetection
Args:
delta: A delta-mode settings object.
if isinstance(self._session_properties, SessionProperties):
current_properties = self._session_properties
else:
current_properties = SessionProperties(**self._session_properties)
Returns:
Dict mapping changed field names to their previous values.
"""
changed = self._settings.apply_update(delta)
if key == "turn_detection" and isinstance(value, dict):
turn_detection = TurnDetection(**value)
setattr(current_properties, key, turn_detection)
else:
setattr(current_properties, key, value)
if "model" in changed:
self._sync_model_name_to_metrics()
validated_properties = SessionProperties.model_validate(
current_properties.model_dump()
)
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._session_properties = validated_properties.model_dump()
except Exception as e:
logger.warning(f"Unexpected error updating session property {key}: {e}")
elif key == "model":
logger.info(f"Updating LLM setting {key} to: [{value}]")
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for {self.name} service: {key}")
if changed:
logger.info(f"{self.name}: updated settings fields: {set(changed)}")
return changed
def _warn_unhandled_updated_settings(self, unhandled):
"""Log a warning for settings changes that won't take effect at runtime.
Convenience helper for ``_update_settings`` overrides. Accepts any
iterable of field names (a ``dict``, ``set``, ``dict_keys``, etc.).
Args:
unhandled: Field names that changed but are not applied.
"""
if unhandled:
fields = ", ".join(sorted(unhandled))
logger.warning(f"{self.name}: runtime update of [{fields}] is not currently supported")
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and handle service lifecycle.

168
src/pipecat/services/anthropic/llm.py
View File

@@ -16,8 +16,8 @@ import copy
import io
import json
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, Union
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Literal, Optional, Union
import httpx
from loguru import logger
@@ -42,7 +42,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
UserImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -59,6 +58,8 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
from pipecat.services.settings import LLMSettings, _NotGiven, is_given
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -69,6 +70,50 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
class AnthropicThinkingConfig(BaseModel):
"""Configuration for extended thinking.
Parameters:
type: Type of thinking mode (currently only "enabled" or "disabled").
budget_tokens: Maximum number of tokens for thinking.
With today's models, the minimum is 1024.
Only allowed if type is "enabled".
"""
# Why `| str` here? To not break compatibility in case Anthropic adds
# more types in the future.
type: Literal["enabled", "disabled"] | str
# Why not enforce minimnum of 1024 here? To not break compatibility in
# case Anthropic changes this requirement in the future.
budget_tokens: int
@dataclass
class AnthropicLLMSettings(LLMSettings):
"""Settings for Anthropic LLM services.
Parameters:
enable_prompt_caching: Whether to enable prompt caching.
thinking: Extended thinking configuration.
"""
enable_prompt_caching: bool | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
thinking: AnthropicThinkingConfig | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
@classmethod
def from_mapping(cls, settings):
"""Convert a plain dict to settings, coercing thinking dicts.
For backward compatibility, a ``thinking`` value that is a plain dict
is converted to a :class:`AnthropicThinkingConfig`.
"""
instance = super().from_mapping(settings)
if is_given(instance.thinking) and isinstance(instance.thinking, dict):
instance.thinking = AnthropicThinkingConfig(**instance.thinking)
return instance
@dataclass
class AnthropicContextAggregatorPair:
"""Pair of context aggregators for Anthropic conversations.
@@ -115,26 +160,13 @@ class AnthropicLLMService(LLMService):
Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex.
"""
_settings: AnthropicLLMSettings
# Overriding the default adapter to use the Anthropic one.
adapter_class = AnthropicLLMAdapter
class ThinkingConfig(BaseModel):
"""Configuration for extended thinking.
Parameters:
type: Type of thinking mode (currently only "enabled" or "disabled").
budget_tokens: Maximum number of tokens for thinking.
With today's models, the minimum is 1024.
Only allowed if type is "enabled".
"""
# Why `| str` here? To not break compatibility in case Anthropic adds
# more types in the future.
type: Literal["enabled", "disabled"] | str
# Why not enforce minimnum of 1024 here? To not break compatibility in
# case Anthropic changes this requirement in the future.
budget_tokens: int
# Backward compatibility: ThinkingConfig used to be defined inline here.
ThinkingConfig = AnthropicThinkingConfig
class InputParams(BaseModel):
"""Input parameters for Anthropic model inference.
@@ -163,9 +195,7 @@ class AnthropicLLMService(LLMService):
temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
top_k: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0)
top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
thinking: Optional["AnthropicLLMService.ThinkingConfig"] = Field(
default_factory=lambda: NOT_GIVEN
)
thinking: Optional[AnthropicThinkingConfig] = Field(default_factory=lambda: NOT_GIVEN)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def model_post_init(self, __context):
@@ -184,7 +214,7 @@ class AnthropicLLMService(LLMService):
self,
*,
api_key: str,
model: str = "claude-sonnet-4-5-20250929",
model: str = "claude-sonnet-4-6",
params: Optional[InputParams] = None,
client=None,
retry_timeout_secs: Optional[float] = 5.0,
@@ -195,38 +225,46 @@ class AnthropicLLMService(LLMService):
Args:
api_key: Anthropic API key for authentication.
model: Model name to use. Defaults to "claude-sonnet-4-5-20250929".
model: Model name to use. Defaults to "claude-sonnet-4-6".
params: Optional model parameters for inference.
client: Optional custom Anthropic client instance.
retry_timeout_secs: Request timeout in seconds for retry logic.
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AnthropicLLMService.InputParams()
super().__init__(
settings=AnthropicLLMSettings(
model=model,
max_tokens=params.max_tokens,
enable_prompt_caching=(
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._client = client or AsyncAnthropic(
api_key=api_key
) # if the client is provided, use it and remove it, otherwise create a new one
self.set_model_name(model)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = {
"max_tokens": params.max_tokens,
"enable_prompt_caching": (
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"thinking": params.thinking,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
def can_generate_metrics(self) -> bool:
"""Check if this service can generate usage metrics.
@@ -280,7 +318,7 @@ class AnthropicLLMService(LLMService):
if isinstance(context, LLMContext):
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
invocation_params = adapter.get_llm_invocation_params(
context, enable_prompt_caching=self._settings["enable_prompt_caching"]
context, enable_prompt_caching=self._settings.enable_prompt_caching
)
messages = invocation_params["messages"]
system = invocation_params["system"]
@@ -293,21 +331,21 @@ class AnthropicLLMService(LLMService):
# Build params using the same method as streaming completions
params = {
"model": self.model_name,
"max_tokens": max_tokens if max_tokens is not None else self._settings["max_tokens"],
"model": self._settings.model,
"max_tokens": max_tokens if max_tokens is not None else self._settings.max_tokens,
"stream": False,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"temperature": self._settings.temperature,
"top_k": self._settings.top_k,
"top_p": self._settings.top_p,
"messages": messages,
"system": system,
"tools": tools,
"betas": ["interleaved-thinking-2025-05-14"],
}
if self._settings["thinking"]:
params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
if self._settings.thinking:
params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
params.update(self._settings["extra"])
params.update(self._settings.extra)
# LLM completion
response = await self._client.beta.messages.create(**params)
@@ -358,14 +396,14 @@ class AnthropicLLMService(LLMService):
if isinstance(context, LLMContext):
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
params = adapter.get_llm_invocation_params(
context, enable_prompt_caching=self._settings["enable_prompt_caching"]
context, enable_prompt_caching=self._settings.enable_prompt_caching
)
return params
# Anthropic-specific context
messages = (
context.get_messages_with_cache_control_markers()
if self._settings["enable_prompt_caching"]
if self._settings.enable_prompt_caching
else context.messages
)
return AnthropicLLMInvocationParams(
@@ -407,22 +445,22 @@ class AnthropicLLMService(LLMService):
await self.start_ttfb_metrics()
params = {
"model": self.model_name,
"max_tokens": self._settings["max_tokens"],
"model": self._settings.model,
"max_tokens": self._settings.max_tokens,
"stream": True,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"temperature": self._settings.temperature,
"top_k": self._settings.top_k,
"top_p": self._settings.top_p,
}
# Add thinking parameter if set
if self._settings["thinking"]:
params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
if self._settings.thinking:
params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
# Messages, system, tools
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
# "Interleaved thinking" needed to allow thinking between sequences
# of function calls, when extended thinking is enabled.
@@ -576,11 +614,9 @@ class AnthropicLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = AnthropicLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMEnablePromptCachingFrame):
logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
self._settings["enable_prompt_caching"] = frame.enable
self._settings.enable_prompt_caching = frame.enable
else:
await self.push_frame(frame, direction)

71
src/pipecat/services/assemblyai/stt.py
View File

@@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription.
import asyncio
import json
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional
from urllib.parse import urlencode
@@ -29,6 +30,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import ASSEMBLYAI_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -52,6 +54,21 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AssemblyAISTTSettings(STTSettings):
"""Settings for the AssemblyAI STT service.
See :class:`AssemblyAIConnectionParams` for detailed parameter descriptions.
Parameters:
connection_params: Connection configuration parameters.
"""
connection_params: AssemblyAIConnectionParams | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class AssemblyAISTTService(WebsocketSTTService):
"""AssemblyAI real-time speech-to-text service.
@@ -60,6 +77,8 @@ class AssemblyAISTTService(WebsocketSTTService):
for audio processing and connection management.
"""
_settings: AssemblyAISTTSettings
def __init__(
self,
*,
@@ -92,13 +111,18 @@ class AssemblyAISTTService(WebsocketSTTService):
connection_params = self._configure_manual_turn_mode(connection_params)
super().__init__(
sample_rate=connection_params.sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs
sample_rate=connection_params.sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AssemblyAISTTSettings(
model=None,
language=language,
connection_params=connection_params,
),
**kwargs,
)
self._api_key = api_key
self._language = language
self._api_endpoint_base_url = api_endpoint_base_url
self._connection_params = connection_params
self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._termination_event = asyncio.Event()
@@ -165,6 +189,37 @@ class AssemblyAISTTService(WebsocketSTTService):
"""
return True
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
Args:
delta: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# # Re-apply manual turn mode config if vad_force_turn_endpoint is active
# # and connection_params were updated.
# if self._vad_force_turn_endpoint and "connection_params" in changed:
# self._settings.connection_params = self._configure_manual_turn_mode(
# self._settings.connection_params
# )
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
@@ -239,7 +294,7 @@ class AssemblyAISTTService(WebsocketSTTService):
def _build_ws_url(self) -> str:
"""Build WebSocket URL with query parameters using urllib.parse.urlencode."""
params = {}
for k, v in self._connection_params.model_dump().items():
for k, v in self._settings.connection_params.model_dump().items():
if v is not None:
if k == "keyterms_prompt":
params[k] = json.dumps(v)
@@ -415,18 +470,18 @@ class AssemblyAISTTService(WebsocketSTTService):
if not message.transcript:
return
if message.end_of_turn and (
not self._connection_params.formatted_finals or message.turn_is_formatted
not self._settings.connection_params.formatted_finals or message.turn_is_formatted
):
await self.push_frame(
TranscriptionFrame(
message.transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
message,
)
)
await self._trace_transcription(message.transcript, True, self._language)
await self._trace_transcription(message.transcript, True, self._settings.language)
await self.stop_processing_metrics()
else:
await self.push_frame(
@@ -434,7 +489,7 @@ class AssemblyAISTTService(WebsocketSTTService):
message.transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
message,
)
)

169
src/pipecat/services/asyncai/tts.py
View File

@@ -9,7 +9,8 @@
import asyncio
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Mapping, Optional
import aiohttp
from loguru import logger
@@ -20,14 +21,14 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -72,12 +73,40 @@ def language_to_async_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class AsyncAITTSSettings(TTSSettings):
"""Settings for Async AI TTS services.
Parameters:
output_container: Audio container format (e.g. "raw").
output_encoding: Audio encoding format (e.g. "pcm_s16le").
output_sample_rate: Audio sample rate in Hz.
"""
output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "AsyncAITTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``output_format``."""
flat = dict(settings)
nested = flat.pop("output_format", None)
if isinstance(nested, dict):
flat.setdefault("output_container", nested.get("container"))
flat.setdefault("output_encoding", nested.get("encoding"))
flat.setdefault("output_sample_rate", nested.get("sample_rate"))
return super().from_mapping(flat)
class AsyncAITTSService(AudioContextTTSService):
"""Async TTS service with WebSocket streaming.
Provides text-to-speech using Async's streaming WebSocket API.
"""
_settings: AsyncAITTSSettings
class InputParams(BaseModel):
"""Input parameters for Async TTS configuration.
@@ -99,7 +128,8 @@ class AsyncAITTSService(AudioContextTTSService):
encoding: str = "pcm_s16le",
container: str = "raw",
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize the Async TTS service.
@@ -115,39 +145,56 @@ class AsyncAITTSService(AudioContextTTSService):
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to the parent service.
"""
params = params or AsyncAITTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs,
)
params = params or AsyncAITTSService.InputParams()
self._api_key = api_key
self._api_version = version
self._url = url
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._receive_task = None
self._keepalive_task = None
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
self._warn_unhandled_updated_settings(changed)
return changed
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -178,7 +225,7 @@ class AsyncAITTSService(AudioContextTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -232,10 +279,14 @@ class AsyncAITTSService(AudioContextTTSService):
f"{self._url}?api_key={self._api_key}&version={self._api_version}"
)
init_msg = {
"model_id": self._model_name,
"voice": {"mode": "id", "id": self._voice_id},
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"model_id": self._settings.model,
"voice": {"mode": "id", "id": self._settings.voice},
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"language": self._settings.language,
}
await self._get_websocket().send(json.dumps(init_msg))
@@ -346,18 +397,29 @@ class AsyncAITTSService(AudioContextTTSService):
logger.warning(f"{self} keepalive error: {e}")
break
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by closing the current context."""
context_id = self.get_active_audio_context_id()
await super()._handle_interruption(frame, direction)
# Close the current context when interrupted without closing the websocket
async def _close_context(self, context_id: str):
# Async AI requires explicit context closure to free server-side resources,
# both on interruption and on normal completion.
if context_id and self._websocket:
try:
await self._websocket.send(
json.dumps({"context_id": context_id, "close_context": True, "transcript": ""})
)
except Exception as e:
logger.error(f"Error closing context on interruption: {e}")
logger.error(f"{self}: Error closing context {context_id}: {e}")
async def on_audio_context_interrupted(self, context_id: str):
"""Close the Async AI context when the bot is interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Close the Async AI context after all audio has been played.
Async AI does not send a server-side signal when a context is
exhausted, so Pipecat must explicitly close it with
``close_context: True`` to free server-side resources.
"""
await self._close_context(context_id)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -404,6 +466,8 @@ class AsyncAIHttpTTSService(TTSService):
connection is not required or desired.
"""
_settings: AsyncAITTSSettings
class InputParams(BaseModel):
"""Input parameters for Async API.
@@ -443,25 +507,26 @@ class AsyncAIHttpTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AsyncAIHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs,
)
self._api_key = api_key
self._base_url = url
self._api_version = version
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
}
self.set_voice(voice_id)
self.set_model_name(model)
self._session = aiohttp_session
@@ -491,7 +556,7 @@ class AsyncAIHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -507,14 +572,18 @@ class AsyncAIHttpTTSService(TTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
voice_config = {"mode": "id", "id": self._voice_id}
voice_config = {"mode": "id", "id": self._settings.voice}
await self.start_ttfb_metrics()
payload = {
"model_id": self._model_name,
"model_id": self._settings.model,
"transcript": text,
"voice": voice_config,
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"language": self._settings.language,
}
yield TTSStartedFrame(context_id=context_id)
headers = {

81
src/pipecat/services/aws/llm.py
View File

@@ -18,8 +18,8 @@ import io
import json
import os
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Optional
from loguru import logger
from PIL import Image
@@ -40,7 +40,6 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
UserImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -57,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -71,6 +71,21 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AWSBedrockLLMSettings(LLMSettings):
"""Settings for AWS Bedrock LLM services.
Parameters:
latency: Performance mode - "standard" or "optimized".
additional_model_request_fields: Additional model-specific parameters.
"""
latency: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
additional_model_request_fields: Dict[str, Any] | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
@dataclass
class AWSBedrockContextAggregatorPair:
"""Container for AWS Bedrock context aggregators.
@@ -730,6 +745,8 @@ class AWSBedrockLLMService(LLMService):
vision capabilities.
"""
_settings: AWSBedrockLLMSettings
# Overriding the default adapter to use the Anthropic one.
adapter_class = AWSBedrockLLMAdapter
@@ -780,10 +797,28 @@ class AWSBedrockLLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AWSBedrockLLMService.InputParams()
super().__init__(
settings=AWSBedrockLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
latency=params.latency,
additional_model_request_fields=params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
),
**kwargs,
)
# Initialize the AWS Bedrock client
if not client_config:
client_config = Config(
@@ -803,18 +838,8 @@ class AWSBedrockLLMService(LLMService):
"config": client_config,
}
self.set_model_name(model)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
"top_p": params.top_p,
"latency": params.latency,
"additional_model_request_fields": params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
}
logger.info(f"Using AWS Bedrock model: {model}")
@@ -836,12 +861,12 @@ class AWSBedrockLLMService(LLMService):
Dictionary containing only the inference parameters that are not None.
"""
inference_config = {}
if self._settings["max_tokens"] is not None:
inference_config["maxTokens"] = self._settings["max_tokens"]
if self._settings["temperature"] is not None:
inference_config["temperature"] = self._settings["temperature"]
if self._settings["top_p"] is not None:
inference_config["topP"] = self._settings["top_p"]
if self._settings.max_tokens is not None:
inference_config["maxTokens"] = self._settings.max_tokens
if self._settings.temperature is not None:
inference_config["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
inference_config["topP"] = self._settings.top_p
return inference_config
async def run_inference(
@@ -877,9 +902,9 @@ class AWSBedrockLLMService(LLMService):
inference_config["maxTokens"] = max_tokens
request_params = {
"modelId": self.model_name,
"modelId": self._settings.model,
"messages": messages,
"additionalModelRequestFields": self._settings["additional_model_request_fields"],
"additionalModelRequestFields": self._settings.additional_model_request_fields,
}
if inference_config:
@@ -1034,9 +1059,9 @@ class AWSBedrockLLMService(LLMService):
# Prepare request parameters
request_params = {
"modelId": self.model_name,
"modelId": self._settings.model,
"messages": messages,
"additionalModelRequestFields": self._settings["additional_model_request_fields"],
"additionalModelRequestFields": self._settings.additional_model_request_fields,
}
# Only add inference config if it has parameters
@@ -1081,8 +1106,8 @@ class AWSBedrockLLMService(LLMService):
request_params["toolConfig"] = tool_config
# Add performance config if latency is specified
if self._settings["latency"] in ["standard", "optimized"]:
request_params["performanceConfig"] = {"latency": self._settings["latency"]}
if self._settings.latency in ["standard", "optimized"]:
request_params["performanceConfig"] = {"latency": self._settings.latency}
# Log request params with messages redacted for logging
if isinstance(context, LLMContext):
@@ -1207,8 +1232,6 @@ class AWSBedrockLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = AWSBedrockLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

117
src/pipecat/services/aws/nova_sonic/llm.py
View File

@@ -16,7 +16,7 @@ import json
import time
import uuid
import wave
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from importlib.resources import files
from typing import Any, List, Optional
@@ -60,6 +60,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
try:
@@ -185,6 +186,20 @@ class Params(BaseModel):
endpointing_sensitivity: Optional[str] = Field(default=None)
@dataclass
class AWSNovaSonicLLMSettings(LLMSettings):
"""Settings for AWS Nova Sonic LLM service.
Parameters:
voice_id: Voice for speech synthesis.
endpointing_sensitivity: Controls how quickly Nova Sonic decides the
user has stopped speaking. Can be "LOW", "MEDIUM", or "HIGH".
"""
voice_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSNovaSonicLLMService(LLMService):
"""AWS Nova Sonic speech-to-speech LLM service.
@@ -192,6 +207,8 @@ class AWSNovaSonicLLMService(LLMService):
and function calling capabilities using AWS Nova Sonic model.
"""
_settings: AWSNovaSonicLLMSettings
# Override the default adapter to use the AWSNovaSonicLLMAdapter one
adapter_class = AWSNovaSonicLLMAdapter
@@ -237,28 +254,51 @@ class AWSNovaSonicLLMService(LLMService):
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs)
params = params or Params()
super().__init__(
settings=AWSNovaSonicLLMSettings(
model=model,
voice_id=voice_id,
temperature=params.temperature,
max_tokens=params.max_tokens,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
endpointing_sensitivity=params.endpointing_sensitivity,
),
**kwargs,
)
self._secret_access_key = secret_access_key
self._access_key_id = access_key_id
self._session_token = session_token
self._region = region
self._model = model
self._client: Optional[BedrockRuntimeClient] = None
self._voice_id = voice_id
self._params = params or Params()
# Audio I/O config (hardware settings, not runtime-tunable)
self._input_sample_rate = params.input_sample_rate
self._input_sample_size = params.input_sample_size
self._input_channel_count = params.input_channel_count
self._output_sample_rate = params.output_sample_rate
self._output_sample_size = params.output_sample_size
self._output_channel_count = params.output_channel_count
self._system_instruction = system_instruction
self._tools = tools
# Validate endpointing_sensitivity parameter
if (
self._params.endpointing_sensitivity
self._settings.endpointing_sensitivity
and not self._is_endpointing_sensitivity_supported()
):
logger.warning(
f"endpointing_sensitivity is not supported for model '{model}' and will be ignored. "
"This parameter is only supported starting with Nova 2 Sonic (amazon.nova-2-sonic-v1:0)."
)
self._params.endpointing_sensitivity = None
self._settings.endpointing_sensitivity = None
if not send_transcription_frames:
import warnings
@@ -302,6 +342,29 @@ class AWSNovaSonicLLMService(LLMService):
with wave.open(file_path.open("rb"), "rb") as wav_file:
self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes())
#
# settings
#
async def _update_settings(self, delta: AWSNovaSonicLLMSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._start_connecting()
self._warn_unhandled_updated_settings(changed)
return changed
#
# standard AIService frame handling
#
@@ -472,7 +535,7 @@ class AWSNovaSonicLLMService(LLMService):
# Start the bidirectional stream
self._stream = await self._client.invoke_model_with_bidirectional_stream(
InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model)
InvokeModelWithBidirectionalStreamOperationInput(model_id=self._settings.model)
)
# Send session start event
@@ -639,7 +702,7 @@ class AWSNovaSonicLLMService(LLMService):
def _is_first_generation_sonic_model(self) -> bool:
# Nova Sonic (the older model) is identified by "amazon.nova-sonic-v1:0"
return self._model == "amazon.nova-sonic-v1:0"
return self._settings.model == "amazon.nova-sonic-v1:0"
def _is_endpointing_sensitivity_supported(self) -> bool:
# endpointing_sensitivity is only supported with Nova 2 Sonic (and,
@@ -658,9 +721,9 @@ class AWSNovaSonicLLMService(LLMService):
turn_detection_config = (
f""",
"turnDetectionConfiguration": {{
"endpointingSensitivity": "{self._params.endpointing_sensitivity}"
"endpointingSensitivity": "{self._settings.endpointing_sensitivity}"
}}"""
if self._params.endpointing_sensitivity
if self._settings.endpointing_sensitivity
else ""
)
@@ -669,9 +732,9 @@ class AWSNovaSonicLLMService(LLMService):
"event": {{
"sessionStart": {{
"inferenceConfiguration": {{
"maxTokens": {self._params.max_tokens},
"topP": {self._params.top_p},
"temperature": {self._params.temperature}
"maxTokens": {self._settings.max_tokens},
"topP": {self._settings.top_p},
"temperature": {self._settings.temperature}
}}{turn_detection_config}
}}
}}
@@ -706,10 +769,10 @@ class AWSNovaSonicLLMService(LLMService):
}},
"audioOutputConfiguration": {{
"mediaType": "audio/lpcm",
"sampleRateHertz": {self._params.output_sample_rate},
"sampleSizeBits": {self._params.output_sample_size},
"channelCount": {self._params.output_channel_count},
"voiceId": "{self._voice_id}",
"sampleRateHertz": {self._output_sample_rate},
"sampleSizeBits": {self._output_sample_size},
"channelCount": {self._output_channel_count},
"voiceId": "{self._settings.voice_id}",
"encoding": "base64",
"audioType": "SPEECH"
}}{tools_config}
@@ -734,9 +797,9 @@ class AWSNovaSonicLLMService(LLMService):
"role": "USER",
"audioInputConfiguration": {{
"mediaType": "audio/lpcm",
"sampleRateHertz": {self._params.input_sample_rate},
"sampleSizeBits": {self._params.input_sample_size},
"channelCount": {self._params.input_channel_count},
"sampleRateHertz": {self._input_sample_rate},
"sampleSizeBits": {self._input_sample_size},
"channelCount": {self._input_channel_count},
"audioType": "SPEECH",
"encoding": "base64"
}}
@@ -1019,8 +1082,8 @@ class AWSNovaSonicLLMService(LLMService):
audio = base64.b64decode(audio_content)
frame = TTSAudioRawFrame(
audio=audio,
sample_rate=self._params.output_sample_rate,
num_channels=self._params.output_channel_count,
sample_rate=self._output_sample_rate,
num_channels=self._output_channel_count,
)
await self.push_frame(frame)
@@ -1304,7 +1367,7 @@ class AWSNovaSonicLLMService(LLMService):
"""
if not self._is_assistant_response_trigger_needed():
logger.warning(
f"Assistant response trigger not needed for model '{self._model}'; skipping. "
f"Assistant response trigger not needed for model '{self._settings.model}'; skipping. "
"An LLMRunFrame() should be sufficient to prompt the assistant to respond, "
"assuming the context ends in a user message."
)
@@ -1332,9 +1395,9 @@ class AWSNovaSonicLLMService(LLMService):
chunk_duration = 0.02 # what we might get from InputAudioRawFrame
chunk_size = int(
chunk_duration
* self._params.input_sample_rate
* self._params.input_channel_count
* (self._params.input_sample_size / 8)
* self._input_sample_rate
* self._input_channel_count
* (self._input_sample_size / 8)
) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes
# Lead with a bit of blank audio, if needed.

97
src/pipecat/services/aws/stt.py
View File

@@ -14,7 +14,8 @@ import json
import os
import random
import string
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -28,6 +29,7 @@ from pipecat.frames.frames import (
TranscriptionFrame,
)
from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import AWS_TRANSCRIBE_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -43,6 +45,25 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AWSTranscribeSTTSettings(STTSettings):
"""Settings for the AWS Transcribe STT service.
Parameters:
sample_rate: Audio sample rate in Hz (8000 or 16000).
media_encoding: Audio encoding format (e.g. "linear16").
number_of_channels: Number of audio channels.
show_speaker_label: Whether to show speaker labels.
enable_channel_identification: Whether to enable channel identification.
"""
sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
media_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
number_of_channels: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
show_speaker_label: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_channel_identification: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSTranscribeSTTService(WebsocketSTTService):
"""AWS Transcribe Speech-to-Text service using WebSocket streaming.
@@ -51,6 +72,8 @@ class AWSTranscribeSTTService(WebsocketSTTService):
final transcription results.
"""
_settings: AWSTranscribeSTTSettings
def __init__(
self,
*,
@@ -76,23 +99,25 @@ class AWSTranscribeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._settings = {
"sample_rate": sample_rate,
"language": language,
"media_encoding": "linear16", # AWS expects raw PCM
"number_of_channels": 1,
"show_speaker_label": False,
"enable_channel_identification": False,
}
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=AWSTranscribeSTTSettings(
language=self.language_to_service_language(language) or "en-US",
sample_rate=sample_rate,
media_encoding="linear16",
number_of_channels=1,
show_speaker_label=False,
enable_channel_identification=False,
),
**kwargs,
)
# Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz
if sample_rate not in [8000, 16000]:
logger.warning(
f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz."
)
self._settings["sample_rate"] = 16000
self._settings.sample_rate = 16000
self._credentials = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -103,6 +128,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
self._receive_task = None
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as AWS Transcribe STT supports metrics generation.
"""
return True
def get_service_encoding(self, encoding: str) -> str:
"""Convert internal encoding format to AWS Transcribe format.
@@ -117,6 +150,26 @@ class AWSTranscribeSTTService(WebsocketSTTService):
}
return encoding_map.get(encoding, encoding)
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if changed and self._websocket:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Initialize the connection when the service starts.
@@ -208,9 +261,9 @@ class AWSTranscribeSTTService(WebsocketSTTService):
logger.debug("Connecting to AWS Transcribe WebSocket")
language_code = self.language_to_service_language(Language(self._settings["language"]))
language_code = self._settings.language
if not language_code:
raise ValueError(f"Unsupported language: {self._settings['language']}")
raise ValueError(f"Unsupported language: {language_code}")
# Generate random websocket key
websocket_key = "".join(
@@ -237,14 +290,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
},
language_code=language_code,
media_encoding=self.get_service_encoding(
self._settings["media_encoding"]
self._settings.media_encoding
), # Convert to AWS format
sample_rate=self._settings["sample_rate"],
number_of_channels=self._settings["number_of_channels"],
sample_rate=self._settings.sample_rate,
number_of_channels=self._settings.number_of_channels,
enable_partial_results_stabilization=True,
partial_results_stability="high",
show_speaker_label=self._settings["show_speaker_label"],
enable_channel_identification=self._settings["enable_channel_identification"],
show_speaker_label=self._settings.show_speaker_label,
enable_channel_identification=self._settings.enable_channel_identification,
)
logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...")
@@ -479,14 +532,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
result=result,
)
)
await self._handle_transcription(
transcript,
is_final,
self._settings["language"],
self._settings.language,
)
await self.stop_processing_metrics()
else:
@@ -495,7 +548,7 @@ class AWSTranscribeSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
result=result,
)
)

76
src/pipecat/services/aws/tts.py
View File

@@ -11,6 +11,7 @@ supporting multiple languages, voices, and SSML features.
"""
import os
from dataclasses import dataclass, field
from typing import AsyncGenerator, List, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -121,6 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class AWSPollyTTSSettings(TTSSettings):
"""Settings for AWS Polly TTS service.
Parameters:
engine: TTS engine to use ('standard', 'neural', etc.).
pitch: Voice pitch adjustment (for standard engine only).
rate: Speech rate adjustment.
volume: Voice volume adjustment.
lexicon_names: List of pronunciation lexicons to apply.
"""
engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
lexicon_names: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSPollyTTSService(TTSService):
"""AWS Polly text-to-speech service.
@@ -129,6 +150,8 @@ class AWSPollyTTSService(TTSService):
options including prosody controls.
"""
_settings: AWSPollyTTSSettings
class InputParams(BaseModel):
"""Input parameters for AWS Polly TTS configuration.
@@ -172,10 +195,25 @@ class AWSPollyTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AWSPollyTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AWSPollyTTSSettings(
model=None,
voice=voice_id,
engine=params.engine,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
volume=params.volume,
lexicon_names=params.lexicon_names,
),
**kwargs,
)
# Get credentials from environment variables if not provided
self._aws_params = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -185,21 +223,9 @@ class AWSPollyTTSService(TTSService):
}
self._aws_session = aioboto3.Session()
self._settings = {
"engine": params.engine,
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
"lexicon_names": params.lexicon_names,
}
self._resampler = create_stream_resampler()
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -222,19 +248,19 @@ class AWSPollyTTSService(TTSService):
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
language = self._settings["language"]
language = self._settings.language
ssml += f"<lang xml:lang='{language}'>"
prosody_attrs = []
# Prosody tags are only supported for standard and neural engines
if self._settings["engine"] == "standard":
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings.engine == "standard":
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
@@ -275,11 +301,11 @@ class AWSPollyTTSService(TTSService):
"Text": ssml,
"TextType": "ssml",
"OutputFormat": "pcm",
"VoiceId": self._voice_id,
"Engine": self._settings["engine"],
"VoiceId": self._settings.voice,
"Engine": self._settings.engine,
# AWS only supports 8000 and 16000 for PCM. We select 16000.
"SampleRate": "16000",
"LexiconNames": self._settings["lexicon_names"],
"LexiconNames": self._settings.lexicon_names,
}
# Filter out None values

14
src/pipecat/services/azure/image.py
View File

@@ -12,6 +12,7 @@ using REST endpoints for creating images from text prompts.
import asyncio
import io
from dataclasses import dataclass
from typing import AsyncGenerator
import aiohttp
@@ -19,6 +20,16 @@ from PIL import Image
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class AzureImageGenSettings(ImageGenSettings):
"""Settings for the Azure image generation service.
Parameters:
model: Azure image generation model identifier.
"""
class AzureImageGenServiceREST(ImageGenService):
@@ -49,12 +60,11 @@ class AzureImageGenServiceREST(ImageGenService):
aiohttp_session: Shared aiohttp session for HTTP requests.
api_version: Azure API version string. Defaults to "2023-06-01-preview".
"""
super().__init__()
super().__init__(settings=AzureImageGenSettings(model=model))
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self.set_model_name(model)
self._image_size = image_size
self._aiohttp_session = aiohttp_session

88
src/pipecat/services/azure/stt.py
View File

@@ -11,7 +11,8 @@ Speech SDK for real-time audio transcription.
"""
import asyncio
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -25,6 +26,7 @@ from pipecat.frames.frames import (
TranscriptionFrame,
)
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import AZURE_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -33,6 +35,7 @@ from pipecat.utils.tracing.service_decorators import traced_stt
try:
from azure.cognitiveservices.speech import (
CancellationReason,
ResultReason,
SpeechConfig,
SpeechRecognizer,
@@ -48,6 +51,19 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AzureSTTSettings(STTSettings):
"""Settings for the Azure STT service.
Parameters:
region: Azure region for the Speech service.
sample_rate: Audio sample rate in Hz.
"""
region: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AzureSTTService(STTService):
"""Azure Speech-to-Text service for real-time audio transcription.
@@ -56,6 +72,8 @@ class AzureSTTService(STTService):
provides real-time transcription results with timing information.
"""
_settings: AzureSTTSettings
def __init__(
self,
*,
@@ -63,6 +81,7 @@ class AzureSTTService(STTService):
region: str,
language: Language = Language.EN_US,
sample_rate: Optional[int] = None,
private_endpoint: Optional[str] = None,
endpoint_id: Optional[str] = None,
ttfs_p99_latency: Optional[float] = AZURE_TTFS_P99,
**kwargs,
@@ -74,17 +93,30 @@ class AzureSTTService(STTService):
region: Azure region for the Speech service (e.g., 'eastus').
language: Language for speech recognition. Defaults to English (US).
sample_rate: Audio sample rate in Hz. If None, uses service default.
private_endpoint: Private endpoint for STT behind firewall.
See https://docs.azure.cn/en-us/ai-services/speech-service/speech-services-private-link?tabs=portal
endpoint_id: Custom model endpoint id.
ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AzureSTTSettings(
model=None,
region=region,
language=language_to_azure_language(language),
sample_rate=sample_rate,
),
**kwargs,
)
self._speech_config = SpeechConfig(
subscription=api_key,
region=region,
speech_recognition_language=language_to_azure_language(language),
endpoint=private_endpoint,
)
if endpoint_id:
@@ -92,11 +124,6 @@ class AzureSTTService(STTService):
self._audio_stream = None
self._speech_recognizer = None
self._settings = {
"region": region,
"language": language_to_azure_language(language),
"sample_rate": sample_rate,
}
def can_generate_metrics(self) -> bool:
"""Check if this service can generate performance metrics.
@@ -106,6 +133,38 @@ class AzureSTTService(STTService):
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Azure service-specific language code.
Args:
language: The language to convert.
Returns:
The Azure-specific language identifier, or None if not supported.
"""
return language_to_azure_language(language)
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active recognizer.
"""
changed = await super()._update_settings(delta)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if "language" in changed:
# self._speech_config.speech_recognition_language = self._settings.language
# if self._speech_recognizer:
# # Requires refactoring to set up and tear down recognizer, as
# # language is applied at recognizer initialization
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text conversion.
@@ -151,6 +210,7 @@ class AzureSTTService(STTService):
)
self._speech_recognizer.recognizing.connect(self._on_handle_recognizing)
self._speech_recognizer.recognized.connect(self._on_handle_recognized)
self._speech_recognizer.canceled.connect(self._on_handle_canceled)
self._speech_recognizer.start_continuous_recognition_async()
except Exception as e:
await self.push_error(
@@ -198,7 +258,7 @@ class AzureSTTService(STTService):
def _on_handle_recognized(self, event):
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
language = getattr(event.result, "language", None) or self._settings.get("language")
language = getattr(event.result, "language", None) or self._settings.language
frame = TranscriptionFrame(
event.result.text,
self._user_id,
@@ -213,7 +273,7 @@ class AzureSTTService(STTService):
def _on_handle_recognizing(self, event):
if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0:
language = getattr(event.result, "language", None) or self._settings.get("language")
language = getattr(event.result, "language", None) or self._settings.language
frame = InterimTranscriptionFrame(
event.result.text,
self._user_id,
@@ -222,3 +282,13 @@ class AzureSTTService(STTService):
result=event,
)
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
def _on_handle_canceled(self, event):
details = event.result.cancellation_details
if details.reason == CancellationReason.Error:
error_msg = f"Azure STT recognition canceled: {details.reason}"
if details.error_details:
error_msg += f" - {details.error_details}"
asyncio.run_coroutine_threadsafe(
self.push_error(error_msg=error_msg), self.get_event_loop()
)

158
src/pipecat/services/azure/tts.py
View File

@@ -7,6 +7,7 @@
"""Azure Cognitive Services Text-to-Speech service implementations."""
import asyncio
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -25,7 +26,8 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.tts_service import TTSService, WordTTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -65,6 +67,31 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma
return sample_rate_map.get(sample_rate, SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm)
@dataclass
class AzureTTSSettings(TTSSettings):
"""Settings for Azure TTS services.
Parameters:
emphasis: Emphasis level for speech ("strong", "moderate", "reduced").
language: Language for synthesis. Defaults to English (US).
pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high").
rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast").
role: Voice role for expression (e.g., "YoungAdultFemale").
style: Speaking style (e.g., "cheerful", "sad", "excited").
style_degree: Intensity of the speaking style (0.01 to 2.0).
volume: Volume level (e.g., "+20%", "loud", "x-soft").
"""
emphasis: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
role: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style_degree: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AzureBaseTTSService:
"""Base mixin class for Azure Cognitive Services text-to-speech implementations.
@@ -73,6 +100,8 @@ class AzureBaseTTSService:
This is a mixin class and should be used alongside TTSService or its subclasses.
"""
_settings: AzureTTSSettings
# Define SSML escape mappings based on SSML reserved characters
# See - https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-structure
SSML_ESCAPE_CHARS = {
@@ -112,7 +141,6 @@ class AzureBaseTTSService:
api_key: str,
region: str,
voice: str = "en-US-SaraNeural",
params: Optional[InputParams] = None,
):
"""Initialize Azure-specific configuration.
@@ -122,26 +150,9 @@ class AzureBaseTTSService:
api_key: Azure Cognitive Services subscription key.
region: Azure region identifier (e.g., "eastus", "westus2").
voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
params: Voice and synthesis parameters configuration.
"""
params = params or AzureBaseTTSService.InputParams()
self._settings = {
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"role": params.role,
"style": params.style,
"style_degree": params.style_degree,
"volume": params.volume,
}
self._api_key = api_key
self._region = region
self._voice_id = voice
self._speech_synthesizer = None
def language_to_service_language(self, language: Language) -> Optional[str]:
@@ -156,7 +167,7 @@ class AzureBaseTTSService:
return language_to_azure_language(language)
def _construct_ssml(self, text: str) -> str:
language = self._settings["language"]
language = self._settings.language
# Escape special characters
escaped_text = self._escape_text(text)
@@ -165,42 +176,42 @@ class AzureBaseTTSService:
f"<speak version='1.0' xml:lang='{language}' "
"xmlns='http://www.w3.org/2001/10/synthesis' "
"xmlns:mstts='http://www.w3.org/2001/mstts'>"
f"<voice name='{self._voice_id}'>"
f"<voice name='{self._settings.voice}'>"
"<mstts:silence type='Sentenceboundary' value='20ms' />"
)
if self._settings["style"]:
ssml += f"<mstts:express-as style='{self._settings['style']}'"
if self._settings["style_degree"]:
ssml += f" styledegree='{self._settings['style_degree']}'"
if self._settings["role"]:
ssml += f" role='{self._settings['role']}'"
if self._settings.style:
ssml += f"<mstts:express-as style='{self._settings.style}'"
if self._settings.style_degree:
ssml += f" styledegree='{self._settings.style_degree}'"
if self._settings.role:
ssml += f" role='{self._settings.role}'"
ssml += ">"
prosody_attrs = []
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
# Only wrap in prosody tag if there are prosody attributes
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
if self._settings.emphasis:
ssml += f"<emphasis level='{self._settings.emphasis}'>"
ssml += escaped_text
if self._settings["emphasis"]:
if self._settings.emphasis:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
if self._settings["style"]:
if self._settings.style:
ssml += "</mstts:express-as>"
ssml += "</voice></speak>"
@@ -229,7 +240,7 @@ class AzureBaseTTSService:
return escaped_text
class AzureTTSService(WordTTSService, AzureBaseTTSService):
class AzureTTSService(TTSService, AzureBaseTTSService):
"""Azure Cognitive Services streaming TTS service with word timestamps.
Provides real-time text-to-speech synthesis using Azure's WebSocket-based
@@ -245,7 +256,8 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
voice: str = "en-US-SaraNeural",
sample_rate: Optional[int] = None,
params: Optional[AzureBaseTTSService.InputParams] = None,
aggregate_sentences: bool = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize the Azure streaming TTS service.
@@ -256,21 +268,43 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
sample_rate: Audio sample rate in Hz. If None, uses service default.
params: Voice and synthesis parameters configuration.
aggregate_sentences: Whether to aggregate sentences before synthesis.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to parent WordTTSService.
"""
# Initialize WordTTSService first to set up word timestamp tracking
params = params or AzureBaseTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_text_frames=False, # We'll push text frames based on word timestamps
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs,
)
# Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None
self._speech_synthesizer = None
@@ -314,7 +348,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
subscription=self._api_key,
region=self._region,
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.speech_synthesis_language = self._settings.language
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)
@@ -364,7 +398,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
Returns:
True if the language is CJK, False otherwise.
"""
language = self._settings.get("language", "").lower()
language = (self._settings.language if self._settings.language else "").lower()
# Check if language starts with CJK language codes
return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu"))
@@ -527,9 +561,13 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
# User cancellation (from interruption) is expected, not an error
if reason == CancellationReason.CancelledByUser:
logger.debug(f"{self}: Speech synthesis canceled by user (interruption)")
self._audio_queue.put_nowait(None)
else:
logger.warning(f"{self}: Speech synthesis canceled: {reason}")
self._audio_queue.put_nowait(None)
details = evt.result.cancellation_details
error_msg = f"Azure TTS synthesis canceled: {reason}"
if details.error_details:
error_msg += f" - {details.error_details}"
self._audio_queue.put_nowait(Exception(error_msg))
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame and handle state changes.
@@ -642,6 +680,9 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
chunk = await self._audio_queue.get()
if chunk is None: # End of stream
break
if isinstance(chunk, Exception): # Error from _handle_canceled
yield ErrorFrame(error=str(chunk))
break
if self._first_chunk:
await self.stop_ttfb_metrics()
@@ -704,10 +745,29 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
params: Voice and synthesis parameters configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AzureBaseTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs,
)
# Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None
self._speech_synthesizer = None
@@ -735,7 +795,7 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
subscription=self._api_key,
region=self._region,
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.speech_synthesis_language = self._settings.language
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)

57
src/pipecat/services/camb/tts.py
View File

@@ -16,6 +16,7 @@ Features:
- Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz)
"""
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional
from camb import StreamTtsOutputConfiguration
@@ -31,6 +32,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -133,6 +135,18 @@ def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]:
return buffer[:aligned_size], buffer[aligned_size:]
@dataclass
class CambTTSSettings(TTSSettings):
"""Settings for Camb.ai TTS service.
Parameters:
user_instructions: Custom instructions for mars-instruct model only.
Ignored for other models. Max 1000 characters.
"""
user_instructions: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class CambTTSService(TTSService):
"""Camb.ai MARS text-to-speech service using the official SDK.
@@ -156,6 +170,8 @@ class CambTTSService(TTSService):
)
"""
_settings: CambTTSSettings
class InputParams(BaseModel):
"""Input parameters for Camb.ai TTS configuration.
@@ -197,11 +213,6 @@ class CambTTSService(TTSService):
params: Additional voice parameters. If None, uses defaults.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._timeout = timeout
params = params or CambTTSService.InputParams()
# Warn if sample rate doesn't match model's supported rate
@@ -211,17 +222,23 @@ class CambTTSService(TTSService):
f"sample rate. Current rate of {sample_rate}Hz may cause issues."
)
# Build settings
self._settings = {
"language": (
self.language_to_service_language(params.language) if params.language else "en-us"
super().__init__(
sample_rate=sample_rate,
settings=CambTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language)
if params.language
else "en-us"
),
user_instructions=params.user_instructions,
),
"user_instructions": params.user_instructions,
}
**kwargs,
)
self.set_model_name(model)
self.set_voice(str(voice_id))
self._voice_id = voice_id
self._api_key = api_key
self._timeout = timeout
self._client = None
@@ -256,7 +273,7 @@ class CambTTSService(TTSService):
# Use model-specific sample rate if not explicitly specified
if not self._init_sample_rate:
self._sample_rate = MODEL_SAMPLE_RATES.get(self.model_name, 22050)
self._sample_rate = MODEL_SAMPLE_RATES.get(self._settings.model, 22050)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -282,15 +299,15 @@ class CambTTSService(TTSService):
# Build SDK parameters
tts_kwargs: Dict[str, Any] = {
"text": text,
"voice_id": self._voice_id,
"language": self._settings["language"],
"speech_model": self.model_name,
"voice_id": self._settings.voice,
"language": self._settings.language,
"speech_model": self._settings.model,
"output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"),
}
# Add user instructions if using mars-instruct model
if self._model_name == "mars-instruct" and self._settings.get("user_instructions"):
tts_kwargs["user_instructions"] = self._settings["user_instructions"]
if self._settings.model == "mars-instruct" and self._settings.user_instructions:
tts_kwargs["user_instructions"] = self._settings.user_instructions
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame(context_id=context_id)

67
src/pipecat/services/cartesia/stt.py
View File

@@ -12,7 +12,8 @@ the Cartesia Live transcription API for real-time speech recognition.
import json
import urllib.parse
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import CARTESIA_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -42,6 +44,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class CartesiaSTTSettings(STTSettings):
"""Settings for the Cartesia STT service.
Parameters:
encoding: Audio encoding format (e.g. ``"pcm_s16le"``).
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class CartesiaLiveOptions:
"""Configuration options for Cartesia Live STT service.
@@ -136,6 +149,8 @@ class CartesiaSTTService(WebsocketSTTService):
See: https://docs.cartesia.ai/api-reference/stt/stt
"""
_settings: CartesiaSTTSettings
def __init__(
self,
*,
@@ -158,13 +173,6 @@ class CartesiaSTTService(WebsocketSTTService):
**kwargs: Additional arguments passed to parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=120,
keepalive_interval=30,
**kwargs,
)
default_options = CartesiaLiveOptions(
model="ink-whisper",
@@ -181,8 +189,19 @@ class CartesiaSTTService(WebsocketSTTService):
k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None"
}
self._settings = merged_options
self.set_model_name(merged_options["model"])
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=120,
keepalive_interval=30,
settings=CartesiaSTTSettings(
model=merged_options["model"],
language=merged_options.get("language"),
encoding=merged_options.get("encoding", "pcm_s16le"),
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url or "api.cartesia.ai"
self._receive_task = None
@@ -275,13 +294,39 @@ class CartesiaSTTService(WebsocketSTTService):
await self._disconnect_websocket()
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Args:
delta: A :class:`STTSettings` (or ``CartesiaSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if changed:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def _connect_websocket(self):
try:
if self._websocket and self._websocket.state is State.OPEN:
return
logger.debug("Connecting to Cartesia STT")
params = self._settings
params = {
"model": self._settings.model,
"language": self._settings.language,
"encoding": self._settings.encoding,
"sample_rate": str(self.sample_rate),
}
ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}"
headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key}

252
src/pipecat/services/cartesia/tts.py
View File

@@ -9,8 +9,9 @@
import base64
import json
import warnings
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, List, Literal, Optional
from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional
from loguru import logger
from pydantic import BaseModel, Field
@@ -20,14 +21,13 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
@@ -191,7 +191,43 @@ class CartesiaEmotion(str, Enum):
DETERMINED = "determined"
class CartesiaTTSService(AudioContextWordTTSService):
@dataclass
class CartesiaTTSSettings(TTSSettings):
"""Settings for Cartesia TTS services.
Parameters:
output_container: Audio container format (e.g. "raw").
output_encoding: Audio encoding format (e.g. "pcm_s16le").
output_sample_rate: Audio sample rate in Hz.
speed: Voice speed control for non-Sonic-3 models (literal values).
emotion: List of emotion controls for non-Sonic-3 models.
generation_config: Generation configuration for Sonic-3 models. Includes volume,
speed (numeric), and emotion (string) parameters.
pronunciation_dict_id: The ID of the pronunciation dictionary to use for
custom pronunciations.
"""
output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: Literal["slow", "normal", "fast"] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emotion: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
generation_config: GenerationConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pronunciation_dict_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "CartesiaTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``output_format``."""
flat = dict(settings)
nested = flat.pop("output_format", None)
if isinstance(nested, dict):
flat.setdefault("output_container", nested.get("container"))
flat.setdefault("output_encoding", nested.get("encoding"))
flat.setdefault("output_sample_rate", nested.get("sample_rate"))
return super().from_mapping(flat)
class CartesiaTTSService(AudioContextTTSService):
"""Cartesia TTS service with WebSocket streaming and word timestamps.
Provides text-to-speech using Cartesia's streaming WebSocket API.
@@ -199,6 +235,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
customization options including speed and emotion controls.
"""
_settings: CartesiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Cartesia TTS configuration.
@@ -234,7 +272,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
container: str = "raw",
params: Optional[InputParams] = None,
text_aggregator: Optional[BaseTextAggregator] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize the Cartesia TTS service.
@@ -254,25 +293,51 @@ class CartesiaTTSService(AudioContextWordTTSService):
.. deprecated:: 0.0.95
Use an LLMTextProcessor before the TTSService for custom text aggregation.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to the parent service.
"""
# Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
# 3 model, and it's worth it for the better audio quality.
# By default, we aggregate sentences before sending to TTS. This adds
# ~200-300ms of latency per sentence (waiting for the sentence-ending
# punctuation token from the LLM). Setting
# text_aggregation_mode=TextAggregationMode.TOKEN streams tokens
# directly, which reduces latency. Streaming quality is good but less
# tested than sentence aggregation.
# TODO: Consider making TOKEN the default for Cartesia in 1.0.
#
# We also don't want to automatically push LLM response text frames,
# because the context aggregators will add them to the LLM context even
# if we're interrupted. Cartesia gives us word-by-word timestamps. We
# can use those to generate text frames ourselves aligned with the
# playout timing of the audio!
params = params or CartesiaTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
text_aggregator=text_aggregator,
settings=CartesiaTTSSettings(
model=model,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
voice=voice_id,
),
**kwargs,
)
@@ -282,29 +347,13 @@ class CartesiaTTSService(AudioContextWordTTSService):
# The preferred way of taking advantage of Cartesia SSML Tags is
# to use an LLMTextProcessor and/or a text_transformer to identify
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("<spell>", "</spell>")])
params = params or CartesiaTTSService.InputParams()
self._text_aggregator = SkipTagsAggregator(
[("<spell>", "</spell>")], aggregation_type=self._text_aggregation_mode
)
self._api_key = api_key
self._cartesia_version = cartesia_version
self._url = url
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
"speed": params.speed,
"emotion": params.emotion,
"generation_config": params.generation_config,
"pronunciation_dict_id": params.pronunciation_dict_id,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._receive_task = None
@@ -316,16 +365,6 @@ class CartesiaTTSService(AudioContextWordTTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model.
Args:
model: The model name to use for synthesis.
"""
self._model_id = model
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Cartesia language format.
@@ -390,7 +429,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
Returns:
List of (word, start_time) tuples processed for the language.
"""
current_language = self._settings.get("language")
current_language = self._settings.language
# Check if this is a CJK language (if language is None, treat as non-CJK)
if current_language and self._is_cjk_language(current_language):
@@ -411,9 +450,9 @@ class CartesiaTTSService(AudioContextWordTTSService):
):
voice_config = {}
voice_config["mode"] = "id"
voice_config["id"] = self._voice_id
voice_config["id"] = self._settings.voice
if self._settings["emotion"]:
if self._settings.emotion:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -422,33 +461,36 @@ class CartesiaTTSService(AudioContextWordTTSService):
stacklevel=2,
)
voice_config["__experimental_controls"] = {}
if self._settings["emotion"]:
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
voice_config["__experimental_controls"]["emotion"] = self._settings.emotion
msg = {
"transcript": text,
"continue": continue_transcript,
"context_id": self.get_active_audio_context_id(),
"model_id": self.model_name,
"model_id": self._settings.model,
"voice": voice_config,
"output_format": self._settings["output_format"],
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"add_timestamps": add_timestamps,
"use_original_timestamps": False if self.model_name == "sonic" else True,
"use_original_timestamps": False if self._settings.model == "sonic" else True,
}
if self._settings["language"]:
msg["language"] = self._settings["language"]
if self._settings.language:
msg["language"] = self._settings.language
if self._settings["speed"]:
msg["speed"] = self._settings["speed"]
if self._settings.speed:
msg["speed"] = self._settings.speed
if self._settings["generation_config"]:
msg["generation_config"] = self._settings["generation_config"].model_dump(
if self._settings.generation_config:
msg["generation_config"] = self._settings.generation_config.model_dump(
exclude_none=True
)
if self._settings["pronunciation_dict_id"]:
msg["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
if self._settings.pronunciation_dict_id:
msg["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
return json.dumps(msg)
@@ -459,7 +501,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -530,14 +572,22 @@ class CartesiaTTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
context_id = self.get_active_audio_context_id()
await super()._handle_interruption(frame, direction)
async def on_audio_context_interrupted(self, context_id: str):
"""Cancel the active Cartesia context when the bot is interrupted."""
await self.stop_all_metrics()
if context_id:
cancel_msg = json.dumps({"context_id": context_id, "cancel": True})
await self._get_websocket().send(cancel_msg)
async def on_audio_context_completed(self, context_id: str):
"""Close the Cartesia context after all audio has been played.
No close message is needed: the server already considers the context
done once it has sent its ``done`` message, which is handled in
``_process_messages``.
"""
pass
async def flush_audio(self):
"""Flush any pending audio and finalize the current context."""
context_id = self.get_active_audio_context_id()
@@ -601,7 +651,10 @@ class CartesiaTTSService(AudioContextWordTTSService):
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]")
if not self._is_streaming_tokens:
logger.debug(f"{self}: Generating TTS [{text}]")
else:
logger.trace(f"{self}: Generating TTS [{text}]")
try:
if not self._websocket or self._websocket.state is State.CLOSED:
@@ -636,6 +689,8 @@ class CartesiaHttpTTSService(TTSService):
integration is preferred.
"""
_settings: CartesiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Cartesia HTTP TTS configuration.
@@ -686,29 +741,30 @@ class CartesiaHttpTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or CartesiaHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=CartesiaTTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._cartesia_version = cartesia_version
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
"speed": params.speed,
"emotion": params.emotion,
"generation_config": params.generation_config,
"pronunciation_dict_id": params.pronunciation_dict_id,
}
self.set_voice(voice_id)
self.set_model_name(model)
self._client = AsyncCartesia(
api_key=api_key,
@@ -741,7 +797,7 @@ class CartesiaHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
async def stop(self, frame: EndFrame):
"""Stop the Cartesia HTTP TTS service.
@@ -775,9 +831,9 @@ class CartesiaHttpTTSService(TTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
voice_config = {"mode": "id", "id": self._voice_id}
voice_config = {"mode": "id", "id": self._settings.voice}
if self._settings["emotion"]:
if self._settings.emotion:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -785,30 +841,36 @@ class CartesiaHttpTTSService(TTSService):
DeprecationWarning,
stacklevel=2,
)
voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]}
voice_config["__experimental_controls"] = {"emotion": self._settings.emotion}
await self.start_ttfb_metrics()
payload = {
"model_id": self._model_name,
"transcript": text,
"voice": voice_config,
"output_format": self._settings["output_format"],
output_format = {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
}
if self._settings["language"]:
payload["language"] = self._settings["language"]
payload = {
"model_id": self._settings.model,
"transcript": text,
"voice": voice_config,
"output_format": output_format,
}
if self._settings["speed"]:
payload["speed"] = self._settings["speed"]
if self._settings.language:
payload["language"] = self._settings.language
if self._settings["generation_config"]:
payload["generation_config"] = self._settings["generation_config"].model_dump(
if self._settings.speed:
payload["speed"] = self._settings.speed
if self._settings.generation_config:
payload["generation_config"] = self._settings.generation_config.model_dump(
exclude_none=True
)
if self._settings["pronunciation_dict_id"]:
payload["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
if self._settings.pronunciation_dict_id:
payload["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
yield TTSStartedFrame(context_id=context_id)

12
src/pipecat/services/cerebras/llm.py
View File

@@ -66,16 +66,16 @@ class CerebrasLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"seed": self._settings.seed,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_completion_tokens": self._settings.max_completion_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

104
src/pipecat/services/deepgram/flux/stt.py
View File

@@ -9,6 +9,7 @@
import asyncio
import json
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator, Dict, Optional
from urllib.parse import urlencode
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
@@ -67,6 +69,34 @@ class FluxEventType(str, Enum):
UPDATE = "Update"
@dataclass
class DeepgramFluxSTTSettings(STTSettings):
"""Settings for the Deepgram Flux STT service.
Parameters:
eager_eot_threshold: EagerEndOfTurn/TurnResumed threshold. Off by default.
Lower values = more aggressive (faster response, more LLM calls).
Higher values = more conservative (slower response, fewer LLM calls).
eot_threshold: End-of-turn confidence required to finish a turn (default 0.7).
eot_timeout_ms: Time in ms after speech to finish a turn regardless of EOT
confidence (default 5000).
keyterm: Keyterms to boost recognition accuracy for specialized terminology.
mip_opt_out: Opt out of the Deepgram Model Improvement Program (default False).
tag: Tags to label requests for identification during usage reporting.
min_confidence: Minimum confidence required to create a TranscriptionFrame.
encoding: Audio encoding format (e.g. ``"linear16"``).
"""
eager_eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
eot_timeout_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
keyterm: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
mip_opt_out: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
tag: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_confidence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramFluxSTTService(WebsocketSTTService):
"""Deepgram Flux speech-to-text service.
@@ -89,6 +119,8 @@ class DeepgramFluxSTTService(WebsocketSTTService):
...
"""
_settings: DeepgramFluxSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Deepgram Flux API.
@@ -175,20 +207,27 @@ class DeepgramFluxSTTService(WebsocketSTTService):
# was never destroyed.
# So we can keep it here as false, because inside the method send_with_retry, it will
# already try to reconnect if needed.
params = params or DeepgramFluxSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
reconnect_on_error=False,
settings=DeepgramFluxSTTSettings(
model=model,
language=Language.EN,
encoding=flux_encoding,
eager_eot_threshold=params.eager_eot_threshold,
eot_threshold=params.eot_threshold,
eot_timeout_ms=params.eot_timeout_ms,
keyterm=params.keyterm or [],
mip_opt_out=params.mip_opt_out,
tag=params.tag or [],
min_confidence=params.min_confidence,
),
**kwargs,
)
self._api_key = api_key
self._url = url
self._model = model
self._params = params or DeepgramFluxSTTService.InputParams()
self._should_interrupt = should_interrupt
self._flux_encoding = flux_encoding
# This is the currently only supported language
self._language = Language.EN
self._websocket_url = None
self._receive_task = None
# Flux event handlers
@@ -343,6 +382,25 @@ class DeepgramFluxSTTService(WebsocketSTTService):
"""
return True
async def _update_settings(self, delta: DeepgramFluxSTTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram Flux STT service.
@@ -355,29 +413,29 @@ class DeepgramFluxSTTService(WebsocketSTTService):
await super().start(frame)
url_params = [
f"model={self._model}",
f"model={self._settings.model}",
f"sample_rate={self.sample_rate}",
f"encoding={self._flux_encoding}",
f"encoding={self._settings.encoding}",
]
if self._params.eager_eot_threshold is not None:
url_params.append(f"eager_eot_threshold={self._params.eager_eot_threshold}")
if self._settings.eager_eot_threshold is not None:
url_params.append(f"eager_eot_threshold={self._settings.eager_eot_threshold}")
if self._params.eot_threshold is not None:
url_params.append(f"eot_threshold={self._params.eot_threshold}")
if self._settings.eot_threshold is not None:
url_params.append(f"eot_threshold={self._settings.eot_threshold}")
if self._params.eot_timeout_ms is not None:
url_params.append(f"eot_timeout_ms={self._params.eot_timeout_ms}")
if self._settings.eot_timeout_ms is not None:
url_params.append(f"eot_timeout_ms={self._settings.eot_timeout_ms}")
if self._params.mip_opt_out is not None:
url_params.append(f"mip_opt_out={str(self._params.mip_opt_out).lower()}")
if self._settings.mip_opt_out is not None:
url_params.append(f"mip_opt_out={str(self._settings.mip_opt_out).lower()}")
# Add keyterm parameters (can have multiple)
for keyterm in self._params.keyterm:
for keyterm in self._settings.keyterm:
url_params.append(urlencode({"keyterm": keyterm}))
# Add tag parameters (can have multiple)
for tag_value in self._params.tag:
for tag_value in self._settings.tag:
url_params.append(urlencode({"tag": tag_value}))
self._websocket_url = f"{self._url}?{'&'.join(url_params)}"
@@ -617,7 +675,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
self._user_is_speaking = True
await self.broadcast_frame(UserStartedSpeakingFrame)
if self._should_interrupt:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
await self.start_metrics()
await self._call_event_handler("on_start_of_turn", transcript)
if transcript:
@@ -676,7 +734,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
# Compute the average confidence
average_confidence = self._calculate_average_confidence(data)
if not self._params.min_confidence or average_confidence > self._params.min_confidence:
if not self._settings.min_confidence or average_confidence > self._settings.min_confidence:
# EndOfTurn means Flux has determined the turn is complete,
# so this TranscriptionFrame is always finalized
await self.push_frame(
@@ -684,7 +742,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
result=data,
finalized=True,
)
@@ -694,7 +752,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
f"Transcription confidence below min_confidence threshold: {average_confidence}"
)
await self._handle_transcription(transcript, True, self._language)
await self._handle_transcription(transcript, True, self._settings.language)
await self.stop_processing_metrics()
await self.broadcast_frame(UserStoppedSpeakingFrame)
await self._call_event_handler("on_end_of_turn", transcript)
@@ -738,7 +796,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
result=data,
)
)

235
src/pipecat/services/deepgram/stt.py
View File

@@ -6,7 +6,9 @@
"""Deepgram speech-to-text service implementation."""
from typing import AsyncGenerator, Dict, Optional
import inspect
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Mapping, Optional, Type
from loguru import logger
@@ -23,6 +25,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import _S, NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import DEEPGRAM_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -45,6 +48,168 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class _DeepgramSTTSettingsBase(STTSettings):
"""Base settings for Deepgram STT services that use ``LiveOptions``.
Shared by ``DeepgramSTTSettings`` and ``DeepgramSageMakerSTTSettings``.
Not intended for other Deepgram services that don't use ``LiveOptions``.
Wraps the Deepgram SDK's ``LiveOptions`` in a single ``live_options``
field and provides delta-merge semantics: when used as a delta (e.g.
via ``STTUpdateSettingsFrame``), only the non-None fields of
``live_options`` are merged into the stored options rather than
replacing them wholesale.
``model`` and ``language`` are kept in sync bidirectionally between
the top-level settings fields and the nested ``live_options``.
Parameters:
live_options: Deepgram ``LiveOptions`` for STT configuration.
In delta mode only its non-None fields are merged into the
stored options.
"""
live_options: LiveOptions | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
# Valid LiveOptions __init__ parameter names (cached at class level).
_live_options_params: set[str] | None = field(default=None, init=False, repr=False)
@classmethod
def _get_live_options_params(cls) -> set[str]:
"""Return the set of valid ``LiveOptions.__init__`` parameter names."""
if cls._live_options_params is None:
cls._live_options_params = set(inspect.signature(LiveOptions.__init__).parameters) - {
"self"
}
return cls._live_options_params
def _merge_live_options_delta(self, delta: LiveOptions) -> Dict[str, Any]:
"""Merge a ``LiveOptions`` delta into the stored ``live_options``.
Non-None fields from *delta* overwrite corresponding fields in the
stored ``LiveOptions``. ``model`` and ``language`` are synced to
the top-level settings fields when they change.
Args:
delta: A ``LiveOptions`` whose non-None fields are the desired
overrides.
Returns:
Dict mapping each changed key to its **previous** value (same
contract as ``apply_update``).
"""
old_dict = self.live_options.to_dict() # type: ignore[union-attr]
delta_dict = delta.to_dict()
# Deepgram SDK bug: model initialised to the *string* "None".
if delta_dict.get("model") == "None":
del delta_dict["model"]
if not delta_dict:
return {}
merged = {**old_dict, **delta_dict}
self.live_options = LiveOptions(**merged)
# Track what changed.
changed: Dict[str, Any] = {}
for key in delta_dict:
old_val = old_dict.get(key, NOT_GIVEN)
if old_val != delta_dict[key]:
changed[key] = old_val
# Sync model/language from live_options delta to top-level fields.
if "model" in delta_dict and delta_dict["model"] != self.model:
changed.setdefault("model", self.model)
self.model = delta_dict["model"]
if "language" in delta_dict and delta_dict["language"] != self.language:
changed.setdefault("language", self.language)
self.language = delta_dict["language"]
return changed
def apply_update(self: _S, delta: _S) -> Dict[str, Any]:
"""Merge a delta into this store, with delta-merge for ``live_options``.
``live_options`` is merged field-by-field via
``_merge_live_options_delta`` rather than being replaced wholesale.
``model`` and ``language`` are kept in sync bidirectionally between
the top-level settings fields and ``live_options``.
"""
# Pull live_options out of the delta so super() doesn't replace it.
delta_lo = getattr(delta, "live_options", NOT_GIVEN)
if is_given(delta_lo):
delta.live_options = NOT_GIVEN # type: ignore[assignment]
# Let the base class handle model, language, extra.
changed = super().apply_update(delta)
# Sync top-level model/language changes into stored live_options.
if "model" in changed:
self.live_options.model = self.model # type: ignore[union-attr]
if "language" in changed:
self.live_options.language = self.language # type: ignore[union-attr]
# Merge live_options delta. Top-level model/language take precedence
# over conflicting values in live_options, so write them into the
# delta before merging.
if is_given(delta_lo):
if "model" in changed:
delta_lo.model = self.model
if "language" in changed:
delta_lo.language = self.language
for key, old_val in self._merge_live_options_delta(delta_lo).items():
changed.setdefault(key, old_val)
return changed
@classmethod
def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S:
"""Build a delta from a plain dict, routing LiveOptions keys correctly.
Keys that are valid ``LiveOptions.__init__`` parameters (and not
top-level ``STTSettings`` fields like ``model`` / ``language``) are
collected into a ``LiveOptions`` object. ``model`` and ``language``
are routed to the top-level settings fields. Truly unknown keys go
to ``extra``.
"""
lo_params = cls._get_live_options_params()
stt_field_names = {"model", "language"}
kwargs: Dict[str, Any] = {}
lo_kwargs: Dict[str, Any] = {}
extra: Dict[str, Any] = {}
for key, value in settings.items():
canonical = cls._aliases.get(key, key)
if canonical in stt_field_names:
kwargs[canonical] = value
elif canonical in lo_params:
lo_kwargs[canonical] = value
else:
extra[key] = value
if lo_kwargs:
kwargs["live_options"] = LiveOptions(**lo_kwargs)
instance = cls(**kwargs)
instance.extra = extra
return instance
@dataclass
class DeepgramSTTSettings(_DeepgramSTTSettingsBase):
"""Settings for the Deepgram STT service.
See ``_DeepgramSTTSettingsBase`` for full documentation.
"""
pass
class DeepgramSTTService(STTService):
"""Deepgram speech-to-text service.
@@ -63,6 +228,8 @@ class DeepgramSTTService(STTService):
...
"""
_settings: DeepgramSTTSettings
def __init__(
self,
*,
@@ -87,7 +254,9 @@ class DeepgramSTTService(STTService):
base_url: Custom Deepgram API base URL.
sample_rate: Audio sample rate. If None, uses default or live_options value.
live_options: Deepgram LiveOptions for detailed configuration.
live_options: Deepgram LiveOptions configuration. Treated as a
delta from a set of sensible defaults — only the fields you
set are overridden; all others keep their default values.
addons: Additional Deepgram features to enable.
should_interrupt: Determine whether the bot should be interrupted when Deepgram VAD events are enabled and the system detects that the user is speaking.
@@ -102,7 +271,6 @@ class DeepgramSTTService(STTService):
The `vad_events` option in LiveOptions is deprecated as of version 0.0.99 and will be removed in a future version. Please use the Silero VAD instead.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if url:
import warnings
@@ -127,24 +295,25 @@ class DeepgramSTTService(STTService):
vad_events=False,
)
merged_options = default_options.to_dict()
settings = DeepgramSTTSettings(
model=default_options.model,
language=default_options.language,
live_options=default_options,
)
if live_options:
default_model = default_options.model
merged_options.update(live_options.to_dict())
# NOTE(aleix): Fixes an in deepgram-sdk where `model` is initialized
# to the string "None" instead of the value `None`.
if "model" in merged_options and merged_options["model"] == "None":
merged_options["model"] = default_model
settings._merge_live_options_delta(live_options)
if "language" in merged_options and isinstance(merged_options["language"], Language):
merged_options["language"] = merged_options["language"].value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
self.set_model_name(merged_options["model"])
self._settings = merged_options
self._addons = addons
self._should_interrupt = should_interrupt
if merged_options.get("vad_events"):
if self._settings.live_options.vad_events:
import warnings
with warnings.catch_warnings():
@@ -175,7 +344,7 @@ class DeepgramSTTService(STTService):
Returns:
True if VAD events are enabled in the current settings.
"""
return self._settings["vad_events"]
return self._settings.live_options.vad_events
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -185,28 +354,17 @@ class DeepgramSTTService(STTService):
"""
return True
async def set_model(self, model: str):
"""Set the Deepgram model and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if anything changed."""
changed = await super()._update_settings(delta)
if not changed:
return changed
Args:
model: The Deepgram model name to use.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._settings["model"] = model
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
Args:
language: The language to use for speech recognition.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram STT service.
@@ -215,7 +373,6 @@ class DeepgramSTTService(STTService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -268,7 +425,11 @@ class DeepgramSTTService(STTService):
self._on_utterance_end,
)
if not await self._connection.start(options=self._settings, addons=self._addons):
live_options = LiveOptions(
**{**self._settings.live_options.to_dict(), "sample_rate": self.sample_rate}
)
if not await self._connection.start(options=live_options, addons=self._addons):
await self.push_error(error_msg=f"Unable to connect to Deepgram")
else:
headers = {
@@ -310,7 +471,7 @@ class DeepgramSTTService(STTService):
await self._call_event_handler("on_speech_started", *args, **kwargs)
await self.broadcast_frame(UserStartedSpeakingFrame)
if self._should_interrupt:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
async def _on_utterance_end(self, *args, **kwargs):
await self._call_event_handler("on_utterance_end", *args, **kwargs)

99
src/pipecat/services/deepgram/stt_sagemaker.py
View File

@@ -14,7 +14,8 @@ languages, and various Deepgram features.
import asyncio
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Dict, Optional
from loguru import logger
@@ -31,6 +32,8 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
from pipecat.services.deepgram.stt import _DeepgramSTTSettingsBase
from pipecat.services.settings import STTSettings
from pipecat.services.stt_latency import DEEPGRAM_SAGEMAKER_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -47,6 +50,16 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class DeepgramSageMakerSTTSettings(_DeepgramSTTSettingsBase):
"""Settings for the Deepgram SageMaker STT service.
See ``_DeepgramSTTSettingsBase`` for full documentation.
"""
pass
class DeepgramSageMakerSTTService(STTService):
"""Deepgram speech-to-text service for AWS SageMaker.
@@ -75,6 +88,8 @@ class DeepgramSageMakerSTTService(STTService):
)
"""
_settings: DeepgramSageMakerSTTSettings
def __init__(
self,
*,
@@ -93,19 +108,15 @@ class DeepgramSageMakerSTTService(STTService):
region: AWS region where the endpoint is deployed (e.g., "us-east-2").
sample_rate: Audio sample rate in Hz. If None, uses value from
live_options or defaults to the value from StartFrame.
live_options: Deepgram LiveOptions for detailed configuration. If None,
uses sensible defaults (nova-3 model, English, interim results enabled).
live_options: Deepgram LiveOptions configuration. Treated as a
delta from a set of sensible defaults — only the fields you
set are overridden; all others keep their default values.
ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._endpoint_name = endpoint_name
self._region = region
# Create default options similar to DeepgramSTTService
default_options = LiveOptions(
encoding="linear16",
language=Language.EN,
@@ -115,21 +126,23 @@ class DeepgramSageMakerSTTService(STTService):
punctuate=True,
)
# Merge with provided options
merged_options = default_options.to_dict()
settings = DeepgramSageMakerSTTSettings(
model=default_options.model,
language=default_options.language,
live_options=default_options,
)
if live_options:
default_model = default_options.model
merged_options.update(live_options.to_dict())
# Handle the "None" string bug from deepgram-sdk
if "model" in merged_options and merged_options["model"] == "None":
merged_options["model"] = default_model
settings._merge_live_options_delta(live_options)
# Convert Language enum to string if needed
if "language" in merged_options and isinstance(merged_options["language"], Language):
merged_options["language"] = merged_options["language"].value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
self.set_model_name(merged_options["model"])
self._settings = merged_options
self._endpoint_name = endpoint_name
self._region = region
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None
@@ -143,35 +156,21 @@ class DeepgramSageMakerSTTService(STTService):
"""
return True
async def set_model(self, model: str):
"""Set the Deepgram model and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and warn about unhandled changes."""
changed = await super()._update_settings(delta)
Disconnects from the current session, updates the model setting, and
establishes a new connection with the updated model.
if not changed:
return changed
Args:
model: The Deepgram model name to use (e.g., "nova-3").
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._settings["model"] = model
await self._disconnect()
await self._connect()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
self._warn_unhandled_updated_settings(changed)
Disconnects from the current session, updates the language setting, and
establishes a new connection with the updated language.
Args:
language: The language to use for speech recognition (e.g., Language.EN,
Language.ES).
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram SageMaker STT service.
@@ -180,7 +179,6 @@ class DeepgramSageMakerSTTService(STTService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -226,12 +224,13 @@ class DeepgramSageMakerSTTService(STTService):
"""
logger.debug("Connecting to Deepgram on SageMaker...")
# Update sample rate in settings
self._settings["sample_rate"] = self.sample_rate
live_options = LiveOptions(
**{**self._settings.live_options.to_dict(), "sample_rate": self.sample_rate}
)
# Build query string from settings, converting booleans to strings
# Build query string from live_options, converting booleans to strings
query_params = {}
for key, value in self._settings.items():
for key, value in live_options.to_dict().items():
if value is not None:
# Convert boolean values to lowercase strings for Deepgram API
if isinstance(value, bool):

74
src/pipecat/services/deepgram/tts.py
View File

@@ -11,7 +11,8 @@ for generating speech from text using various voice models.
"""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -29,6 +30,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService, WebsocketTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -43,6 +45,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class DeepgramTTSSettings(TTSSettings):
"""Settings for Deepgram TTS service.
Parameters:
encoding: Audio encoding format (linear16, mulaw, alaw).
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramTTSService(WebsocketTTSService):
"""Deepgram WebSocket-based text-to-speech service.
@@ -51,6 +64,8 @@ class DeepgramTTSService(WebsocketTTSService):
message for conversational AI use cases.
"""
_settings: DeepgramTTSSettings
SUPPORTED_ENCODINGS = ("linear16", "mulaw", "alaw")
def __init__(
@@ -86,15 +101,17 @@ class DeepgramTTSService(WebsocketTTSService):
pause_frame_processing=True,
push_stop_frames=True,
append_trailing_space=True,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._settings = {
"encoding": encoding,
}
self.set_voice(voice)
self._receive_task = None
self._context_id: Optional[str] = None
@@ -166,6 +183,28 @@ class DeepgramTTSService(WebsocketTTSService):
await self._disconnect_websocket()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Args:
delta: A :class:`TTSSettings` (or ``DeepgramTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
# Deepgram uses voice as the model, so keep them in sync for metrics
if "voice" in changed:
self._settings.model = self._settings.voice
self._sync_model_name_to_metrics()
if changed:
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to Deepgram WebSocket API with configured settings."""
try:
@@ -176,8 +215,8 @@ class DeepgramTTSService(WebsocketTTSService):
# Build WebSocket URL with query parameters
params = []
params.append(f"model={self._voice_id}")
params.append(f"encoding={self._settings['encoding']}")
params.append(f"model={self._settings.voice}")
params.append(f"encoding={self._settings.encoding}")
params.append(f"sample_rate={self.sample_rate}")
url = f"{self._base_url}/v1/speak?{'&'.join(params)}"
@@ -330,6 +369,8 @@ class DeepgramHttpTTSService(TTSService):
configurable sample rates and quality settings.
"""
_settings: DeepgramTTSSettings
def __init__(
self,
*,
@@ -352,15 +393,20 @@ class DeepgramHttpTTSService(TTSService):
encoding: Audio encoding format. Defaults to "linear16".
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
super().__init__(
sample_rate=sample_rate,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = base_url
self._settings = {
"encoding": encoding,
}
self.set_voice(voice)
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
@@ -389,8 +435,8 @@ class DeepgramHttpTTSService(TTSService):
headers = {"Authorization": f"Token {self._api_key}", "Content-Type": "application/json"}
params = {
"model": self._voice_id,
"encoding": self._settings["encoding"],
"model": self._settings.voice,
"encoding": self._settings.encoding,
"sample_rate": self.sample_rate,
"container": "none",
}

53
src/pipecat/services/deepgram/tts_sagemaker.py
View File

@@ -14,7 +14,8 @@ streaming audio output.
import asyncio
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -32,10 +33,22 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@dataclass
class DeepgramSageMakerTTSSettings(TTSSettings):
"""Settings for Deepgram SageMaker TTS service.
Parameters:
encoding: Audio encoding format (e.g. "linear16").
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramSageMakerTTSService(TTSService):
"""Deepgram text-to-speech service for AWS SageMaker.
@@ -58,6 +71,8 @@ class DeepgramSageMakerTTSService(TTSService):
)
"""
_settings: DeepgramSageMakerTTSSettings
def __init__(
self,
*,
@@ -84,13 +99,17 @@ class DeepgramSageMakerTTSService(TTSService):
push_stop_frames=True,
pause_frame_processing=True,
append_trailing_space=True,
settings=DeepgramSageMakerTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._endpoint_name = endpoint_name
self._region = region
self._encoding = encoding
self.set_voice(voice)
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None
@@ -156,7 +175,8 @@ class DeepgramSageMakerTTSService(TTSService):
logger.debug("Connecting to Deepgram TTS on SageMaker...")
query_string = (
f"model={self._voice_id}&encoding={self._encoding}&sample_rate={self.sample_rate}"
f"model={self._settings.voice}&encoding={self._settings.encoding}"
f"&sample_rate={self.sample_rate}"
)
self._client = SageMakerBidiClient(
@@ -200,6 +220,31 @@ class DeepgramSageMakerTTSService(TTSService):
logger.debug("Disconnected from Deepgram TTS on SageMaker")
await self._call_event_handler("on_disconnected")
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if necessary.
Since all settings are part of the SageMaker session query string,
any setting change requires reconnecting to apply the new values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# Deepgram uses voice as the model, so keep them in sync for metrics
if "voice" in changed:
self._settings.model = self._settings.voice
self._sync_model_name_to_metrics()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def _process_responses(self):
"""Process streaming responses from Deepgram TTS on SageMaker.

14
src/pipecat/services/deepseek/llm.py
View File

@@ -65,18 +65,18 @@ class DeepSeekLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

242
src/pipecat/services/elevenlabs/stt.py
View File

@@ -11,11 +11,13 @@ using segmented audio processing. The service uploads audio files and receives
transcription results directly.
"""
import asyncio
import base64
import io
import json
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, Optional
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -33,6 +35,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import ELEVENLABS_REALTIME_TTFS_P99, ELEVENLABS_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -167,6 +170,51 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
class CommitStrategy(str, Enum):
"""Commit strategies for transcript segmentation."""
MANUAL = "manual"
VAD = "vad"
@dataclass
class ElevenLabsSTTSettings(STTSettings):
"""Settings for the ElevenLabs file-based STT service.
Parameters:
tag_audio_events: Whether to include audio event tags in transcription.
"""
tag_audio_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class ElevenLabsRealtimeSTTSettings(STTSettings):
"""Settings for the ElevenLabs Realtime STT service.
See ``ElevenLabsRealtimeSTTService.InputParams`` for detailed descriptions.
Parameters:
commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD).
vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0).
vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive).
min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms).
min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms).
include_timestamps: Whether to include word-level timestamps in transcripts.
enable_logging: Whether to enable logging on ElevenLabs' side.
include_language_detection: Whether to include language detection in transcripts.
"""
commit_strategy: CommitStrategy | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_silence_threshold_secs: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_speech_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_silence_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_timestamps: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_logging: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_language_detection: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class ElevenLabsSTTService(SegmentedSTTService):
"""Speech-to-text service using ElevenLabs' file-based API.
@@ -175,6 +223,8 @@ class ElevenLabsSTTService(SegmentedSTTService):
The service uploads audio files to ElevenLabs and receives transcription results directly.
"""
_settings: ElevenLabsSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for ElevenLabs STT API.
@@ -211,25 +261,24 @@ class ElevenLabsSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
params = params or ElevenLabsSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=ElevenLabsSTTSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
tag_audio_events=params.tag_audio_events,
),
**kwargs,
)
params = params or ElevenLabsSTTService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
self._model_id = model
self._tag_audio_events = params.tag_audio_events
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "eng",
}
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -250,28 +299,6 @@ class ElevenLabsSTTService(SegmentedSTTService):
"""
return language_to_elevenlabs_language(language)
async def set_language(self, language: Language):
"""Set the transcription language.
Args:
language: The language to use for speech-to-text transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = self.language_to_service_language(language)
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
Note:
ElevenLabs STT API does not currently support model selection.
This method is provided for interface compatibility.
"""
await super().set_model(model)
logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model")
async def _transcribe_audio(self, audio_data: bytes) -> dict:
"""Upload audio data to ElevenLabs and get transcription result.
@@ -297,9 +324,9 @@ class ElevenLabsSTTService(SegmentedSTTService):
)
# Add required model_id, language_code, and tag_audio_events
data.add_field("model_id", self._model_id)
data.add_field("language_code", self._settings["language"])
data.add_field("tag_audio_events", str(self._tag_audio_events).lower())
data.add_field("model_id", self._settings.model)
data.add_field("language_code", self._settings.language)
data.add_field("tag_audio_events", str(self._settings.tag_audio_events).lower())
async with self._session.post(url, data=data, headers=headers) as response:
if response.status != 200:
@@ -385,13 +412,6 @@ def audio_format_from_sample_rate(sample_rate: int) -> str:
return "pcm_16000"
class CommitStrategy(str, Enum):
"""Commit strategies for transcript segmentation."""
MANUAL = "manual"
VAD = "vad"
class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""Speech-to-text service using ElevenLabs' Realtime WebSocket API.
@@ -404,6 +424,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
commit transcript segments, providing consistency with other STT services.
"""
_settings: ElevenLabsRealtimeSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for ElevenLabs Realtime STT API.
@@ -456,24 +478,35 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to WebsocketSTTService.
"""
params = params or ElevenLabsRealtimeSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=10,
keepalive_interval=5,
settings=ElevenLabsRealtimeSTTSettings(
model=model,
language=params.language_code,
commit_strategy=params.commit_strategy,
vad_silence_threshold_secs=params.vad_silence_threshold_secs,
vad_threshold=params.vad_threshold,
min_speech_duration_ms=params.min_speech_duration_ms,
min_silence_duration_ms=params.min_silence_duration_ms,
include_timestamps=params.include_timestamps,
enable_logging=params.enable_logging,
include_language_detection=params.include_language_detection,
),
**kwargs,
)
params = params or ElevenLabsRealtimeSTTService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._model_id = model
self._params = params
self._audio_format = "" # initialized in start()
self._receive_task = None
self._settings = {"language": params.language_code}
self._connected_event = asyncio.Event()
self._connected_event.set()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -483,42 +516,24 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the transcription language.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if anything changed.
Args:
language: The language to use for speech-to-text transcription.
delta: A :class:`STTSettings` (or ``ElevenLabsRealtimeSTTSettings``) delta.
Note:
Changing language requires reconnecting to the WebSocket.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.info(f"Switching STT language to: [{language}]")
new_language = (
language_to_elevenlabs_language(language)
if isinstance(language, Language)
else language
)
self._params.language_code = new_language
self._settings["language"] = new_language
# Reconnect with new settings
changed = await super()._update_settings(delta)
if not changed:
return changed
await self._disconnect()
await self._connect()
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
Note:
Changing model requires reconnecting to the WebSocket.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._model_id = model
# Reconnect with new settings
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the STT service and establish WebSocket connection.
@@ -566,7 +581,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
# Send commit when user stops speaking (manual commit mode)
if self._params.commit_strategy == CommitStrategy.MANUAL:
if self._settings.commit_strategy == CommitStrategy.MANUAL:
if self._websocket and self._websocket.state is State.OPEN:
try:
commit_message = {
@@ -589,6 +604,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Yields:
None - transcription results are handled via WebSocket responses.
"""
# Wait for any in-flight _connect() to finish before checking state
await self._connected_event.wait()
# Reconnect if connection is closed
if not self._websocket or self._websocket.state is State.CLOSED:
await self._connect()
@@ -613,12 +631,18 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
async def _connect(self):
"""Establish WebSocket connection to ElevenLabs Realtime STT."""
await self._connect_websocket()
self._connected_event.clear()
try:
await self._connect_websocket()
await super()._connect()
await super()._connect()
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(
self._receive_task_handler(self._report_error)
)
finally:
self._connected_event.set()
async def _disconnect(self):
"""Close WebSocket connection and cleanup tasks."""
@@ -654,38 +678,42 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
logger.debug("Connecting to ElevenLabs Realtime STT")
# Build query parameters
params = [f"model_id={self._model_id}"]
params = [f"model_id={self._settings.model}"]
if self._params.language_code:
params.append(f"language_code={self._params.language_code}")
if self._settings.language:
params.append(f"language_code={self._settings.language}")
params.append(f"audio_format={self._audio_format}")
params.append(f"commit_strategy={self._params.commit_strategy.value}")
params.append(f"commit_strategy={self._settings.commit_strategy.value}")
# Add optional parameters
if self._params.include_timestamps:
params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}")
if self._params.enable_logging:
params.append(f"enable_logging={str(self._params.enable_logging).lower()}")
if self._params.include_language_detection:
if self._settings.include_timestamps:
params.append(
f"include_language_detection={str(self._params.include_language_detection).lower()}"
f"include_timestamps={str(self._settings.include_timestamps).lower()}"
)
if self._settings.enable_logging:
params.append(f"enable_logging={str(self._settings.enable_logging).lower()}")
if self._settings.include_language_detection:
params.append(
f"include_language_detection={str(self._settings.include_language_detection).lower()}"
)
# Add VAD parameters if using VAD commit strategy and values are specified
if self._params.commit_strategy == CommitStrategy.VAD:
if self._params.vad_silence_threshold_secs is not None:
if self._settings.commit_strategy == CommitStrategy.VAD:
if self._settings.vad_silence_threshold_secs is not None:
params.append(
f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}"
f"vad_silence_threshold_secs={self._settings.vad_silence_threshold_secs}"
)
if self._settings.vad_threshold is not None:
params.append(f"vad_threshold={self._settings.vad_threshold}")
if self._settings.min_speech_duration_ms is not None:
params.append(f"min_speech_duration_ms={self._settings.min_speech_duration_ms}")
if self._settings.min_silence_duration_ms is not None:
params.append(
f"min_silence_duration_ms={self._settings.min_silence_duration_ms}"
)
if self._params.vad_threshold is not None:
params.append(f"vad_threshold={self._params.vad_threshold}")
if self._params.min_speech_duration_ms is not None:
params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}")
if self._params.min_silence_duration_ms is not None:
params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}")
ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}"
@@ -817,7 +845,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""
# If timestamps are enabled, skip this message and wait for the
# committed_transcript_with_timestamps message which contains all the data
if self._params.include_timestamps:
if self._settings.include_timestamps:
return
text = data.get("text", "").strip()
@@ -833,6 +861,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await self._handle_transcription(text, True, language)
finalized = self._settings.commit_strategy == CommitStrategy.MANUAL
await self.push_frame(
TranscriptionFrame(
text,
@@ -840,6 +870,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
time_now_iso8601(),
language,
result=data,
finalized=finalized,
)
)
@@ -874,6 +905,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await self._handle_transcription(text, True, language)
finalized = self._settings.commit_strategy == CommitStrategy.MANUAL
# This message is sent after committed_transcript when include_timestamps=true.
# It contains the full transcript data including text and word-level timestamps.
await self.push_frame(
@@ -883,5 +916,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
time_now_iso8601(),
language,
result=data,
finalized=finalized,
)
)

370
src/pipecat/services/elevenlabs/tts.py
View File

@@ -13,7 +13,19 @@ with support for streaming audio, word timestamps, and voice customization.
import asyncio
import base64
import json
from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union
from dataclasses import dataclass, field
from typing import (
Any,
AsyncGenerator,
ClassVar,
Dict,
List,
Literal,
Mapping,
Optional,
Tuple,
Union,
)
import aiohttp
from loguru import logger
@@ -32,9 +44,11 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import (
AudioContextWordTTSService,
WordTTSService,
AudioContextTTSService,
TextAggregationMode,
TTSService,
)
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -136,12 +150,12 @@ def output_format_from_sample_rate(sample_rate: int) -> str:
def build_elevenlabs_voice_settings(
settings: Dict[str, Any],
settings: Union[Dict[str, Any], "TTSSettings"],
) -> Optional[Dict[str, Union[float, bool]]]:
"""Build voice settings dictionary for ElevenLabs based on provided settings.
Args:
settings: Dictionary containing voice settings parameters.
settings: Dictionary or settings containing voice settings parameters.
Returns:
Dictionary of voice settings or None if no valid settings are provided.
@@ -150,8 +164,11 @@ def build_elevenlabs_voice_settings(
voice_settings = {}
for key in voice_setting_keys:
if key in settings and settings[key] is not None:
voice_settings[key] = settings[key]
val = (
getattr(settings, key, None) if isinstance(settings, TTSSettings) else settings.get(key)
)
if val is not None:
voice_settings[key] = val
return voice_settings or None
@@ -168,6 +185,79 @@ class PronunciationDictionaryLocator(BaseModel):
version_id: str
@dataclass
class ElevenLabsTTSSettings(TTSSettings):
"""Settings for the ElevenLabs WebSocket TTS service.
Fields that appear in the WebSocket URL (``voice``, ``model``,
``language``) require a full reconnect when changed. Fields that
affect the voice character (``stability``, ``similarity_boost``,
``style``, ``use_speaker_boost``, ``speed``) can be applied by closing
the current audio context so a new one is opened with updated settings.
Parameters:
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.7 to 1.2).
auto_mode: Whether to enable automatic mode optimization.
enable_ssml_parsing: Whether to parse SSML tags in text.
enable_logging: Whether to enable ElevenLabs logging.
apply_text_normalization: Text normalization mode ("auto", "on", "off").
"""
stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
auto_mode: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_ssml_parsing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_logging: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
#: Fields in the WS URL — changing any of these requires a reconnect.
URL_FIELDS: ClassVar[frozenset[str]] = frozenset({"voice", "model", "language"})
#: Fields affecting voice character — changing these requires closing the
#: current audio context so the next one picks up new settings.
VOICE_SETTINGS_FIELDS: ClassVar[frozenset[str]] = frozenset(
{"stability", "similarity_boost", "style", "use_speaker_boost", "speed"}
)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@dataclass
class ElevenLabsHttpTTSSettings(TTSSettings):
"""Settings for the ElevenLabs HTTP TTS service.
Parameters:
optimize_streaming_latency: Latency optimization level (0-4).
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.25 to 4.0).
apply_text_normalization: Text normalization mode ("auto", "on", "off").
"""
optimize_streaming_latency: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
def calculate_word_times(
alignment_info: Mapping[str, Any],
cumulative_time: float,
@@ -228,7 +318,7 @@ def calculate_word_times(
return (word_times, new_partial_word, new_partial_word_start_time)
class ElevenLabsTTSService(AudioContextWordTTSService):
class ElevenLabsTTSService(AudioContextTTSService):
"""ElevenLabs WebSocket-based TTS service with word timestamps.
Provides real-time text-to-speech using ElevenLabs' WebSocket streaming API.
@@ -236,6 +326,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
customization options including stability, similarity boost, and speed controls.
"""
_settings: ElevenLabsTTSSettings
class InputParams(BaseModel):
"""Input parameters for ElevenLabs TTS configuration.
@@ -274,7 +366,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
url: str = "wss://api.elevenlabs.io",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize the ElevenLabs TTS service.
@@ -286,13 +379,20 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
url: WebSocket URL for ElevenLabs TTS API.
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to the parent service.
"""
# Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
# 3 model, and it's worth it for the better audio quality.
# By default, we aggregate sentences before sending to TTS. This adds
# ~200-300ms of latency per sentence (waiting for the sentence-ending
# punctuation token from the LLM). Setting
# text_aggregation_mode=TextAggregationMode.TOKEN streams tokens
# directly. To use this mode, you must set auto_mode=False. This
# eliminates aggregation time, but slows down ElevenLabs.
#
# We also don't want to automatically push LLM response text frames,
# because the context aggregators will add them to the LLM context even
@@ -303,35 +403,38 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
# Finally, ElevenLabs doesn't provide information on when the bot stops
# speaking for a while, so we want the parent class to send TTSStopFrame
# after a short period not receiving any audio.
params = params or ElevenLabsTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=ElevenLabsTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else None
),
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
auto_mode=str(params.auto_mode).lower(),
enable_ssml_parsing=params.enable_ssml_parsing,
enable_logging=params.enable_logging,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs,
)
params = params or ElevenLabsTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else None,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
"speed": params.speed,
"auto_mode": str(params.auto_mode).lower(),
"enable_ssml_parsing": params.enable_ssml_parsing,
"enable_logging": params.enable_logging,
"apply_text_normalization": params.apply_text_normalization,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
@@ -365,54 +468,57 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
return language_to_elevenlabs_language(language)
def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings)
ts = self._settings
voice_setting_keys = [
"stability",
"similarity_boost",
"style",
"use_speaker_boost",
"speed",
]
voice_settings = {}
for key in voice_setting_keys:
val = getattr(ts, key, None)
if val is not None:
voice_settings[key] = val
return voice_settings or None
async def set_model(self, model: str):
"""Set the TTS model and reconnect.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta, reconnecting as needed.
Uses the declarative ``URL_FIELDS`` and ``VOICE_SETTINGS_FIELDS``
sets on :class:`ElevenLabsTTSSettings` to decide whether to
reconnect the WebSocket or close the current audio context.
Args:
model: The model name to use for synthesis.
delta: A :class:`TTSSettings` (or ``ElevenLabsTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
changed = await super()._update_settings(delta)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice, model, or language changed."""
# Track previous values for settings that require reconnection
prev_voice = self._voice_id
prev_model = self.model_name
prev_language = self._settings.get("language")
# Create snapshot of current voice settings to detect changes after update
prev_voice_settings = self._voice_settings.copy() if self._voice_settings else None
if not changed:
return changed
await super()._update_settings(settings)
# Update voice settings for the next context creation
# Rebuild voice settings for next context
self._voice_settings = self._set_voice_settings()
# Check if URL-level settings changed (these require reconnection)
url_changed = (
prev_voice != self._voice_id
or prev_model != self.model_name
or prev_language != self._settings.get("language")
)
# Check if only voice settings changed (speed, stability, etc.)
voice_settings_changed = prev_voice_settings != self._voice_settings
url_changed = bool(changed.keys() & ElevenLabsTTSSettings.URL_FIELDS)
voice_settings_changed = bool(changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS)
if url_changed:
# These settings are in the WebSocket URL, so we need to reconnect
logger.debug(
f"URL-level setting changed (voice/model/language), reconnecting WebSocket"
f"URL-level setting changed ({changed.keys() & ElevenLabsTTSSettings.URL_FIELDS}), "
f"reconnecting WebSocket"
)
await self._disconnect()
await self._connect()
elif voice_settings_changed and self.has_active_audio_context():
# Voice settings can be updated by closing current context
# so new one gets created with updated voice settings
logger.debug(f"Voice settings changed, closing current context to apply changes")
logger.debug(
f"Voice settings changed ({changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS}), "
f"closing current context to apply changes"
)
context_id = self.get_active_audio_context_id()
try:
if self._websocket:
@@ -423,6 +529,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
self.reset_active_audio_context()
if not url_changed:
# Reconnect applies all settings; only warn about fields not handled
# by voice settings or URL changes.
handled = ElevenLabsTTSSettings.URL_FIELDS | ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS
self._warn_unhandled_updated_settings(changed.keys() - handled)
return changed
async def start(self, frame: StartFrame):
"""Start the ElevenLabs TTS service.
@@ -503,22 +617,22 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
logger.debug("Connecting to ElevenLabs")
voice_id = self._voice_id
model = self.model_name
voice_id = self._settings.voice
model = self._settings.model
output_format = self._output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings.auto_mode}"
if self._settings["enable_ssml_parsing"]:
url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}"
if self._settings.enable_ssml_parsing:
url += f"&enable_ssml_parsing={self._settings.enable_ssml_parsing}"
if self._settings["enable_logging"]:
url += f"&enable_logging={self._settings['enable_logging']}"
if self._settings.enable_logging:
url += f"&enable_logging={self._settings.enable_logging}"
if self._settings["apply_text_normalization"] is not None:
url += f"&apply_text_normalization={self._settings['apply_text_normalization']}"
if self._settings.apply_text_normalization is not None:
url += f"&apply_text_normalization={self._settings.apply_text_normalization}"
# Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS
language = self._settings["language"]
language = self._settings.language
if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None:
url += f"&language_code={language}"
logger.debug(f"Using language code: {language}")
@@ -561,14 +675,11 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by closing the current context."""
# Close the current context when interrupted without closing the websocket
context_id = self.get_active_audio_context_id()
await super()._handle_interruption(frame, direction)
async def _close_context(self, context_id: str):
# ElevenLabs requires that Pipecat explicitly closes contexts to free
# server-side resources, both on interruption and on normal completion.
if context_id and self._websocket:
logger.trace(f"Closing context {context_id} due to interruption")
logger.trace(f"{self}: Closing context {context_id}")
try:
# ElevenLabs requires that Pipecat manages the contexts and closes them
# when they're not longer in use. Since an InterruptionFrame is pushed
@@ -581,8 +692,21 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
)
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
self._partial_word = ""
self._partial_word_start_time = 0.0
self._partial_word = ""
self._partial_word_start_time = 0.0
async def on_audio_context_interrupted(self, context_id: str):
"""Close the ElevenLabs context when the bot is interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Close the ElevenLabs context after all audio has been played.
ElevenLabs does not send a server-side signal when a context is
exhausted, so Pipecat must explicitly close it with
``close_context: True`` to free server-side resources.
"""
await self._close_context(context_id)
async def _receive_messages(self):
"""Handle incoming WebSocket messages from ElevenLabs."""
@@ -734,7 +858,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
yield ErrorFrame(error=f"Unknown error occurred: {e}")
class ElevenLabsHttpTTSService(WordTTSService):
class ElevenLabsHttpTTSService(TTSService):
"""ElevenLabs HTTP-based TTS service with word timestamps.
Provides text-to-speech using ElevenLabs' HTTP streaming API for simpler,
@@ -742,6 +866,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
connection is not required or desired.
"""
_settings: ElevenLabsHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for ElevenLabs HTTP TTS configuration.
@@ -777,7 +903,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
base_url: str = "https://api.elevenlabs.io",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize the ElevenLabs HTTP TTS service.
@@ -790,38 +917,44 @@ class ElevenLabsHttpTTSService(WordTTSService):
base_url: Base URL for ElevenLabs HTTP API.
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to the parent service.
"""
params = params or ElevenLabsHttpTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=ElevenLabsHttpTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(params.language)
if params.language
else None,
optimize_streaming_latency=params.optimize_streaming_latency,
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs,
)
params = params or ElevenLabsHttpTTSService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._params = params
self._session = aiohttp_session
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
"speed": params.speed,
"apply_text_normalization": params.apply_text_normalization,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
@@ -858,10 +991,19 @@ class ElevenLabsHttpTTSService(WordTTSService):
def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings)
async def _update_settings(self, settings: Mapping[str, Any]):
await super()._update_settings(settings)
# Update voice settings for the next context creation
self._voice_settings = self._set_voice_settings()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and rebuild voice settings.
Args:
delta: A :class:`TTSSettings` (or ``ElevenLabsHttpTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if changed:
self._voice_settings = self._set_voice_settings()
return changed
def _reset_state(self):
"""Reset internal state variables."""
@@ -979,11 +1121,11 @@ class ElevenLabsHttpTTSService(WordTTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
# Use the with-timestamps endpoint
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream/with-timestamps"
url = f"{self._base_url}/v1/text-to-speech/{self._settings.voice}/stream/with-timestamps"
payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = {
"text": text,
"model_id": self._model_name,
"model_id": self._settings.model,
}
# Include previous text as context if available
@@ -998,11 +1140,11 @@ class ElevenLabsHttpTTSService(WordTTSService):
locator.model_dump() for locator in self._pronunciation_dictionary_locators
]
if self._settings["apply_text_normalization"] is not None:
payload["apply_text_normalization"] = self._settings["apply_text_normalization"]
if self._settings.apply_text_normalization is not None:
payload["apply_text_normalization"] = self._settings.apply_text_normalization
language = self._settings["language"]
if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language:
language = self._settings.language
if self._settings.model in ELEVENLABS_MULTILINGUAL_MODELS and language:
payload["language_code"] = language
logger.debug(f"Using language code: {language}")
elif language:
@@ -1019,8 +1161,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
params = {
"output_format": self._output_format,
}
if self._settings["optimize_streaming_latency"] is not None:
params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"]
if self._settings.optimize_streaming_latency is not None:
params["optimize_streaming_latency"] = self._settings.optimize_streaming_latency
try:
await self.start_ttfb_metrics()

16
src/pipecat/services/fal/image.py
View File

@@ -13,6 +13,7 @@ for creating images from text prompts using various AI models.
import asyncio
import io
import os
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Optional, Union
import aiohttp
@@ -22,6 +23,7 @@ from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try:
import fal_client
@@ -31,6 +33,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class FalImageGenSettings(ImageGenSettings):
"""Settings for the Fal image generation service.
Parameters:
model: Fal.ai model identifier.
"""
class FalImageGenService(ImageGenService):
"""Fal's image generation service.
@@ -77,8 +88,7 @@ class FalImageGenService(ImageGenService):
key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable.
**kwargs: Additional arguments passed to parent ImageGenService.
"""
super().__init__(**kwargs)
self.set_model_name(model)
super().__init__(settings=FalImageGenSettings(model=model), **kwargs)
self._params = params
self._aiohttp_session = aiohttp_session
if key:
@@ -103,7 +113,7 @@ class FalImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
response = await fal_client.run_async(
self.model_name,
self._settings.model,
arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)},
)

67
src/pipecat/services/fal/stt.py
View File

@@ -11,12 +11,14 @@ transcription using segmented audio processing.
"""
import os
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import FAL_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -146,6 +148,22 @@ def language_to_fal_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class FalSTTSettings(STTSettings):
"""Settings for the Fal Wizper STT service.
Parameters:
task: Task to perform ('transcribe' or 'translate'). Defaults to
'transcribe'.
chunk_level: Level of chunking ('segment'). Defaults to 'segment'.
version: Version of Wizper model to use. Defaults to '3'.
"""
task: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
chunk_level: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
version: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class FalSTTService(SegmentedSTTService):
"""Speech-to-text service using Fal's Wizper API.
@@ -153,6 +171,8 @@ class FalSTTService(SegmentedSTTService):
segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
"""
_settings: FalSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Fal's Wizper API.
@@ -187,14 +207,23 @@ class FalSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
params = params or FalSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=FalSTTSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en",
task=params.task,
chunk_level=params.chunk_level,
version=params.version,
),
**kwargs,
)
params = params or FalSTTService.InputParams()
if api_key:
os.environ["FAL_KEY"] = api_key
elif "FAL_KEY" not in os.environ:
@@ -203,14 +232,6 @@ class FalSTTService(SegmentedSTTService):
)
self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY"))
self._settings = {
"task": params.task,
"language": self.language_to_service_language(params.language)
if params.language
else "en",
"chunk_level": params.chunk_level,
"version": params.version,
}
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -231,24 +252,6 @@ class FalSTTService(SegmentedSTTService):
"""
return language_to_fal_language(language)
async def set_language(self, language: Language):
"""Set the transcription language.
Args:
language: The language to use for speech-to-text transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = self.language_to_service_language(language)
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[str] = None
@@ -276,19 +279,19 @@ class FalSTTService(SegmentedSTTService):
data_uri = fal_client.encode(audio, "audio/x-wav")
response = await self._fal_client.run(
"fal-ai/wizper",
arguments={"audio_url": data_uri, **self._settings},
arguments={"audio_url": data_uri, **self._settings.given_fields()},
)
if response and "text" in response:
text = response["text"].strip()
if text: # Only yield non-empty text
await self._handle_transcription(text, True, self._settings["language"])
await self._handle_transcription(text, True, self._settings.language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text,
self._user_id,
time_now_iso8601(),
Language(self._settings["language"]),
Language(self._settings.language),
result=response,
)

14
src/pipecat/services/fireworks/llm.py
View File

@@ -66,17 +66,17 @@ class FireworksLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

119
src/pipecat/services/fish/tts.py
View File

@@ -11,7 +11,8 @@ for streaming text-to-speech synthesis with customizable voice parameters.
"""
import uuid
from typing import AsyncGenerator, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Literal, Mapping, Optional
from loguru import logger
from pydantic import BaseModel
@@ -28,6 +29,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -45,6 +47,41 @@ except ModuleNotFoundError as e:
FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"]
@dataclass
class FishAudioTTSSettings(TTSSettings):
"""Settings for Fish Audio TTS service.
Parameters:
fish_sample_rate: Audio sample rate sent to the API.
latency: Latency mode ("normal" or "balanced"). Defaults to "normal".
format: Audio output format.
normalize: Whether to normalize audio output. Defaults to True.
prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0.
prosody_volume: Volume adjustment in dB. Defaults to 0.
reference_id: Reference ID of the voice model.
"""
fish_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
latency: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
normalize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_volume: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
reference_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "fish_sample_rate"}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "FishAudioTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``prosody``."""
flat = dict(settings)
nested = flat.pop("prosody", None)
if isinstance(nested, dict):
flat.setdefault("prosody_speed", nested.get("speed"))
flat.setdefault("prosody_volume", nested.get("volume"))
return super().from_mapping(flat)
class FishAudioTTSService(InterruptibleTTSService):
"""Fish Audio text-to-speech service with WebSocket streaming.
@@ -53,6 +90,8 @@ class FishAudioTTSService(InterruptibleTTSService):
audio generation with interruption handling.
"""
_settings: FishAudioTTSSettings
class InputParams(BaseModel):
"""Input parameters for Fish Audio TTS configuration.
@@ -99,13 +138,6 @@ class FishAudioTTSService(InterruptibleTTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent service.
"""
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or FishAudioTTSService.InputParams()
# Validation for model and reference_id parameters
@@ -130,26 +162,30 @@ class FishAudioTTSService(InterruptibleTTSService):
)
reference_id = model
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
settings=FishAudioTTSSettings(
model=model_id,
voice=reference_id,
fish_sample_rate=0,
latency=params.latency,
format=output_format,
normalize=params.normalize,
prosody_speed=params.prosody_speed,
prosody_volume=params.prosody_volume,
reference_id=reference_id,
),
**kwargs,
)
self._api_key = api_key
self._base_url = "wss://api.fish.audio/v1/tts/live"
self._websocket = None
self._receive_task = None
self._request_id = None
self._settings = {
"sample_rate": 0,
"latency": params.latency,
"format": output_format,
"normalize": params.normalize,
"prosody": {
"speed": params.prosody_speed,
"volume": params.prosody_volume,
},
"reference_id": reference_id,
}
self.set_model_name(model_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -158,16 +194,24 @@ class FishAudioTTSService(InterruptibleTTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model and reconnect.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if needed.
Any change to voice or model triggers a WebSocket reconnect.
Args:
model: The model name to use for synthesis.
delta: A :class:`TTSSettings` (or ``FishAudioTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
changed = await super()._update_settings(delta)
if changed:
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Fish Audio TTS service.
@@ -176,7 +220,7 @@ class FishAudioTTSService(InterruptibleTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.fish_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -221,11 +265,22 @@ class FishAudioTTSService(InterruptibleTTSService):
logger.debug("Connecting to Fish Audio")
headers = {"Authorization": f"Bearer {self._api_key}"}
headers["model"] = self.model_name
headers["model"] = self._settings.model
self._websocket = await websocket_connect(self._base_url, additional_headers=headers)
# Send initial start message with ormsgpack
start_message = {"event": "start", "request": {"text": "", **self._settings}}
request_settings = {
"sample_rate": self._settings.fish_sample_rate,
"latency": self._settings.latency,
"format": self._settings.format,
"normalize": self._settings.normalize,
"prosody": {
"speed": self._settings.prosody_speed,
"volume": self._settings.prosody_volume,
},
"reference_id": self._settings.reference_id,
}
start_message = {"event": "start", "request": {"text": "", **request_settings}}
await self._websocket.send(ormsgpack.packb(start_message))
logger.debug("Sent start message to Fish Audio")

173
src/pipecat/services/gladia/stt.py
View File

@@ -14,6 +14,7 @@ import asyncio
import base64
import json
import warnings
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Literal, Optional
import aiohttp
@@ -31,7 +32,14 @@ from pipecat.frames.frames import (
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.services.gladia.config import GladiaInputParams
from pipecat.services.gladia.config import (
GladiaInputParams,
LanguageConfig,
MessagesConfig,
PreProcessingConfig,
RealtimeProcessingConfig,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GLADIA_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -178,6 +186,43 @@ class _InputParamsDescriptor:
return GladiaInputParams
@dataclass
class GladiaSTTSettings(STTSettings):
"""Settings for Gladia STT service.
Parameters:
encoding: Audio encoding format.
bit_depth: Audio bit depth.
channels: Number of audio channels.
custom_metadata: Additional metadata to include with requests.
endpointing: Silence duration in seconds to mark end of speech.
maximum_duration_without_endpointing: Maximum utterance duration without silence.
language_config: Detailed language configuration.
pre_processing: Audio pre-processing options.
realtime_processing: Real-time processing features.
messages_config: WebSocket message filtering options.
enable_vad: Enable VAD to trigger end of utterance detection.
"""
encoding: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
bit_depth: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
channels: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
custom_metadata: Dict[str, Any] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
endpointing: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
maximum_duration_without_endpointing: int | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
language_config: LanguageConfig | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pre_processing: PreProcessingConfig | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
realtime_processing: RealtimeProcessingConfig | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
messages_config: MessagesConfig | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_vad: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GladiaSTTService(WebsocketSTTService):
"""Speech-to-Text service using Gladia's API.
@@ -191,6 +236,8 @@ class GladiaSTTService(WebsocketSTTService):
Use :class:`~pipecat.services.gladia.config.GladiaInputParams` directly instead.
"""
_settings: GladiaSTTSettings
# Maintain backward compatibility
InputParams = _InputParamsDescriptor()
@@ -231,14 +278,6 @@ class GladiaSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService parent class.
"""
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=20,
keepalive_interval=5,
**kwargs,
)
params = params or GladiaInputParams()
if params.language is not None:
@@ -261,13 +300,40 @@ class GladiaSTTService(WebsocketSTTService):
stacklevel=2,
)
# Resolve deprecated language → language_config at init time
language_config = params.language_config
if not language_config and params.language:
language_code = self.language_to_service_language(params.language)
if language_code:
language_config = LanguageConfig(languages=[language_code], code_switching=False)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=20,
keepalive_interval=5,
settings=GladiaSTTSettings(
model=model,
language=None,
encoding=params.encoding,
bit_depth=params.bit_depth,
channels=params.channels,
custom_metadata=params.custom_metadata,
endpointing=params.endpointing,
maximum_duration_without_endpointing=params.maximum_duration_without_endpointing,
language_config=language_config,
pre_processing=params.pre_processing,
realtime_processing=params.realtime_processing,
messages_config=params.messages_config,
enable_vad=params.enable_vad,
),
**kwargs,
)
self._api_key = api_key
self._region = region
self._url = url
self.set_model_name(model)
self._params = params
self._receive_task = None
self._settings = {}
# Session management
self._session_url = None
@@ -307,53 +373,43 @@ class GladiaSTTService(WebsocketSTTService):
return language_to_gladia_language(language)
def _prepare_settings(self) -> Dict[str, Any]:
s = self._settings
settings = {
"encoding": self._params.encoding or "wav/pcm",
"bit_depth": self._params.bit_depth or 16,
"encoding": s.encoding or "wav/pcm",
"bit_depth": s.bit_depth or 16,
"sample_rate": self.sample_rate,
"channels": self._params.channels or 1,
"model": self._model_name,
"channels": s.channels or 1,
"model": s.model,
}
# Add custom_metadata if provided
settings["custom_metadata"] = dict(self._params.custom_metadata or {})
settings["custom_metadata"] = dict(s.custom_metadata or {})
settings["custom_metadata"]["pipecat"] = pipecat_version()
# Add endpointing parameters if provided
if self._params.endpointing is not None:
settings["endpointing"] = self._params.endpointing
if self._params.maximum_duration_without_endpointing is not None:
if s.endpointing is not None:
settings["endpointing"] = s.endpointing
if s.maximum_duration_without_endpointing is not None:
settings["maximum_duration_without_endpointing"] = (
self._params.maximum_duration_without_endpointing
s.maximum_duration_without_endpointing
)
# Add language configuration (prioritize language_config over deprecated language)
if self._params.language_config:
settings["language_config"] = self._params.language_config.model_dump(exclude_none=True)
elif self._params.language: # Backward compatibility for deprecated parameter
language_code = self.language_to_service_language(self._params.language)
if language_code:
settings["language_config"] = {
"languages": [language_code],
"code_switching": False,
}
# Add language configuration
if s.language_config:
settings["language_config"] = s.language_config.model_dump(exclude_none=True)
# Add pre_processing configuration if provided
if self._params.pre_processing:
settings["pre_processing"] = self._params.pre_processing.model_dump(exclude_none=True)
if s.pre_processing:
settings["pre_processing"] = s.pre_processing.model_dump(exclude_none=True)
# Add realtime_processing configuration if provided
if self._params.realtime_processing:
settings["realtime_processing"] = self._params.realtime_processing.model_dump(
exclude_none=True
)
if s.realtime_processing:
settings["realtime_processing"] = s.realtime_processing.model_dump(exclude_none=True)
# Add messages_config if provided
if self._params.messages_config:
settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True)
# Store settings for tracing
self._settings = settings
if s.messages_config:
settings["messages_config"] = s.messages_config.model_dump(exclude_none=True)
return settings
@@ -366,6 +422,33 @@ class GladiaSTTService(WebsocketSTTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, delta: GladiaSTTSettings) -> dict[str, Any]:
"""Apply settings delta.
Settings are stored but not applied to the active session.
Args:
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# self._session_url = None
# self._session_id = None
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def stop(self, frame: EndFrame):
"""Stop the Gladia STT websocket connection.
@@ -522,7 +605,7 @@ class GladiaSTTService(WebsocketSTTService):
Broadcasts UserStartedSpeakingFrame and optionally triggers interruption
when VAD is enabled.
"""
if not self._params.enable_vad or self._is_speaking:
if not self._settings.enable_vad or self._is_speaking:
return
logger.debug(f"{self} User started speaking")
@@ -530,14 +613,14 @@ class GladiaSTTService(WebsocketSTTService):
await self.broadcast_frame(UserStartedSpeakingFrame)
if self._should_interrupt:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
async def _on_speech_ended(self):
"""Handle speech end event from Gladia.
Broadcasts UserStoppedSpeakingFrame when VAD is enabled.
"""
if not self._params.enable_vad or not self._is_speaking:
if not self._settings.enable_vad or not self._is_speaking:
return
self._is_speaking = False
await self.broadcast_frame(UserStoppedSpeakingFrame)

167
src/pipecat/services/google/gemini_live/llm.py
View File

@@ -17,9 +17,9 @@ import io
import time
import uuid
import warnings
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from typing import Any, ClassVar, Dict, List, Optional, Union
from loguru import logger
from PIL import Image
@@ -47,7 +47,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
TranscriptionFrame,
TTSAudioRawFrame,
@@ -77,6 +76,7 @@ from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.string import match_endofsentence
from pipecat.utils.time import time_now_iso8601
@@ -602,6 +602,33 @@ class InputParams(BaseModel):
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
@dataclass
class GeminiLiveLLMSettings(LLMSettings):
"""Settings for Gemini Live LLM services.
Parameters:
modalities: Response modalities.
language: Language for generation.
media_resolution: Media resolution setting.
vad: Voice activity detection parameters.
context_window_compression: Context window compression configuration.
thinking: Thinking configuration.
enable_affective_dialog: Whether to enable affective dialog.
proactivity: Proactivity configuration.
"""
modalities: GeminiModalities | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
media_resolution: GeminiMediaResolution | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad: GeminiVADParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
context_window_compression: ContextWindowCompressionParams | dict | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
thinking: ThinkingConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_affective_dialog: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
proactivity: ProactivityConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GeminiLiveLLMService(LLMService):
"""Provides access to Google's Gemini Live API.
@@ -610,6 +637,8 @@ class GeminiLiveLLMService(LLMService):
responses, and tool usage.
"""
_settings: GeminiLiveLLMSettings
# Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter
@@ -666,13 +695,40 @@ class GeminiLiveLLMService(LLMService):
stacklevel=2,
)
super().__init__(base_url=base_url, **kwargs)
params = params or InputParams()
super().__init__(
base_url=base_url,
settings=GeminiLiveLLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
max_tokens=params.max_tokens,
presence_penalty=params.presence_penalty,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
modalities=params.modalities,
language=language_to_gemini_language(params.language)
if params.language
else "en-US",
media_resolution=params.media_resolution,
vad=params.vad,
context_window_compression=params.context_window_compression.model_dump()
if params.context_window_compression
else {},
thinking=params.thinking or {},
enable_affective_dialog=params.enable_affective_dialog or False,
proactivity=params.proactivity or {},
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._last_sent_time = 0
self._base_url = base_url
self.set_model_name(model)
self._voice_id = voice_id
self._language_code = params.language
@@ -714,26 +770,6 @@ class GeminiLiveLLMService(LLMService):
self._consecutive_failures = 0
self._connection_start_time = None
self._settings = {
"frequency_penalty": params.frequency_penalty,
"max_tokens": params.max_tokens,
"presence_penalty": params.presence_penalty,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"modalities": params.modalities,
"language": self._language_code,
"media_resolution": params.media_resolution,
"vad": params.vad,
"context_window_compression": params.context_window_compression.model_dump()
if params.context_window_compression
else {},
"thinking": params.thinking or {},
"enable_affective_dialog": params.enable_affective_dialog or False,
"proactivity": params.proactivity or {},
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self._file_api_base_url = file_api_base_url
self._file_api: Optional[GeminiFileAPI] = None
@@ -776,6 +812,25 @@ class GeminiLiveLLMService(LLMService):
"""
return True
async def _update_settings(self, delta: LLMSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
def set_audio_input_paused(self, paused: bool):
"""Set the audio input pause state.
@@ -798,7 +853,7 @@ class GeminiLiveLLMService(LLMService):
Args:
modalities: The modalities to use for responses.
"""
self._settings["modalities"] = modalities
self._settings.modalities = modalities
def set_language(self, language: Language):
"""Set the language for generation.
@@ -808,7 +863,7 @@ class GeminiLiveLLMService(LLMService):
"""
self._language = language
self._language_code = language_to_gemini_language(language) or "en-US"
self._settings["language"] = self._language_code
self._settings.language = self._language_code
logger.info(f"Set Gemini language to: {self._language_code}")
async def set_context(self, context: OpenAILLMContext):
@@ -866,7 +921,7 @@ class GeminiLiveLLMService(LLMService):
async def _handle_interruption(self):
if self._bot_is_responding:
await self._set_bot_is_responding(False)
if self._settings.get("modalities") == GeminiModalities.AUDIO:
if self._settings.modalities == GeminiModalities.AUDIO:
await self.push_frame(TTSStoppedFrame())
# Do not send LLMFullResponseEndFrame here - an interruption
# already tells the assistant context aggregator that the response
@@ -947,10 +1002,9 @@ class GeminiLiveLLMService(LLMService):
# uses this frame *without* a user context aggregator still works
# (we have an example that does just that, actually).
await self._create_single_response(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
# TODO: implement runtime tool updates for Gemini Live.
pass
else:
await self.push_frame(frame, direction)
@@ -1074,20 +1128,20 @@ class GeminiLiveLLMService(LLMService):
# Assemble basic configuration
config = LiveConnectConfig(
generation_config=GenerationConfig(
frequency_penalty=self._settings["frequency_penalty"],
max_output_tokens=self._settings["max_tokens"],
presence_penalty=self._settings["presence_penalty"],
temperature=self._settings["temperature"],
top_k=self._settings["top_k"],
top_p=self._settings["top_p"],
response_modalities=[Modality(self._settings["modalities"].value)],
frequency_penalty=self._settings.frequency_penalty,
max_output_tokens=self._settings.max_tokens,
presence_penalty=self._settings.presence_penalty,
temperature=self._settings.temperature,
top_k=self._settings.top_k,
top_p=self._settings.top_p,
response_modalities=[Modality(self._settings.modalities.value)],
speech_config=SpeechConfig(
voice_config=VoiceConfig(
prebuilt_voice_config={"voice_name": self._voice_id}
),
language_code=self._settings["language"],
language_code=self._settings.language,
),
media_resolution=MediaResolution(self._settings["media_resolution"].value),
media_resolution=MediaResolution(self._settings.media_resolution.value),
),
input_audio_transcription=AudioTranscriptionConfig(),
output_audio_transcription=AudioTranscriptionConfig(),
@@ -1095,37 +1149,36 @@ class GeminiLiveLLMService(LLMService):
)
# Add context window compression to configuration, if enabled
if self._settings.get("context_window_compression", {}).get("enabled", False):
cwc = self._settings.context_window_compression or {}
if cwc.get("enabled", False):
compression_config = ContextWindowCompressionConfig()
# Add sliding window (always true if compression is enabled)
compression_config.sliding_window = SlidingWindow()
# Add trigger_tokens if specified
trigger_tokens = self._settings.get("context_window_compression", {}).get(
"trigger_tokens"
)
trigger_tokens = cwc.get("trigger_tokens")
if trigger_tokens is not None:
compression_config.trigger_tokens = trigger_tokens
config.context_window_compression = compression_config
# Add thinking configuration to configuration, if provided
if self._settings.get("thinking"):
config.thinking_config = self._settings["thinking"]
if self._settings.thinking:
config.thinking_config = self._settings.thinking
# Add affective dialog setting, if provided
if self._settings.get("enable_affective_dialog", False):
config.enable_affective_dialog = self._settings["enable_affective_dialog"]
if self._settings.enable_affective_dialog:
config.enable_affective_dialog = self._settings.enable_affective_dialog
# Add proactivity configuration to configuration, if provided
if self._settings.get("proactivity"):
config.proactivity = self._settings["proactivity"]
if self._settings.proactivity:
config.proactivity = self._settings.proactivity
# Add VAD configuration to configuration, if provided
if self._settings.get("vad"):
if self._settings.vad:
vad_config = AutomaticActivityDetection()
vad_params = self._settings["vad"]
vad_params = self._settings.vad
has_vad_settings = False
# Only add parameters that are explicitly set
@@ -1183,7 +1236,9 @@ class GeminiLiveLLMService(LLMService):
await self.push_error(error_msg=f"Initialization error: {e}", exception=e)
async def _connection_task_handler(self, config: LiveConnectConfig):
async with self._client.aio.live.connect(model=self._model_name, config=config) as session:
async with self._client.aio.live.connect(
model=self._settings.model, config=config
) as session:
logger.info("Connected to Gemini service")
# Mark connection start time
@@ -1210,7 +1265,7 @@ class GeminiLiveLLMService(LLMService):
# combination with the context aggregator default
# turn strategies.
logger.debug("Gemini VAD: interrupted signal received")
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
elif message.server_content and message.server_content.model_turn:
await self._handle_msg_model_turn(message)
elif (
@@ -1604,7 +1659,7 @@ class GeminiLiveLLMService(LLMService):
text: The transcription text to push
result: Optional LiveServerMessage that triggered this transcription
"""
await self._handle_user_transcription(text, True, self._settings["language"])
await self._handle_user_transcription(text, True, self._settings.language)
await self.push_frame(
TranscriptionFrame(
text=text,

17
src/pipecat/services/google/image.py
View File

@@ -16,6 +16,7 @@ import os
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -25,6 +26,7 @@ from pydantic import BaseModel, Field
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.google.utils import update_google_client_http_options
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try:
from google import genai
@@ -35,6 +37,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class GoogleImageGenSettings(ImageGenSettings):
"""Settings for the Google image generation service.
Parameters:
model: Google Imagen model identifier.
"""
class GoogleImageGenService(ImageGenService):
"""Google AI image generation service using Imagen models.
@@ -72,14 +83,14 @@ class GoogleImageGenService(ImageGenService):
http_options: HTTP options for the client.
**kwargs: Additional arguments passed to the parent ImageGenService.
"""
super().__init__(**kwargs)
self._params = params or GoogleImageGenService.InputParams()
params = params or GoogleImageGenService.InputParams()
super().__init__(settings=GoogleImageGenSettings(model=params.model), **kwargs)
self._params = params
# Add client header
http_options = update_google_client_http_options(http_options)
self._client = genai.Client(api_key=api_key, http_options=http_options)
self.set_model_name(self._params.model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

163
src/pipecat/services/google/llm.py
View File

@@ -15,8 +15,8 @@ import io
import json
import os
import uuid
from dataclasses import dataclass
from typing import Any, AsyncIterator, Dict, List, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncIterator, ClassVar, Dict, List, Literal, Optional
from loguru import logger
from PIL import Image
@@ -39,7 +39,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -59,6 +58,7 @@ from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven, is_given
from pipecat.utils.tracing.service_decorators import traced_llm
# Suppress gRPC fork warnings
@@ -673,6 +673,62 @@ class GoogleLLMContext(OpenAILLMContext):
self._messages = [m for m in self._messages if m.parts]
class GoogleThinkingConfig(BaseModel):
"""Configuration for controlling the model's internal "thinking" process used before generating a response.
Gemini 2.5 and 3 series models have this thinking process.
Parameters:
thinking_level: Thinking level for Gemini 3 models.
For Gemini 3 Pro, this can be "low" or "high".
For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high".
If not provided, Gemini 3 models default to "high".
Note: Gemini 2.5 series must use thinking_budget instead.
thinking_budget: Token budget for thinking, for Gemini 2.5 series.
-1 for dynamic thinking (model decides), 0 to disable thinking,
or a specific token count (e.g., 128-32768 for 2.5 Pro).
If not provided, most models today default to dynamic thinking.
See https://ai.google.dev/gemini-api/docs/thinking#set-budget
for default values and allowed ranges.
Note: Gemini 3 models must use thinking_level instead.
include_thoughts: Whether to include thought summaries in the response.
Today's models default to not including thoughts (False).
"""
thinking_budget: Optional[int] = Field(default=None)
# Why `| str` here? To not break compatibility in case Google adds more
# levels in the future.
thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field(
default=None
)
include_thoughts: Optional[bool] = Field(default=None)
@dataclass
class GoogleLLMSettings(LLMSettings):
"""Settings for Google LLM services.
Parameters:
thinking: Thinking configuration.
"""
thinking: GoogleThinkingConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings):
"""Convert a plain dict to settings, coercing thinking dicts.
For backward compatibility, a ``thinking`` value that is a plain dict
is converted to a :class:`GoogleThinkingConfig`.
"""
instance = super().from_mapping(settings)
if is_given(instance.thinking) and isinstance(instance.thinking, dict):
instance.thinking = GoogleThinkingConfig(**instance.thinking)
return instance
class GoogleLLMService(LLMService):
"""Google AI (Gemini) LLM service implementation.
@@ -681,40 +737,13 @@ class GoogleLLMService(LLMService):
expected by the Google AI model.
"""
_settings: GoogleLLMSettings
# Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter
class ThinkingConfig(BaseModel):
"""Configuration for controlling the model's internal "thinking" process used before generating a response.
Gemini 2.5 and 3 series models have this thinking process.
Parameters:
thinking_level: Thinking level for Gemini 3 models.
For Gemini 3 Pro, this can be "low" or "high".
For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high".
If not provided, Gemini 3 models default to "high".
Note: Gemini 2.5 series must use thinking_budget instead.
thinking_budget: Token budget for thinking, for Gemini 2.5 series.
-1 for dynamic thinking (model decides), 0 to disable thinking,
or a specific token count (e.g., 128-32768 for 2.5 Pro).
If not provided, most models today default to dynamic thinking.
See https://ai.google.dev/gemini-api/docs/thinking#set-budget
for default values and allowed ranges.
Note: Gemini 3 models must use thinking_level instead.
include_thoughts: Whether to include thought summaries in the response.
Today's models default to not including thoughts (False).
"""
thinking_budget: Optional[int] = Field(default=None)
# Why `| str` here? To not break compatibility in case Google adds more
# levels in the future.
thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field(
default=None
)
include_thoughts: Optional[bool] = Field(default=None)
# Backward compatibility: ThinkingConfig used to be defined inline here.
ThinkingConfig = GoogleThinkingConfig
class InputParams(BaseModel):
"""Input parameters for Google AI models.
@@ -737,7 +766,7 @@ class GoogleLLMService(LLMService):
temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
top_k: Optional[int] = Field(default=None, ge=0)
top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0)
thinking: Optional["GoogleLLMService.ThinkingConfig"] = Field(default=None)
thinking: Optional[GoogleThinkingConfig] = Field(default=None)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def __init__(
@@ -764,23 +793,29 @@ class GoogleLLMService(LLMService):
http_options: HTTP options for the client.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
params = params or GoogleLLMService.InputParams()
self.set_model_name(model)
super().__init__(
settings=GoogleLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._api_key = api_key
self._system_instruction = system_instruction
self._http_options = update_google_client_http_options(http_options)
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"thinking": params.thinking,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self._tools = tools
self._tool_config = tool_config
@@ -840,7 +875,7 @@ class GoogleLLMService(LLMService):
# Use the new google-genai client's async method
response = await self._client.aio.models.generate_content(
model=self._model_name,
model=self._settings.model,
contents=messages,
config=generation_config,
)
@@ -874,10 +909,10 @@ class GoogleLLMService(LLMService):
k: v
for k, v in {
"system_instruction": system_instruction,
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"top_k": self._settings["top_k"],
"max_output_tokens": self._settings["max_tokens"],
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"top_k": self._settings.top_k,
"max_output_tokens": self._settings.max_tokens,
"tools": tools,
"tool_config": tool_config,
}.items()
@@ -885,13 +920,13 @@ class GoogleLLMService(LLMService):
}
# Add thinking parameters if configured
if self._settings["thinking"]:
generation_params["thinking_config"] = self._settings["thinking"].model_dump(
if self._settings.thinking:
generation_params["thinking_config"] = self._settings.thinking.model_dump(
exclude_unset=True
)
if self._settings["extra"]:
generation_params.update(self._settings["extra"])
if self._settings.extra:
generation_params.update(self._settings.extra)
return generation_params
@@ -900,10 +935,10 @@ class GoogleLLMService(LLMService):
# There's no way to introspect on model capabilities, so
# to check for models that we know default to thinkin on
# and can be configured to turn it off.
if not self._model_name.startswith("gemini-2.5-flash"):
if not self._settings.model.startswith("gemini-2.5-flash"):
return
# If we have an image model, we don't use a budget either.
if "image" in self._model_name:
if "image" in self._settings.model:
return
# If thinking_config is already set, don't override it.
if "thinking_config" in generation_params:
@@ -944,7 +979,7 @@ class GoogleLLMService(LLMService):
await self.start_ttfb_metrics()
return await self._client.aio.models.generate_content_stream(
model=self._model_name,
model=self._settings.model,
contents=messages,
config=generation_config,
)
@@ -1190,8 +1225,6 @@ class GoogleLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = GoogleLLMContext(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -1215,14 +1248,6 @@ class GoogleLLMService(LLMService):
# Do nothing - we're shutting down anyway
pass
async def _update_settings(self, settings):
"""Override to handle ThinkingConfig validation."""
# Convert thinking dict to ThinkingConfig if needed
if "thinking" in settings and isinstance(settings["thinking"], dict):
settings = dict(settings) # Make a copy to avoid modifying the original
settings["thinking"] = self.ThinkingConfig(**settings["thinking"])
await super()._update_settings(settings)
def create_context_aggregator(
self,
context: OpenAILLMContext,

254
src/pipecat/services/google/stt.py
View File

@@ -15,13 +15,15 @@ import asyncio
import json
import os
import time
import warnings
from dataclasses import dataclass, field
from pipecat.utils.tracing.service_decorators import traced_stt
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from typing import AsyncGenerator, List, Optional, Union
from typing import Any, AsyncGenerator, List, Optional, Union
from loguru import logger
from pydantic import BaseModel, Field, field_validator
@@ -34,6 +36,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GOOGLE_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -355,6 +358,46 @@ def language_to_google_stt_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class GoogleSTTSettings(STTSettings):
"""Settings for Google Cloud Speech-to-Text V2.
Parameters:
languages: List of ``Language`` enums for recognition
(e.g. ``[Language.EN_US]``). Preferred over ``language_codes``.
language_codes: List of Google STT language code strings
(e.g. ``["en-US"]``).
.. deprecated:: 0.0.104
Use ``languages`` instead. If both are provided, ``languages``
takes precedence. This field is here just for backward
compatibility with dict-based settings updates.
use_separate_recognition_per_channel: Process each audio channel separately.
enable_automatic_punctuation: Add punctuation to transcripts.
enable_spoken_punctuation: Include spoken punctuation in transcript.
enable_spoken_emojis: Include spoken emojis in transcript.
profanity_filter: Filter profanity from transcript.
enable_word_time_offsets: Include timing information for each word.
enable_word_confidence: Include confidence scores for each word.
enable_interim_results: Stream partial recognition results.
enable_voice_activity_events: Detect voice activity in audio.
"""
languages: List[Language] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_codes: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_separate_recognition_per_channel: bool | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
enable_automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_spoken_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_spoken_emojis: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_word_time_offsets: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_word_confidence: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_interim_results: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_voice_activity_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GoogleSTTService(STTService):
"""Google Cloud Speech-to-Text V2 service implementation.
@@ -371,6 +414,8 @@ class GoogleSTTService(STTService):
ValueError: If project ID is not found in credentials.
"""
_settings: GoogleSTTSettings
# Google Cloud's STT service has a connection time limit of 5 minutes per stream.
# They've shared an "endless streaming" example that guided this implementation:
# https://cloud.google.com/speech-to-text/docs/transcribe-streaming-audio#endless-streaming
@@ -454,10 +499,29 @@ class GoogleSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or GoogleSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=GoogleSTTSettings(
language=None,
languages=list(params.language_list),
language_codes=None,
model=params.model,
use_separate_recognition_per_channel=params.use_separate_recognition_per_channel,
enable_automatic_punctuation=params.enable_automatic_punctuation,
enable_spoken_punctuation=params.enable_spoken_punctuation,
enable_spoken_emojis=params.enable_spoken_emojis,
profanity_filter=params.profanity_filter,
enable_word_time_offsets=params.enable_word_time_offsets,
enable_word_confidence=params.enable_word_confidence,
enable_interim_results=params.enable_interim_results,
enable_voice_activity_events=params.enable_voice_activity_events,
),
**kwargs,
)
self._location = location
self._stream = None
self._config = None
@@ -508,22 +572,6 @@ class GoogleSTTService(STTService):
self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options)
self._settings = {
"language_codes": [
self.language_to_service_language(lang) for lang in params.language_list
],
"model": params.model,
"use_separate_recognition_per_channel": params.use_separate_recognition_per_channel,
"enable_automatic_punctuation": params.enable_automatic_punctuation,
"enable_spoken_punctuation": params.enable_spoken_punctuation,
"enable_spoken_emojis": params.enable_spoken_emojis,
"profanity_filter": params.profanity_filter,
"enable_word_time_offsets": params.enable_word_time_offsets,
"enable_word_confidence": params.enable_word_confidence,
"enable_interim_results": params.enable_interim_results,
"enable_voice_activity_events": params.enable_voice_activity_events,
}
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
@@ -545,6 +593,21 @@ class GoogleSTTService(STTService):
return [language_to_google_stt_language(lang) or "en-US" for lang in language]
return language_to_google_stt_language(language) or "en-US"
def _get_language_codes(self) -> List[str]:
"""Resolve the current language settings to Google STT language code strings.
Prefers ``languages`` (``Language`` enums) over the deprecated
``language_codes`` (raw strings). Falls back to ``["en-US"]``.
Returns:
List[str]: Google STT language code strings.
"""
if self._settings.languages:
return [self.language_to_service_language(lang) for lang in self._settings.languages]
if self._settings.language_codes:
return list(self._settings.language_codes)
return ["en-US"]
async def _reconnect_if_needed(self):
"""Reconnect the stream if it's currently active."""
if self._streaming_task:
@@ -552,41 +615,65 @@ class GoogleSTTService(STTService):
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
"""Update the service's recognition language.
A convenience method for setting a single language.
Args:
language: New language for recognition.
"""
logger.debug(f"Switching STT language to: {language}")
await self.set_languages([language])
async def set_languages(self, languages: List[Language]):
"""Update the service's recognition languages.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(languages=...)``
instead.
Args:
languages: List of languages for recognition. First language is primary.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"set_languages() is deprecated. Use STTUpdateSettingsFrame with "
"GoogleSTTSettings(languages=...) instead.",
DeprecationWarning,
)
logger.debug(f"Switching STT languages to: {languages}")
self._settings["language_codes"] = [
self.language_to_service_language(lang) for lang in languages
]
# Recreate stream with new languages
await self._reconnect_if_needed()
await self._update_settings(GoogleSTTSettings(languages=list(languages)))
async def set_model(self, model: str):
"""Update the service's recognition model.
async def _update_settings(self, delta: GoogleSTTSettings) -> dict[str, Any]:
"""Apply settings delta and reconnect if anything changed.
Handles ``language`` from base ``set_language`` by converting it to
``languages``. Emits a deprecation warning if ``language_codes`` is
used. All other fields (model, boolean flags) are applied directly.
Reconnects the stream on any change.
Args:
model: The new recognition model to use.
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.debug(f"Switching STT model to: {model}")
await super().set_model(model)
self._settings["model"] = model
# Recreate stream with new model
await self._reconnect_if_needed()
from pipecat.services.settings import is_given
# If base set_language sent a Language value, convert to languages list
if is_given(delta.language):
delta.languages = [delta.language]
# Clear language so the base class doesn't try to store it
delta.language = NOT_GIVEN
# Warn on deprecated language_codes usage
if is_given(delta.language_codes):
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"GoogleSTTSettings.language_codes is deprecated. "
"Use GoogleSTTSettings.languages (List[Language]) instead.",
DeprecationWarning,
stacklevel=2,
)
changed = await super()._update_settings(delta)
if changed:
await self._reconnect_if_needed()
return changed
async def start(self, frame: StartFrame):
"""Start the STT service and establish connection.
@@ -632,6 +719,10 @@ class GoogleSTTService(STTService):
) -> None:
"""Update service options dynamically.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(...)``
instead.
Args:
languages: New list of recognition languages.
model: New recognition model.
@@ -649,55 +740,42 @@ class GoogleSTTService(STTService):
Changes that affect the streaming configuration will cause
the stream to be reconnected.
"""
# Update settings with new values
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"update_options() is deprecated. Use STTUpdateSettingsFrame with "
"GoogleSTTSettings(...) instead.",
DeprecationWarning,
)
# Build a settings delta from the provided options
delta = GoogleSTTSettings()
if languages is not None:
logger.debug(f"Updating language to: {languages}")
self._settings["language_codes"] = [
self.language_to_service_language(lang) for lang in languages
]
delta.languages = list(languages)
if model is not None:
logger.debug(f"Updating model to: {model}")
self._settings["model"] = model
delta.model = model
if enable_automatic_punctuation is not None:
logger.debug(f"Updating automatic punctuation to: {enable_automatic_punctuation}")
self._settings["enable_automatic_punctuation"] = enable_automatic_punctuation
delta.enable_automatic_punctuation = enable_automatic_punctuation
if enable_spoken_punctuation is not None:
logger.debug(f"Updating spoken punctuation to: {enable_spoken_punctuation}")
self._settings["enable_spoken_punctuation"] = enable_spoken_punctuation
delta.enable_spoken_punctuation = enable_spoken_punctuation
if enable_spoken_emojis is not None:
logger.debug(f"Updating spoken emojis to: {enable_spoken_emojis}")
self._settings["enable_spoken_emojis"] = enable_spoken_emojis
delta.enable_spoken_emojis = enable_spoken_emojis
if profanity_filter is not None:
logger.debug(f"Updating profanity filter to: {profanity_filter}")
self._settings["profanity_filter"] = profanity_filter
delta.profanity_filter = profanity_filter
if enable_word_time_offsets is not None:
logger.debug(f"Updating word time offsets to: {enable_word_time_offsets}")
self._settings["enable_word_time_offsets"] = enable_word_time_offsets
delta.enable_word_time_offsets = enable_word_time_offsets
if enable_word_confidence is not None:
logger.debug(f"Updating word confidence to: {enable_word_confidence}")
self._settings["enable_word_confidence"] = enable_word_confidence
delta.enable_word_confidence = enable_word_confidence
if enable_interim_results is not None:
logger.debug(f"Updating interim results to: {enable_interim_results}")
self._settings["enable_interim_results"] = enable_interim_results
delta.enable_interim_results = enable_interim_results
if enable_voice_activity_events is not None:
logger.debug(f"Updating voice activity events to: {enable_voice_activity_events}")
self._settings["enable_voice_activity_events"] = enable_voice_activity_events
delta.enable_voice_activity_events = enable_voice_activity_events
if location is not None:
logger.debug(f"Updating location to: {location}")
self._location = location
# Reconnect the stream for updates
await self._reconnect_if_needed()
await self._update_settings(delta)
async def _connect(self):
"""Initialize streaming recognition config and stream."""
@@ -714,20 +792,20 @@ class GoogleSTTService(STTService):
sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
language_codes=self._settings["language_codes"],
model=self._settings["model"],
language_codes=self._get_language_codes(),
model=self._settings.model,
features=cloud_speech.RecognitionFeatures(
enable_automatic_punctuation=self._settings["enable_automatic_punctuation"],
enable_spoken_punctuation=self._settings["enable_spoken_punctuation"],
enable_spoken_emojis=self._settings["enable_spoken_emojis"],
profanity_filter=self._settings["profanity_filter"],
enable_word_time_offsets=self._settings["enable_word_time_offsets"],
enable_word_confidence=self._settings["enable_word_confidence"],
enable_automatic_punctuation=self._settings.enable_automatic_punctuation,
enable_spoken_punctuation=self._settings.enable_spoken_punctuation,
enable_spoken_emojis=self._settings.enable_spoken_emojis,
profanity_filter=self._settings.profanity_filter,
enable_word_time_offsets=self._settings.enable_word_time_offsets,
enable_word_confidence=self._settings.enable_word_confidence,
),
),
streaming_features=cloud_speech.StreamingRecognitionFeatures(
enable_voice_activity_events=self._settings["enable_voice_activity_events"],
interim_results=self._settings["enable_interim_results"],
enable_voice_activity_events=self._settings.enable_voice_activity_events,
interim_results=self._settings.enable_interim_results,
),
)
@@ -857,7 +935,7 @@ class GoogleSTTService(STTService):
if not transcript:
continue
primary_language = self._settings["language_codes"][0]
primary_language = self._get_language_codes()[0]
if result.is_final:
self._last_transcript_was_final = True

285
src/pipecat/services/google/tts.py
View File

@@ -23,7 +23,8 @@ from pipecat.utils.tracing.service_decorators import traced_tts
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional
from loguru import logger
from pydantic import BaseModel
@@ -36,6 +37,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
@@ -474,6 +476,71 @@ def language_to_gemini_tts_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class GoogleHttpTTSSettings(TTSSettings):
"""Settings for Google HTTP TTS service.
Parameters:
pitch: Voice pitch adjustment (e.g., "+2st", "-50%").
rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). Used for
SSML prosody tags (non-Chirp voices).
speaking_rate: Speaking rate for AudioConfig (Chirp/Journey voices).
Range [0.25, 2.0].
volume: Volume adjustment (e.g., "loud", "soft", "+6dB").
emphasis: Emphasis level for the text.
language: Language for synthesis. Defaults to English.
gender: Voice gender preference.
google_style: Google-specific voice style.
"""
pitch: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emphasis: Literal["strong", "moderate", "reduced", "none"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
gender: Literal["male", "female", "neutral"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
google_style: (
Literal["apologetic", "calm", "empathetic", "firm", "lively"] | None | _NotGiven
) = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class GoogleStreamTTSSettings(TTSSettings):
"""Settings for Google streaming TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
speaking_rate: The speaking rate, in the range [0.25, 2.0].
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class GeminiTTSSettings(TTSSettings):
"""Settings for Gemini TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
prompt: Optional style instructions for how to synthesize the content.
multi_speaker: Whether to enable multi-speaker support.
speaker_configs: List of speaker configurations for multi-speaker mode.
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
multi_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_configs: list[dict[str, Any]] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class GoogleHttpTTSService(TTSService):
"""Google Cloud Text-to-Speech HTTP service with SSML support.
@@ -488,6 +555,8 @@ class GoogleHttpTTSService(TTSService):
Chirp and Journey voices don't support SSML and will use plain text input.
"""
_settings: GoogleHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for Google HTTP TTS voice customization.
@@ -533,24 +602,28 @@ class GoogleHttpTTSService(TTSService):
params: Voice customization parameters including pitch, rate, volume, etc.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=GoogleHttpTTSSettings(
model=None,
pitch=params.pitch,
rate=params.rate,
speaking_rate=params.speaking_rate,
volume=params.volume,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
gender=params.gender,
google_style=params.google_style,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._settings = {
"pitch": params.pitch,
"rate": params.rate,
"speaking_rate": params.speaking_rate,
"volume": params.volume,
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"gender": params.gender,
"google_style": params.google_style,
}
self.set_voice(voice_id)
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
@@ -619,61 +692,60 @@ class GoogleHttpTTSService(TTSService):
"""
return language_to_google_tts_language(language)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle speaking_rate updates for Chirp/Journey voices.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Override to handle speaking_rate validation.
Args:
settings: Dictionary of settings to update. Can include 'speaking_rate' (float)
delta: Settings delta. Can include 'speaking_rate' (float).
"""
if "speaking_rate" in settings:
rate_value = float(settings["speaking_rate"])
if 0.25 <= rate_value <= 2.0:
self._settings["speaking_rate"] = rate_value
else:
if isinstance(delta, GoogleHttpTTSSettings) and is_given(delta.speaking_rate):
rate_value = float(delta.speaking_rate)
if not (0.25 <= rate_value <= 2.0):
logger.warning(
f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0"
)
await super()._update_settings(settings)
delta.speaking_rate = NOT_GIVEN
return await super()._update_settings(delta)
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
# Voice tag
voice_attrs = [f"name='{self._voice_id}'"]
voice_attrs = [f"name='{self._settings.voice}'"]
language = self._settings["language"]
language = self._settings.language
voice_attrs.append(f"language='{language}'")
if self._settings["gender"]:
voice_attrs.append(f"gender='{self._settings['gender']}'")
if self._settings.gender:
voice_attrs.append(f"gender='{self._settings.gender}'")
ssml += f"<voice {' '.join(voice_attrs)}>"
# Prosody tag
prosody_attrs = []
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
# Emphasis tag
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
if self._settings.emphasis:
ssml += f"<emphasis level='{self._settings.emphasis}'>"
# Google style tag
if self._settings["google_style"]:
ssml += f"<google:style name='{self._settings['google_style']}'>"
if self._settings.google_style:
ssml += f"<google:style name='{self._settings.google_style}'>"
ssml += text
# Close tags
if self._settings["google_style"]:
if self._settings.google_style:
ssml += "</google:style>"
if self._settings["emphasis"]:
if self._settings.emphasis:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
@@ -698,8 +770,8 @@ class GoogleHttpTTSService(TTSService):
await self.start_ttfb_metrics()
# Check if the voice is a Chirp voice (including Chirp 3) or Journey voice
is_chirp_voice = "chirp" in self._voice_id.lower()
is_journey_voice = "journey" in self._voice_id.lower()
is_chirp_voice = "chirp" in self._settings.voice.lower()
is_journey_voice = "journey" in self._settings.voice.lower()
# Create synthesis input based on voice_id
if is_chirp_voice or is_journey_voice:
@@ -710,7 +782,7 @@ class GoogleHttpTTSService(TTSService):
synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], name=self._voice_id
language_code=self._settings.language, name=self._settings.voice
)
# Build audio config with conditional speaking_rate
audio_config_params = {
@@ -719,8 +791,8 @@ class GoogleHttpTTSService(TTSService):
}
# For Chirp and Journey voices, include speaking_rate in AudioConfig
if (is_chirp_voice or is_journey_voice) and self._settings["speaking_rate"] is not None:
audio_config_params["speaking_rate"] = self._settings["speaking_rate"]
if (is_chirp_voice or is_journey_voice) and self._settings.speaking_rate is not None:
audio_config_params["speaking_rate"] = self._settings.speaking_rate
audio_config = texttospeech_v1.AudioConfig(**audio_config_params)
@@ -910,6 +982,8 @@ class GoogleTTSService(GoogleBaseTTSService):
)
"""
_settings: GoogleStreamTTSSettings
class InputParams(BaseModel):
"""Input parameters for Google streaming TTS configuration.
@@ -945,38 +1019,41 @@ class GoogleTTSService(GoogleBaseTTSService):
params: Language configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=GoogleStreamTTSSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
speaking_rate=params.speaking_rate,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"speaking_rate": params.speaking_rate,
}
self.set_voice(voice_id)
self._voice_cloning_key = voice_cloning_key
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle speaking_rate updates for streaming API.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Override to handle speaking_rate validation.
Args:
settings: Dictionary of settings to update. Can include 'speaking_rate' (float)
delta: Settings delta. Can include 'speaking_rate' (float).
"""
if "speaking_rate" in settings:
rate_value = float(settings["speaking_rate"])
if 0.25 <= rate_value <= 2.0:
self._settings["speaking_rate"] = rate_value
else:
if isinstance(delta, GoogleStreamTTSSettings) and is_given(delta.speaking_rate):
rate_value = float(delta.speaking_rate)
if not (0.25 <= rate_value <= 2.0):
logger.warning(
f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0"
)
await super()._update_settings(settings)
delta.speaking_rate = NOT_GIVEN
return await super()._update_settings(delta)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -1000,11 +1077,11 @@ class GoogleTTSService(GoogleBaseTTSService):
voice_cloning_key=self._voice_cloning_key
)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], voice_clone=voice_clone_params
language_code=self._settings.language, voice_clone=voice_clone_params
)
else:
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], name=self._voice_id
language_code=self._settings.language, name=self._settings.voice
)
# Create streaming config
@@ -1013,7 +1090,7 @@ class GoogleTTSService(GoogleBaseTTSService):
streaming_audio_config=texttospeech_v1.StreamingAudioConfig(
audio_encoding=texttospeech_v1.AudioEncoding.PCM,
sample_rate_hertz=self.sample_rate,
speaking_rate=self._settings["speaking_rate"],
speaking_rate=self._settings.speaking_rate,
),
)
@@ -1052,6 +1129,8 @@ class GeminiTTSService(GoogleBaseTTSService):
)
"""
_settings: GeminiTTSSettings
GOOGLE_SAMPLE_RATE = 24000 # Google TTS always outputs at 24kHz
# List of available Gemini TTS voices
@@ -1149,25 +1228,27 @@ class GeminiTTSService(GoogleBaseTTSService):
f"Google TTS only supports {self.GOOGLE_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GeminiTTSService.InputParams()
if voice_id not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
self._location = location
self._model = model
self._voice_id = voice_id
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"prompt": params.prompt,
"multi_speaker": params.multi_speaker,
"speaker_configs": params.speaker_configs,
}
super().__init__(
sample_rate=sample_rate,
settings=GeminiTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
prompt=params.prompt,
multi_speaker=params.multi_speaker,
speaker_configs=params.speaker_configs,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
@@ -1183,16 +1264,6 @@ class GeminiTTSService(GoogleBaseTTSService):
"""
return language_to_gemini_tts_language(language)
def set_voice(self, voice_id: str):
"""Set the voice for TTS generation.
Args:
voice_id: Name of the voice to use from AVAILABLE_VOICES.
"""
if voice_id not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
self._voice_id = voice_id
async def start(self, frame: StartFrame):
"""Start the Gemini TTS service.
@@ -1206,15 +1277,19 @@ class GeminiTTSService(GoogleBaseTTSService):
f"Current rate of {self.sample_rate}Hz may cause issues."
)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle prompt updates.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta with voice validation.
Args:
settings: Dictionary of settings to update. Can include 'prompt' (str)
delta: Settings delta. Can include 'voice', 'prompt', etc.
Returns:
Dict mapping changed field names to their previous values.
"""
if "prompt" in settings:
self._settings["prompt"] = settings["prompt"]
await super()._update_settings(settings)
if is_given(delta.voice) and delta.voice not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{delta.voice}' not in known voices list. Using anyway.")
return await super()._update_settings(delta)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -1234,14 +1309,14 @@ class GeminiTTSService(GoogleBaseTTSService):
await self.start_ttfb_metrics()
# Build voice selection params
if self._settings["multi_speaker"] and self._settings["speaker_configs"]:
if self._settings.multi_speaker and self._settings.speaker_configs:
# Multi-speaker mode
speaker_voice_configs = []
for speaker_config in self._settings["speaker_configs"]:
for speaker_config in self._settings.speaker_configs:
speaker_voice_configs.append(
texttospeech_v1.MultispeakerPrebuiltVoice(
speaker_alias=speaker_config["speaker_alias"],
speaker_id=speaker_config.get("speaker_id", self._voice_id),
speaker_id=speaker_config.get("speaker_id", self._settings.voice),
)
)
@@ -1250,16 +1325,16 @@ class GeminiTTSService(GoogleBaseTTSService):
)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"],
model_name=self._model,
language_code=self._settings.language,
model_name=self._settings.model,
multi_speaker_voice_config=multi_speaker_voice_config,
)
else:
# Single speaker mode
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"],
name=self._voice_id,
model_name=self._model,
language_code=self._settings.language,
name=self._settings.voice,
model_name=self._settings.model,
)
# Create streaming config
@@ -1273,7 +1348,7 @@ class GeminiTTSService(GoogleBaseTTSService):
# Use base class streaming logic with prompt support
async for frame in self._stream_tts(
streaming_config, text, context_id, self._settings["prompt"]
streaming_config, text, context_id, self._settings.prompt
):
yield frame

58
src/pipecat/services/gradium/stt.py
View File

@@ -12,7 +12,8 @@ WebSocket API for streaming audio transcription.
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GRADIUM_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -64,6 +66,18 @@ def language_to_gradium_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class GradiumSTTSettings(STTSettings):
"""Settings for the Gradium STT service.
Parameters:
delay_in_frames: Delay in audio frames (80ms each) before text is
generated. Higher delays allow more context but increase latency.
"""
delay_in_frames: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GradiumSTTService(WebsocketSTTService):
"""Gradium real-time speech-to-text service.
@@ -72,6 +86,8 @@ class GradiumSTTService(WebsocketSTTService):
for audio processing and connection management.
"""
_settings: GradiumSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Gradium STT API.
@@ -113,8 +129,6 @@ class GradiumSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(sample_rate=SAMPLE_RATE, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if json_config is not None:
import warnings
@@ -124,10 +138,22 @@ class GradiumSTTService(WebsocketSTTService):
stacklevel=2,
)
params = params or GradiumSTTService.InputParams()
super().__init__(
sample_rate=SAMPLE_RATE,
ttfs_p99_latency=ttfs_p99_latency,
settings=GradiumSTTSettings(
model=None,
language=params.language,
delay_in_frames=params.delay_in_frames or None,
),
**kwargs,
)
self._api_key = api_key
self._api_endpoint_base_url = api_endpoint_base_url
self._websocket = None
self._params = params or GradiumSTTService.InputParams()
self._json_config = json_config
self._receive_task = None
@@ -149,16 +175,22 @@ class GradiumSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta, sync params, and reconnect.
Args:
language: The language to use for speech recognition.
delta: A :class:`STTSettings` (or ``GradiumSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.info(f"Switching STT language to: [{language}]")
self._params.language = language
changed = await super()._update_settings(delta)
if not changed:
return changed
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
@@ -298,12 +330,12 @@ class GradiumSTTService(WebsocketSTTService):
json_config = {}
if self._json_config:
json_config = json.loads(self._json_config)
if self._params.language:
gradium_language = language_to_gradium_language(self._params.language)
if self._settings.language:
gradium_language = language_to_gradium_language(self._settings.language)
if gradium_language:
json_config["language"] = gradium_language
if self._params.delay_in_frames:
json_config["delay_in_frames"] = self._params.delay_in_frames
if self._settings.delay_in_frames:
json_config["delay_in_frames"] = self._settings.delay_in_frames
if json_config:
setup_msg["json_config"] = json_config
await self._websocket.send(json.dumps(setup_msg))

92
src/pipecat/services/gradium/tts.py
View File

@@ -6,7 +6,8 @@
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
@@ -16,14 +17,13 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
try:
@@ -38,9 +38,22 @@ except ModuleNotFoundError as e:
SAMPLE_RATE = 48000
class GradiumTTSService(AudioContextWordTTSService):
@dataclass
class GradiumTTSSettings(TTSSettings):
"""Settings for the Gradium TTS service.
Parameters:
output_format: Audio output format.
"""
output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GradiumTTSService(AudioContextTTSService):
"""Text-to-Speech service using Gradium's websocket API."""
_settings: GradiumTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Gradium TTS service.
@@ -72,27 +85,27 @@ class GradiumTTSService(AudioContextWordTTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent class.
"""
params = params or GradiumTTSService.InputParams()
super().__init__(
push_stop_frames=True,
push_text_frames=False,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=SAMPLE_RATE,
settings=GradiumTTSSettings(
model=model,
voice=voice_id,
language=None,
output_format="pcm",
),
**kwargs,
)
params = params or GradiumTTSService.InputParams()
# Store service configuration
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._json_config = json_config
self._model = model
self._settings = {
"voice_id": voice_id,
"model_name": model,
"output_format": "pcm",
}
# State tracking
self._receive_task = None
@@ -105,24 +118,22 @@ class GradiumTTSService(AudioContextWordTTSService):
"""
return True
async def set_model(self, model: str):
"""Update the TTS model.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if voice changed.
Args:
model: The model name to use for synthesis.
"""
self._model = model
await super().set_model(model)
delta: A :class:`TTSSettings` (or ``GradiumTTSSettings``) delta.
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
self._settings["voice_id"] = self._voice_id
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if "voice" in changed:
await self._disconnect()
await self._connect()
else:
self._warn_unhandled_updated_settings(changed)
return changed
def _build_msg(self, text: str = "") -> dict:
"""Build JSON message for Gradium API."""
@@ -200,7 +211,7 @@ class GradiumTTSService(AudioContextWordTTSService):
setup_msg = {
"type": "setup",
"output_format": "pcm",
"voice_id": self._voice_id,
"voice_id": self._settings.voice,
"close_ws_on_eos": False,
}
if self._json_config is not None:
@@ -252,21 +263,24 @@ class GradiumTTSService(AudioContextWordTTSService):
except Exception as e:
logger.error(f"{self} exception: {e}")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by resetting context state.
async def on_audio_context_interrupted(self, context_id: str):
"""Called when an audio context is cancelled due to an interruption.
The parent AudioContextTTSService._handle_interruption() cancels the audio context
task and creates a new one. We reset _context_id so the next run_tts() creates a
fresh context. No websocket reconnection needed — audio from the old client_req_id
will be silently dropped since the audio context no longer exists.
Args:
frame: The interruption frame.
direction: The direction of the frame.
No WebSocket message is needed — audio from the interrupted
``client_req_id`` will be silently dropped by the base class once the
audio context no longer exists.
"""
await super()._handle_interruption(frame, direction)
await self.stop_all_metrics()
async def on_audio_context_completed(self, context_id: str):
"""Called after an audio context has finished playing all of its audio.
No close message is needed: Gradium signals completion with an
``end_of_stream`` message (handled in ``_receive_messages``), after
which the server-side context is already closed.
"""
pass
async def _receive_messages(self):
"""Process incoming websocket messages, demultiplexing by client_req_id."""
# TODO(laurent): This should not be necessary as it should happen when

145
src/pipecat/services/grok/realtime/llm.py
View File

@@ -13,8 +13,8 @@ https://docs.x.ai/docs/guides/voice/agent
import base64
import json
import time
from dataclasses import dataclass
from typing import Optional
from dataclasses import dataclass, field
from typing import Any, Optional
from loguru import logger
@@ -56,6 +56,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
from . import events
@@ -85,6 +86,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class GrokRealtimeLLMSettings(LLMSettings):
"""Settings for Grok Realtime LLM services.
Parameters:
session_properties: Grok Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class GrokRealtimeLLMService(LLMService):
"""Grok Realtime Voice Agent LLM service providing real-time audio and text communication.
@@ -101,6 +115,8 @@ class GrokRealtimeLLMService(LLMService):
- Server-side VAD (Voice Activity Detection)
"""
_settings: GrokRealtimeLLMSettings
# Use the Grok-specific adapter
adapter_class = GrokRealtimeLLMAdapter
@@ -129,16 +145,27 @@ class GrokRealtimeLLMService(LLMService):
start_audio_paused: Whether to start with audio input paused. Defaults to False.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(base_url=base_url, **kwargs)
super().__init__(
base_url=base_url,
settings=GrokRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = base_url
# Initialize session_properties
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._websocket = None
self._receive_task = None
@@ -186,13 +213,13 @@ class GrokRealtimeLLMService(LLMService):
Configured sample rate or None if not manually configured.
For PCMU/PCMA formats, returns 8000 Hz (G.711 standard).
"""
if not self._session_properties.audio:
if not self._settings.session_properties.audio:
return None
audio_config = (
self._session_properties.audio.input
self._settings.session_properties.audio.input
if direction == "input"
else self._session_properties.audio.output
else self._settings.session_properties.audio.output
)
if audio_config and audio_config.format:
@@ -222,8 +249,8 @@ class GrokRealtimeLLMService(LLMService):
def _is_turn_detection_enabled(self) -> bool:
"""Check if server-side VAD is enabled."""
if self._session_properties.turn_detection:
return self._session_properties.turn_detection.type == "server_vad"
if self._settings.session_properties.turn_detection:
return self._settings.session_properties.turn_detection.type == "server_vad"
return False
async def _handle_interruption(self):
@@ -281,6 +308,27 @@ class GrokRealtimeLLMService(LLMService):
# Standard AIService frame handling
#
def _ensure_audio_config(self, input_sample_rate: int, output_sample_rate: int):
"""Ensure session_properties.audio has input and output configs.
Fills in any missing audio configuration using the given sample rates.
Args:
input_sample_rate: Sample rate for audio input (Hz).
output_sample_rate: Sample rate for audio output (Hz).
"""
props = self._settings.session_properties
if not props.audio:
props.audio = events.AudioConfiguration()
if not props.audio.input:
props.audio.input = events.AudioInput(
format=events.PCMAudioFormat(rate=input_sample_rate)
)
if not props.audio.output:
props.audio.output = events.AudioOutput(
format=events.PCMAudioFormat(rate=output_sample_rate)
)
async def start(self, frame: StartFrame):
"""Start the service and establish WebSocket connection.
@@ -288,23 +336,7 @@ class GrokRealtimeLLMService(LLMService):
frame: The start frame triggering service initialization.
"""
await super().start(frame)
# Ensure audio configuration exists with both input and output
if not self._session_properties.audio:
self._session_properties.audio = events.AudioConfiguration()
# Fill in missing input configuration
if not self._session_properties.audio.input:
self._session_properties.audio.input = events.AudioInput(
format=events.PCMAudioFormat(rate=frame.audio_in_sample_rate)
)
# Fill in missing output configuration
if not self._session_properties.audio.output:
self._session_properties.audio.output = events.AudioOutput(
format=events.PCMAudioFormat(rate=frame.audio_out_sample_rate)
)
self._ensure_audio_config(frame.audio_in_sample_rate, frame.audio_out_sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -336,6 +368,16 @@ class GrokRealtimeLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.delta path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.delta is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -355,11 +397,8 @@ class GrokRealtimeLLMService(LLMService):
await self._handle_bot_stopped_speaking()
elif isinstance(frame, LLMMessagesAppendFrame):
await self._handle_messages_append(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
await self.push_frame(frame, direction)
@@ -436,9 +475,30 @@ class GrokRealtimeLLMService(LLMService):
return
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update if needed."""
# Capture current sample rates before the update replaces them.
input_rate = self._get_configured_sample_rate("input")
output_rate = self._get_configured_sample_rate("output")
changed = await super()._update_settings(delta)
if "session_properties" in changed:
if input_rate and output_rate:
self._ensure_audio_config(input_rate, output_rate)
else:
logger.warning(
"Attempting to apply session properties update without configured sample rates. "
"Audio configuration may be incomplete."
)
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"})
return changed
async def _send_session_update(self):
"""Update session settings on the server."""
settings = self._session_properties
settings = self._settings.session_properties
adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
@@ -511,12 +571,15 @@ class GrokRealtimeLLMService(LLMService):
elif evt.type == "response.function_call_arguments.done":
await self._handle_evt_function_call_arguments_done(evt)
elif evt.type == "error":
await self._handle_evt_error(evt)
return
if evt.error.code == "response_cancel_not_active":
logger.debug(f"{self} {evt.error.message}")
else:
await self._handle_evt_error(evt)
return
async def _handle_evt_conversation_created(self, evt):
"""Handle conversation.created event - first event after connecting."""
await self._update_settings()
await self._send_session_update()
async def _handle_evt_response_created(self, evt):
"""Handle response.created event - response generation started."""
@@ -671,7 +734,7 @@ class GrokRealtimeLLMService(LLMService):
"""Handle speech started event from VAD."""
await self._truncate_current_audio_response()
await self.broadcast_frame(UserStartedSpeakingFrame)
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
async def _handle_evt_speech_stopped(self, evt):
"""Handle speech stopped event from VAD."""
@@ -719,7 +782,7 @@ class GrokRealtimeLLMService(LLMService):
self._messages_added_manually[evt.item.id] = True
await self.send_client_event(evt)
await self._update_settings()
await self._send_session_update()
self._llm_needs_conversation_setup = False
logger.debug("Creating Grok response")

2
src/pipecat/services/groq/stt.py
View File

@@ -62,7 +62,7 @@ class GroqSTTService(BaseWhisperSTTService):
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"model": self._settings.model,
# Use verbose_json to get probability metrics
"response_format": "verbose_json" if self._include_prob_metrics else "json",
"language": self._language,

54
src/pipecat/services/groq/tts.py
View File

@@ -8,7 +8,8 @@
import io
import wave
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, ClassVar, Dict, Optional
from loguru import logger
from pydantic import BaseModel
@@ -20,6 +21,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -32,6 +34,23 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class GroqTTSSettings(TTSSettings):
"""Settings for the Groq TTS service.
Parameters:
output_format: Audio output format.
speed: Speech speed multiplier. Defaults to 1.0.
groq_sample_rate: Audio sample rate.
"""
output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
groq_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "groq_sample_rate"}
class GroqTTSService(TTSService):
"""Groq text-to-speech service implementation.
@@ -40,6 +59,8 @@ class GroqTTSService(TTSService):
and output formats.
"""
_settings: GroqTTSSettings
class InputParams(BaseModel):
"""Input parameters for Groq TTS configuration.
@@ -78,28 +99,24 @@ class GroqTTSService(TTSService):
if sample_rate != self.GROQ_SAMPLE_RATE:
logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ")
params = params or GroqTTSService.InputParams()
super().__init__(
pause_frame_processing=True,
sample_rate=sample_rate,
settings=GroqTTSSettings(
model=model_name,
voice=voice_id,
language=str(params.language) if params.language else "en",
output_format=output_format,
speed=params.speed,
groq_sample_rate=sample_rate,
),
**kwargs,
)
params = params or GroqTTSService.InputParams()
self._api_key = api_key
self._model_name = model_name
self._output_format = output_format
self._voice_id = voice_id
self._params = params
self._settings = {
"model": model_name,
"voice_id": voice_id,
"output_format": output_format,
"language": str(params.language) if params.language else "en",
"speed": params.speed,
"sample_rate": sample_rate,
}
self._client = AsyncGroq(api_key=self._api_key)
@@ -129,9 +146,12 @@ class GroqTTSService(TTSService):
try:
response = await self._client.audio.speech.create(
model=self._model_name,
voice=self._voice_id,
model=self._settings.model,
voice=self._settings.voice,
response_format=self._output_format,
# Note: as of 2026-02-25, only a speed of 1.0 is supported, but
# here we pass it for completeness and future-proofing
speed=self._settings.speed,
input=text,
)

47
src/pipecat/services/hathora/stt.py
View File

@@ -8,6 +8,7 @@
import base64
import os
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional
import aiohttp
@@ -18,6 +19,7 @@ from pipecat.frames.frames import (
Frame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import HATHORA_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
@@ -27,12 +29,27 @@ from pipecat.utils.tracing.service_decorators import traced_stt
from .utils import ConfigOption
@dataclass
class HathoraSTTSettings(STTSettings):
"""Settings for the Hathora STT service.
Parameters:
config: Some models support additional config, refer to
`docs <https://models.hathora.dev>`_ for each model to see
what is supported.
"""
config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HathoraSTTService(SegmentedSTTService):
"""This service supports several different speech-to-text models hosted by Hathora.
[Documentation](https://models.hathora.dev)
"""
_settings: HathoraSTTSettings
class InputParams(BaseModel):
"""Optional input parameters for Hathora STT configuration.
@@ -72,24 +89,21 @@ class HathoraSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the parent class.
"""
params = params or HathoraSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=HathoraSTTSettings(
model=model,
language=params.language,
config=params.config,
),
**kwargs,
)
self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
params = params or HathoraSTTService.InputParams()
self._settings = {
"language": params.language,
"config": params.config,
}
self.set_model_name(model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -120,15 +134,14 @@ class HathoraSTTService(SegmentedSTTService):
url = f"{self._base_url}"
payload = {
"model": self._model,
"model": self._settings.model,
}
if self._settings["language"] is not None:
payload["language"] = self._settings["language"]
if self._settings["config"] is not None:
if self._settings.language is not None:
payload["language"] = self._settings.language
if self._settings.config is not None:
payload["model_config"] = [
{"name": option.name, "value": option.value}
for option in self._settings["config"]
{"name": option.name, "value": option.value} for option in self._settings.config
]
base64_audio = base64.b64encode(audio).decode("utf-8")
@@ -147,7 +160,7 @@ class HathoraSTTService(SegmentedSTTService):
if text: # Only yield non-empty text
# Hathora's API currently doesn't return language info
# so we default to the requested language or "en"
response_language = self._settings["language"] or "en"
response_language = self._settings.language or "en"
await self._handle_transcription(text, True, response_language)
yield TranscriptionFrame(
text,

62
src/pipecat/services/hathora/tts.py
View File

@@ -9,6 +9,7 @@
import io
import os
import wave
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional, Tuple
import aiohttp
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -45,12 +47,29 @@ def _decode_audio_payload(
return audio_bytes, fallback_sample_rate, fallback_channels
@dataclass
class HathoraTTSSettings(TTSSettings):
"""Settings for Hathora TTS service.
Parameters:
speed: Speech speed multiplier (if supported by model).
config: Some models support additional config, refer to
[docs](https://models.hathora.dev) for each model to see
what is supported.
"""
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HathoraTTSService(TTSService):
"""This service supports several different text-to-speech models hosted by Hathora.
[Documentation](https://models.hathora.dev)
"""
_settings: HathoraTTSSettings
class InputParams(BaseModel):
"""Optional input parameters for Hathora TTS configuration.
@@ -88,23 +107,21 @@ class HathoraTTSService(TTSService):
params: Configuration parameters.
**kwargs: Additional arguments passed to the parent class.
"""
super().__init__(
sample_rate=sample_rate,
**kwargs,
)
self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
params = params or HathoraTTSService.InputParams()
self._settings = {
"speed": params.speed,
"config": params.config,
}
self.set_model_name(model)
self.set_voice(voice_id)
super().__init__(
sample_rate=sample_rate,
settings=HathoraTTSSettings(
model=model,
voice=voice_id,
language=None, # Not applicable here
speed=params.speed,
config=params.config,
),
**kwargs,
)
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -131,16 +148,15 @@ class HathoraTTSService(TTSService):
url = f"{self._base_url}"
payload = {"model": self._model, "text": text}
payload = {"model": self._settings.model, "text": text}
if self._voice_id is not None:
payload["voice"] = self._voice_id
if self._settings["speed"] is not None:
payload["speed"] = self._settings["speed"]
if self._settings["config"] is not None:
if self._settings.voice is not None:
payload["voice"] = self._settings.voice
if self._settings.speed is not None:
payload["speed"] = self._settings.speed
if self._settings.config is not None:
payload["model_config"] = [
{"name": option.name, "value": option.value}
for option in self._settings["config"]
{"name": option.name, "value": option.value} for option in self._settings.config
]
yield TTSStartedFrame(context_id=context_id)

7
src/pipecat/services/heygen/client.py
View File

@@ -62,10 +62,12 @@ class HeyGenCallbacks(BaseModel):
"""Callback handlers for HeyGen events.
Parameters:
on_participant_connected: Called when a participant connects
on_participant_disconnected: Called when a participant disconnects
on_connected: Called when the bot connects to the LiveKit room.
on_participant_connected: Called when a participant connects.
on_participant_disconnected: Called when a participant disconnects.
"""
on_connected: Callable[[], Awaitable[None]]
on_participant_connected: Callable[[str], Awaitable[None]]
on_participant_disconnected: Callable[[str], Awaitable[None]]
@@ -251,6 +253,7 @@ class HeyGenClient:
logger.debug(f"HeyGenClient send_interval: {self._send_interval}")
await self._ws_connect()
await self._livekit_connect()
self._call_event_callback(self._callbacks.on_connected)
async def stop(self) -> None:
"""Stop the client and terminate all connections.

5
src/pipecat/services/heygen/video.py
View File

@@ -128,6 +128,7 @@ class HeyGenVideoService(AIService):
session_request=self._session_request,
service_type=self._service_type,
callbacks=HeyGenCallbacks(
on_connected=self._on_connected,
on_participant_connected=self._on_participant_connected,
on_participant_disconnected=self._on_participant_disconnected,
),
@@ -144,6 +145,10 @@ class HeyGenVideoService(AIService):
await self._client.cleanup()
self._client = None
async def _on_connected(self):
"""Handle bot connected to LiveKit room."""
logger.info("HeyGen bot connected to LiveKit room")
async def _on_participant_connected(self, participant_id: str):
"""Handle participant connected events."""
logger.info(f"Participant connected {participant_id}")

96
src/pipecat/services/hume/tts.py
View File

@@ -6,6 +6,8 @@
import base64
import os
import warnings
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import httpx
@@ -24,7 +26,8 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import WordTTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
try:
@@ -46,7 +49,22 @@ DEFAULT_HEADERS = {
}
class HumeTTSService(WordTTSService):
@dataclass
class HumeTTSSettings(TTSSettings):
"""Settings for Hume TTS service.
Parameters:
description: Natural-language acting directions (up to 100 characters).
speed: Speaking-rate multiplier (0.5-2.0).
trailing_silence: Seconds of silence to append at the end (0-5).
"""
description: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
trailing_silence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HumeTTSService(TTSService):
"""Hume Octave Text-to-Speech service.
Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint
@@ -61,6 +79,8 @@ class HumeTTSService(WordTTSService):
- Provides metrics for Time To First Byte (TTFB) and TTS usage.
"""
_settings: HumeTTSSettings
class InputParams(BaseModel):
"""Optional synthesis parameters for Hume TTS.
@@ -101,11 +121,21 @@ class HumeTTSService(WordTTSService):
f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}"
)
# WordTTSService sets push_text_frames=False by default, which we want
params = params or HumeTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
settings=HumeTTSSettings(
model=None,
voice=voice_id,
language=None, # Not applicable here
description=params.description,
speed=params.speed,
trailing_silence=params.trailing_silence,
),
**kwargs,
)
@@ -114,10 +144,6 @@ class HumeTTSService(WordTTSService):
self._http_client = httpx.AsyncClient(headers=DEFAULT_HEADERS)
self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client)
self._params = params or HumeTTSService.InputParams()
# Store voice in the base class (mirrors other services)
self.set_voice(voice_id)
self._audio_bytes = b""
@@ -183,7 +209,10 @@ class HumeTTSService(WordTTSService):
await self.add_word_timestamps([("Reset", 0)])
async def update_setting(self, key: str, value: Any) -> None:
"""Runtime updates via `TTSUpdateSettingsFrame`.
"""Runtime updates via key/value pair.
.. deprecated:: 0.0.104
Use ``TTSUpdateSettingsFrame(delta=HumeTTSSettings(...))`` instead.
Args:
key: The name of the setting to update. Recognized keys are:
@@ -193,20 +222,29 @@ class HumeTTSService(WordTTSService):
- "trailing_silence"
value: The new value for the setting.
"""
key_l = (key or "").lower()
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'update_setting' is deprecated, use "
"'TTSUpdateSettingsFrame(delta=HumeTTSSettings(...))' instead.",
DeprecationWarning,
stacklevel=2,
)
if key_l == "voice_id":
self.set_voice(str(value))
logger.debug(f"HumeTTSService voice_id set to: {self.voice}")
elif key_l == "description":
self._params.description = None if value is None else str(value)
elif key_l == "speed":
self._params.speed = None if value is None else float(value)
elif key_l == "trailing_silence":
self._params.trailing_silence = None if value is None else float(value)
else:
# Defer unknown keys to the base class
await super().update_setting(key, value)
key_l = (key or "").lower()
known_keys = {"voice_id", "voice", "description", "speed", "trailing_silence"}
if key_l in known_keys:
kwargs: dict[str, Any] = {}
if key_l in ("voice_id", "voice"):
kwargs["voice"] = str(value)
elif key_l == "description":
kwargs["description"] = None if value is None else str(value)
elif key_l == "speed":
kwargs["speed"] = None if value is None else float(value)
elif key_l == "trailing_silence":
kwargs["trailing_silence"] = None if value is None else float(value)
await self._update_settings(HumeTTSSettings(**kwargs))
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -226,14 +264,14 @@ class HumeTTSService(WordTTSService):
# Build the request payload
utterance_kwargs: dict[str, Any] = {
"text": text,
"voice": PostedUtteranceVoiceWithId(id=self._voice_id),
"voice": PostedUtteranceVoiceWithId(id=self._settings.voice),
}
if self._params.description is not None:
utterance_kwargs["description"] = self._params.description
if self._params.speed is not None:
utterance_kwargs["speed"] = self._params.speed
if self._params.trailing_silence is not None:
utterance_kwargs["trailing_silence"] = self._params.trailing_silence
if self._settings.description is not None:
utterance_kwargs["description"] = self._settings.description
if self._settings.speed is not None:
utterance_kwargs["speed"] = self._settings.speed
if self._settings.trailing_silence is not None:
utterance_kwargs["trailing_silence"] = self._settings.trailing_silence
utterance = PostedUtterance(**utterance_kwargs)
@@ -257,7 +295,7 @@ class HumeTTSService(WordTTSService):
# Use version "2" by default if no description is provided
# Version "1" is needed when description is used
version = "1" if self._params.description is not None else "2"
version = "1" if self._settings.description is not None else "2"
# Track the duration of this utterance based on the last timestamp
utterance_duration = 0.0

14
src/pipecat/services/image_service.py
View File

@@ -11,11 +11,12 @@ text prompts into images.
"""
from abc import abstractmethod
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from pipecat.frames.frames import Frame, TextFrame
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import ImageGenSettings
class ImageGenService(AIService):
@@ -26,13 +27,20 @@ class ImageGenService(AIService):
generation functionality using their specific AI service.
"""
def __init__(self, **kwargs):
def __init__(self, *, settings: Optional[ImageGenSettings] = None, **kwargs):
"""Initialize the image generation service.
Args:
settings: The runtime-updatable settings for the image generation service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ImageGenSettings(),
**kwargs,
)
# Renders the image. Returns an Image object.
@abstractmethod

253
src/pipecat/services/inworld/tts.py
View File

@@ -17,7 +17,8 @@ import asyncio
import base64
import json
import uuid
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Tuple
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, List, Literal, Mapping, Optional, Tuple
import aiohttp
import websockets
@@ -28,6 +29,8 @@ from pipecat import version as pipecat_version
USER_AGENT = f"pipecat/{pipecat_version()}"
from pydantic import BaseModel
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
try:
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
@@ -48,17 +51,66 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService, WordTTSService
from pipecat.services.tts_service import AudioContextTTSService, TextAggregationMode, TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
class InworldHttpTTSService(WordTTSService):
@dataclass
class InworldTTSSettings(TTSSettings):
"""Settings for Inworld TTS services.
Parameters:
audio_encoding: Audio encoding format (e.g. LINEAR16).
audio_sample_rate: Audio sample rate in Hz.
speaking_rate: Speaking rate for speech synthesis.
temperature: Temperature for speech synthesis.
auto_mode: Whether to use auto mode. Recommended when texts are sent
in full sentences/phrases. When enabled, the server controls
flushing of buffered text to achieve minimal latency while
maintaining high quality audio output. If None (default),
automatically set based on aggregate_sentences.
apply_text_normalization: Whether to apply text normalization.
timestamp_transport_strategy: Strategy for timestamp transport ("ASYNC" or "SYNC").
"""
audio_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
auto_mode: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
timestamp_transport_strategy: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {
"voice_id": "voice",
"voiceId": "voice",
"modelId": "model",
"applyTextNormalization": "apply_text_normalization",
"autoMode": "auto_mode",
"timestampTransportStrategy": "timestamp_transport_strategy",
}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "InworldTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``audioConfig``."""
flat = dict(settings)
nested = flat.pop("audioConfig", None)
if isinstance(nested, dict):
flat.setdefault("audio_encoding", nested.get("audioEncoding"))
flat.setdefault("audio_sample_rate", nested.get("sampleRateHertz"))
flat.setdefault("speaking_rate", nested.get("speakingRate"))
return super().from_mapping(flat)
class InworldHttpTTSService(TTSService):
"""Inworld AI HTTP-based TTS service.
Supports both streaming and non-streaming modes via the `streaming` parameter.
Outputs LINEAR16 audio at configurable sample rates with word-level timestamps.
"""
_settings: InworldTTSSettings
class InputParams(BaseModel):
"""Input parameters for Inworld TTS configuration.
@@ -98,15 +150,28 @@ class InworldHttpTTSService(WordTTSService):
params: Input parameters for Inworld TTS configuration.
**kwargs: Additional arguments passed to the parent class.
"""
params = params or InworldHttpTTSService.InputParams()
super().__init__(
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=None, # Not applicable for HTTP TTS
apply_text_normalization=None, # Not applicable for HTTP TTS
),
**kwargs,
)
params = params or InworldHttpTTSService.InputParams()
self._api_key = api_key
self._session = aiohttp_session
self._streaming = streaming
@@ -117,27 +182,8 @@ class InworldHttpTTSService(WordTTSService):
else:
self._base_url = "https://api.inworld.ai/tts/v1/voice"
self._settings = {
"voiceId": voice_id,
"modelId": model,
"audioConfig": {
"audioEncoding": encoding,
"sampleRateHertz": 0,
},
}
if params.temperature is not None:
self._settings["temperature"] = params.temperature
if params.speaking_rate is not None:
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
self._cumulative_time = 0.0
self.set_voice(voice_id)
self.set_model_name(model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -153,7 +199,7 @@ class InworldHttpTTSService(WordTTSService):
frame: The start frame.
"""
await super().start(frame)
self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
async def stop(self, frame: EndFrame):
"""Stop the Inworld TTS service.
@@ -232,20 +278,27 @@ class InworldHttpTTSService(WordTTSService):
"""
logger.debug(f"{self}: Generating TTS [{text}] (streaming={self._streaming})")
audio_config = {
"audioEncoding": self._settings.audio_encoding,
"sampleRateHertz": self._settings.audio_sample_rate,
}
if self._settings.speaking_rate is not None:
audio_config["speakingRate"] = self._settings.speaking_rate
payload = {
"text": text,
"voiceId": self._settings["voiceId"],
"modelId": self._settings["modelId"],
"audioConfig": self._settings["audioConfig"],
"voiceId": self._settings.voice,
"modelId": self._settings.model,
"audioConfig": audio_config,
}
if "temperature" in self._settings:
payload["temperature"] = self._settings["temperature"]
if self._settings.temperature is not None:
payload["temperature"] = self._settings.temperature
# Use WORD timestamps for simplicity and correct spacing/capitalization
payload["timestampType"] = self._timestamp_type
if "timestampTransportStrategy" in self._settings:
payload["timestampTransportStrategy"] = self._settings["timestampTransportStrategy"]
if self._settings.timestamp_transport_strategy is not None:
payload["timestampTransportStrategy"] = self._settings.timestamp_transport_strategy
request_id = str(uuid.uuid4())
headers = {
@@ -411,7 +464,7 @@ class InworldHttpTTSService(WordTTSService):
)
class InworldTTSService(AudioContextWordTTSService):
class InworldTTSService(AudioContextTTSService):
"""Inworld AI WebSocket-based TTS service.
Uses bidirectional WebSocket for lower latency streaming. Supports multiple
@@ -419,6 +472,8 @@ class InworldTTSService(AudioContextWordTTSService):
with word-level timestamps.
"""
_settings: InworldTTSSettings
class InputParams(BaseModel):
"""Input parameters for Inworld WebSocket TTS configuration.
@@ -454,7 +509,8 @@ class InworldTTSService(AudioContextWordTTSService):
sample_rate: Optional[int] = None,
encoding: str = "LINEAR16",
params: InputParams = None,
aggregate_sentences: bool = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
append_trailing_space: bool = True,
**kwargs: Any,
):
@@ -468,48 +524,45 @@ class InworldTTSService(AudioContextWordTTSService):
sample_rate: Audio sample rate in Hz.
encoding: Audio encoding format.
params: Input parameters for Inworld WebSocket TTS configuration.
aggregate_sentences: Whether to aggregate sentences before synthesis.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
append_trailing_space: Whether to append a trailing space to text before sending to TTS.
**kwargs: Additional arguments passed to the parent class.
"""
params = params or InworldTTSService.InputParams()
super().__init__(
push_text_frames=False,
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
append_trailing_space=append_trailing_space,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
apply_text_normalization=params.apply_text_normalization,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=params.auto_mode if params.auto_mode is not None else aggregate_sentences,
),
**kwargs,
)
params = params or InworldTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings: Dict[str, Any] = {
"voiceId": voice_id,
"modelId": model,
"audioConfig": {
"audioEncoding": encoding,
"sampleRateHertz": 0,
},
}
self._timestamp_type = "WORD"
if params.temperature is not None:
self._settings["temperature"] = params.temperature
if params.speaking_rate is not None:
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
if params.apply_text_normalization is not None:
self._settings["applyTextNormalization"] = params.apply_text_normalization
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
if params.auto_mode is not None:
self._settings["autoMode"] = params.auto_mode
else:
self._settings["autoMode"] = aggregate_sentences
self._buffer_settings = {
"maxBufferDelayMs": params.max_buffer_delay_ms,
"bufferCharThreshold": params.buffer_char_threshold,
@@ -526,9 +579,6 @@ class InworldTTSService(AudioContextWordTTSService):
# Track the end time of the last word in the current generation
self._generation_end_time = 0.0
self.set_voice(voice_id)
self.set_model_name(model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -544,7 +594,7 @@ class InworldTTSService(AudioContextWordTTSService):
frame: The start frame.
"""
await super().start(frame)
self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -633,28 +683,23 @@ class InworldTTSService(AudioContextWordTTSService):
return word_times
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle an interruption from the Inworld WebSocket TTS service.
Args:
frame: The interruption frame.
direction: The direction of the interruption.
"""
old_context_id = self.get_active_audio_context_id()
logger.trace(f"{self}: Handling interruption, old context: {old_context_id}")
await super()._handle_interruption(frame, direction)
if old_context_id and self._websocket:
logger.trace(f"{self}: Closing context {old_context_id} due to interruption")
async def _close_context(self, context_id: str):
if context_id and self._websocket:
logger.info(f"{self}: Closing context {context_id} due to interruption or completion")
try:
await self._send_close_context(old_context_id)
await self._send_close_context(context_id)
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
self._cumulative_time = 0.0
self._generation_end_time = 0.0
logger.trace(f"{self}: Interruption handled, context reset to None")
async def on_audio_context_interrupted(self, context_id: str):
"""Callback invoked when an audio context has been interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Callback invoked when an audio context has been completed."""
await self._close_context(context_id)
def _get_websocket(self):
"""Get the websocket for the Inworld WebSocket TTS service.
@@ -700,6 +745,21 @@ class InworldTTSService(AudioContextWordTTSService):
await self._disconnect_websocket()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to the Inworld WebSocket TTS service.
@@ -883,22 +943,29 @@ class InworldTTSService(AudioContextWordTTSService):
Args:
context_id: The context ID.
"""
audio_config = {
"audioEncoding": self._settings.audio_encoding,
"sampleRateHertz": self._settings.audio_sample_rate,
}
if self._settings.speaking_rate is not None:
audio_config["speakingRate"] = self._settings.speaking_rate
create_config: Dict[str, Any] = {
"voiceId": self._settings["voiceId"],
"modelId": self._settings["modelId"],
"audioConfig": self._settings["audioConfig"],
"voiceId": self._settings.voice,
"modelId": self._settings.model,
"audioConfig": audio_config,
}
if "temperature" in self._settings:
create_config["temperature"] = self._settings["temperature"]
if "applyTextNormalization" in self._settings:
create_config["applyTextNormalization"] = self._settings["applyTextNormalization"]
if "autoMode" in self._settings:
create_config["autoMode"] = self._settings["autoMode"]
if "timestampTransportStrategy" in self._settings:
create_config["timestampTransportStrategy"] = self._settings[
"timestampTransportStrategy"
]
if self._settings.temperature is not None:
create_config["temperature"] = self._settings.temperature
if self._settings.apply_text_normalization is not None:
create_config["applyTextNormalization"] = self._settings.apply_text_normalization
if self._settings.auto_mode is not None:
create_config["autoMode"] = self._settings.auto_mode
if self._settings.timestamp_transport_strategy is not None:
create_config["timestampTransportStrategy"] = (
self._settings.timestamp_transport_strategy
)
# Set buffer settings for timely audio generation.
# Use provided values or defaults that work well for streaming LLM output.

30
src/pipecat/services/kokoro/tts.py
View File

@@ -7,6 +7,7 @@
"""Kokoro TTS service implementation using kokoro-onnx."""
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import AsyncGenerator, Optional
@@ -22,6 +23,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -87,6 +89,17 @@ def language_to_kokoro_language(language: Language) -> str:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class KokoroTTSSettings(TTSSettings):
"""Settings for the Kokoro TTS service.
Parameters:
lang_code: Kokoro language code for synthesis.
"""
lang_code: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class KokoroTTSService(TTSService):
"""Kokoro TTS service implementation.
@@ -94,6 +107,8 @@ class KokoroTTSService(TTSService):
Automatically downloads model files on first use.
"""
_settings: KokoroTTSSettings
class InputParams(BaseModel):
"""Input parameters for Kokoro TTS configuration.
@@ -122,11 +137,18 @@ class KokoroTTSService(TTSService):
**kwargs: Additional arguments passed to parent `TTSService`.
"""
super().__init__(**kwargs)
params = params or KokoroTTSService.InputParams()
self._voice_id = voice_id
super().__init__(
settings=KokoroTTSSettings(
model=None,
voice=voice_id,
language=language_to_kokoro_language(params.language),
lang_code=language_to_kokoro_language(params.language),
),
**kwargs,
)
self._lang_code = language_to_kokoro_language(params.language)
model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx"
@@ -161,7 +183,7 @@ class KokoroTTSService(TTSService):
yield TTSStartedFrame(context_id=context_id)
stream = self._kokoro.create_stream(
text, voice=self._voice_id, lang=self._lang_code, speed=1.0
text, voice=self._settings.voice, lang=self._lang_code, speed=1.0
)
async for samples, sample_rate in stream:

82
src/pipecat/services/llm_service.py
View File

@@ -44,6 +44,7 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
UserImageRequestFrame,
)
@@ -58,8 +59,10 @@ from pipecat.processors.aggregators.llm_response import (
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import LLMSettings
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
from pipecat.utils.context.llm_context_summarization import (
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMContextSummarizationUtil,
)
@@ -172,12 +175,18 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
logger.info(f"Starting {len(function_calls)} function calls")
"""
_settings: LLMSettings
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
# However, subclasses should override this with a more specific adapter when necessary.
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
def __init__(
self, run_in_parallel: bool = True, function_call_timeout_secs: float = 10.0, **kwargs
self,
run_in_parallel: bool = True,
function_call_timeout_secs: float = 10.0,
settings: Optional[LLMSettings] = None,
**kwargs,
):
"""Initialize the LLM service.
@@ -186,10 +195,17 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
Defaults to True.
function_call_timeout_secs: Timeout in seconds for deferred function calls.
Defaults to 10.0 seconds.
settings: The runtime-updatable settings for the LLM service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or LLMSettings(),
**kwargs,
)
self._run_in_parallel = run_in_parallel
self._function_call_timeout_secs = function_call_timeout_secs
self._filter_incomplete_user_turns: bool = False
@@ -307,34 +323,30 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
await self._cancel_sequential_runner_task()
await self._cancel_summary_task()
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update LLM service settings.
Handles turn completion settings specially since they are not model
parameters and should not be passed to the underlying LLM API.
async def _update_settings(self, delta: LLMSettings) -> dict[str, Any]:
"""Apply a settings delta, handling turn-completion fields.
Args:
settings: Dictionary of settings to update.
"""
# Turn completion settings to extract (not model parameters)
turn_completion_keys = {"filter_incomplete_user_turns", "user_turn_completion_config"}
delta: An LLM settings delta.
# Handle turn completion settings
if "filter_incomplete_user_turns" in settings:
self._filter_incomplete_user_turns = settings["filter_incomplete_user_turns"]
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if "filter_incomplete_user_turns" in changed:
self._filter_incomplete_user_turns = (
self._settings.filter_incomplete_user_turns or False
)
logger.info(
f"{self}: Incomplete turn filtering {'enabled' if self._filter_incomplete_user_turns else 'disabled'}"
f"{self}: Incomplete turn filtering "
f"{'enabled' if self._filter_incomplete_user_turns else 'disabled'}"
)
# Configure the mixin with config object
if self._filter_incomplete_user_turns and "user_turn_completion_config" in settings:
self.set_user_turn_completion_config(settings["user_turn_completion_config"])
if "user_turn_completion_config" in changed and self._filter_incomplete_user_turns:
self.set_user_turn_completion_config(self._settings.user_turn_completion_config)
# Remove turn completion settings before passing to parent
settings = {k: v for k, v in settings.items() if k not in turn_completion_keys}
# Let the parent handle remaining model parameters
await super()._update_settings(settings)
return changed
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame.
@@ -349,6 +361,21 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
await self._handle_interruptions(frame)
elif isinstance(frame, LLMConfigureOutputFrame):
self._skip_tts = frame.skip_tts
elif isinstance(frame, LLMUpdateSettingsFrame):
if frame.delta is not None:
await self._update_settings(frame.delta)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via LLMUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.104, use LLMUpdateSettingsFrame(delta=LLMSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
delta = type(self._settings).from_mapping(frame.settings)
await self._update_settings(delta)
elif isinstance(frame, LLMContextSummaryRequestFrame):
await self._handle_summary_request(frame)
@@ -410,8 +437,15 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
last_index = -1
error = None
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try:
summary, last_index = await self._generate_summary(frame)
summary, last_index = await asyncio.wait_for(
self._generate_summary(frame),
timeout=timeout,
)
except asyncio.TimeoutError:
await self.push_error(error_msg=f"Context summarization timed out after {timeout}s")
except Exception as e:
error = f"Error generating context summary: {e}"
await self.push_error(error, exception=e)

54
src/pipecat/services/lmnt/tts.py
View File

@@ -7,7 +7,8 @@
"""LMNT text-to-speech service implementation."""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -71,6 +73,17 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class LmntTTSSettings(TTSSettings):
"""Settings for LMNT TTS service.
Parameters:
format: Audio output format. Defaults to "raw".
"""
format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class LmntTTSService(InterruptibleTTSService):
"""LMNT real-time text-to-speech service.
@@ -79,6 +92,8 @@ class LmntTTSService(InterruptibleTTSService):
language settings.
"""
_settings: LmntTTSSettings
def __init__(
self,
*,
@@ -103,16 +118,16 @@ class LmntTTSService(InterruptibleTTSService):
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
settings=LmntTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(language),
format="raw",
),
**kwargs,
)
self._api_key = api_key
self.set_voice(voice_id)
self.set_model_name(model)
self._settings = {
"language": self.language_to_service_language(language),
"format": "raw", # Use raw format for direct PCM data
}
self._receive_task = None
self._context_id: Optional[str] = None
@@ -190,6 +205,23 @@ class LmntTTSService(InterruptibleTTSService):
await self._disconnect_websocket()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Args:
delta: A :class:`TTSSettings` (or ``LmntTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if changed:
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to LMNT websocket."""
try:
@@ -201,11 +233,11 @@ class LmntTTSService(InterruptibleTTSService):
# Build initial connection message
init_msg = {
"X-API-Key": self._api_key,
"voice": self._voice_id,
"format": self._settings["format"],
"voice": self._settings.voice,
"format": self._settings.format,
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"model": self.model_name,
"language": self._settings.language,
"model": self._settings.model,
}
# Connect to LMNT's websocket directly

178
src/pipecat/services/minimax/tts.py
View File

@@ -11,7 +11,8 @@ for streaming text-to-speech synthesis.
"""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Mapping, Optional
import aiohttp
from loguru import logger
@@ -25,6 +26,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -85,6 +87,69 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class MiniMaxTTSSettings(TTSSettings):
"""Settings for MiniMax TTS service.
Parameters:
stream: Whether to use streaming mode.
speed: Speech speed (range: 0.5 to 2.0).
volume: Speech volume (range: 0 to 10).
pitch: Pitch adjustment (range: -12 to 12).
emotion: Emotional tone (options: "happy", "sad", "angry", "fearful",
"disgusted", "surprised", "calm", "fluent").
text_normalization: Enable text normalization (Chinese/English).
latex_read: Enable LaTeX formula reading.
audio_bitrate: Audio bitrate in bps.
audio_format: Audio output format.
audio_channel: Number of audio channels.
audio_sample_rate: Audio sample rate in Hz.
language_boost: Language boost string for multilingual support.
"""
stream: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emotion: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
text_normalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
latex_read: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_bitrate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_channel: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_boost: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "MiniMaxTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested dicts.
Handles ``voice_setting`` (with ``vol`` → ``volume`` rename) and
``audio_setting`` (with prefixed field mapping).
"""
flat = dict(settings)
voice = flat.pop("voice_setting", None)
if isinstance(voice, dict):
flat.setdefault("speed", voice.get("speed"))
flat.setdefault("volume", voice.get("vol"))
flat.setdefault("pitch", voice.get("pitch"))
flat.setdefault("emotion", voice.get("emotion"))
flat.setdefault("text_normalization", voice.get("text_normalization"))
flat.setdefault("latex_read", voice.get("latex_read"))
audio = flat.pop("audio_setting", None)
if isinstance(audio, dict):
flat.setdefault("audio_bitrate", audio.get("bitrate"))
flat.setdefault("audio_format", audio.get("format"))
flat.setdefault("audio_channel", audio.get("channel"))
flat.setdefault("audio_sample_rate", audio.get("sample_rate"))
return super().from_mapping(flat)
class MiniMaxHttpTTSService(TTSService):
"""Text-to-speech service using MiniMax's T2A (Text-to-Audio) API.
@@ -96,6 +161,8 @@ class MiniMaxHttpTTSService(TTSService):
https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643
"""
_settings: MiniMaxTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for MiniMax TTS.
@@ -160,41 +227,40 @@ class MiniMaxHttpTTSService(TTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or MiniMaxHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=MiniMaxTTSSettings(
model=model,
voice=voice_id,
language=None,
stream=True,
speed=params.speed,
volume=params.volume,
pitch=params.pitch,
language_boost=None,
emotion=None,
text_normalization=None,
latex_read=None,
audio_bitrate=128000,
audio_format="pcm",
audio_channel=1,
audio_sample_rate=0,
),
**kwargs,
)
self._api_key = api_key
self._group_id = group_id
self._base_url = f"{base_url}?GroupId={group_id}"
self._session = aiohttp_session
self._model_name = model
self._voice_id = voice_id
# Create voice settings
self._settings = {
"stream": True,
"voice_setting": {
"speed": params.speed,
"vol": params.volume,
"pitch": params.pitch,
},
"audio_setting": {
"bitrate": 128000,
"format": "pcm",
"channel": 1,
},
}
# Set voice and model
self.set_voice(voice_id)
self.set_model_name(model)
# Add language boost if provided
if params.language:
service_lang = self.language_to_service_language(params.language)
if service_lang:
self._settings["language_boost"] = service_lang
self._settings.language_boost = service_lang
# Add optional emotion if provided
if params.emotion:
@@ -210,7 +276,7 @@ class MiniMaxHttpTTSService(TTSService):
"fluent",
]
if params.emotion in supported_emotions:
self._settings["voice_setting"]["emotion"] = params.emotion
self._settings.emotion = params.emotion
else:
logger.warning(
f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}"
@@ -226,15 +292,15 @@ class MiniMaxHttpTTSService(TTSService):
"Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.",
DeprecationWarning,
)
self._settings["voice_setting"]["text_normalization"] = params.english_normalization
self._settings.text_normalization = params.english_normalization
# Add text_normalization if provided (corrected parameter name)
if params.text_normalization is not None:
self._settings["voice_setting"]["text_normalization"] = params.text_normalization
self._settings.text_normalization = params.text_normalization
# Add latex_read if provided
if params.latex_read is not None:
self._settings["voice_setting"]["latex_read"] = params.latex_read
self._settings.latex_read = params.latex_read
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -255,24 +321,6 @@ class MiniMaxHttpTTSService(TTSService):
"""
return language_to_minimax_language(language)
def set_model_name(self, model: str):
"""Set the TTS model to use.
Args:
model: The model name to use for synthesis.
"""
self._model_name = model
def set_voice(self, voice: str):
"""Set the voice to use.
Args:
voice: The voice identifier to use for synthesis.
"""
self._voice_id = voice
if "voice_setting" in self._settings:
self._settings["voice_setting"]["voice_id"] = voice
async def start(self, frame: StartFrame):
"""Start the MiniMax TTS service.
@@ -280,7 +328,7 @@ class MiniMaxHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["audio_setting"]["sample_rate"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}")
@traced_tts
@@ -302,10 +350,38 @@ class MiniMaxHttpTTSService(TTSService):
"Authorization": f"Bearer {self._api_key}",
}
# Build voice_setting dict for API
voice_setting = {
"voice_id": self._settings.voice,
"speed": self._settings.speed,
"vol": self._settings.volume,
"pitch": self._settings.pitch,
}
if self._settings.emotion is not None:
voice_setting["emotion"] = self._settings.emotion
if self._settings.text_normalization is not None:
voice_setting["text_normalization"] = self._settings.text_normalization
if self._settings.latex_read is not None:
voice_setting["latex_read"] = self._settings.latex_read
# Build audio_setting dict for API
audio_setting = {
"bitrate": self._settings.audio_bitrate,
"format": self._settings.audio_format,
"channel": self._settings.audio_channel,
"sample_rate": self._settings.audio_sample_rate,
}
# Create payload from settings
payload = self._settings.copy()
payload["model"] = self._model_name
payload["text"] = text
payload = {
"stream": self._settings.stream,
"voice_setting": voice_setting,
"audio_setting": audio_setting,
"model": self._settings.model,
"text": text,
}
if self._settings.language_boost is not None:
payload["language_boost"] = self._settings.language_boost
try:
await self.start_ttfb_metrics()

18
src/pipecat/services/mistral/llm.py
View File

@@ -180,24 +180,24 @@ class MistralLLMService(OpenAILLMService):
fixed_messages = self._apply_mistral_fixups(params_from_context["messages"])
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"messages": fixed_messages,
"tools": params_from_context["tools"],
"tool_choice": params_from_context["tool_choice"],
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Handle Mistral-specific parameter mapping
# Mistral uses "random_seed" instead of "seed"
if self._settings["seed"]:
params["random_seed"] = self._settings["seed"]
if self._settings.seed:
params["random_seed"] = self._settings.seed
# Add any extra parameters
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

15
src/pipecat/services/moondream/vision.py
View File

@@ -11,6 +11,7 @@ for image analysis and description generation.
"""
import asyncio
from dataclasses import dataclass
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
VisionFullResponseStartFrame,
VisionTextFrame,
)
from pipecat.services.settings import VisionSettings
from pipecat.services.vision_service import VisionService
try:
@@ -60,6 +62,15 @@ def detect_device():
return torch.device("cpu"), torch.float32
@dataclass
class MoondreamSettings(VisionSettings):
"""Settings for the Moondream vision service.
Parameters:
model: Moondream model identifier.
"""
class MoondreamService(VisionService):
"""Moondream vision-language model service.
@@ -79,9 +90,7 @@ class MoondreamService(VisionService):
use_cpu: Whether to force CPU usage instead of hardware acceleration.
**kwargs: Additional arguments passed to the parent VisionService.
"""
super().__init__(**kwargs)
self.set_model_name(model)
super().__init__(settings=MoondreamSettings(model=model), **kwargs)
if not use_cpu:
device, dtype = detect_device()

112
src/pipecat/services/neuphonic/tts.py
View File

@@ -13,7 +13,8 @@ text-to-speech API for real-time audio synthesis.
import asyncio
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -34,7 +35,8 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -72,6 +74,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class NeuphonicTTSSettings(TTSSettings):
"""Settings for Neuphonic TTS service.
Parameters:
speed: Speech speed multiplier. Defaults to 1.0.
encoding: Audio encoding format.
sampling_rate: Audio sample rate.
"""
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sampling_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NeuphonicTTSService(InterruptibleTTSService):
"""Neuphonic real-time text-to-speech service using WebSocket streaming.
@@ -80,6 +97,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
parameters for high-quality speech generation.
"""
_settings: NeuphonicTTSSettings
class InputParams(BaseModel):
"""Input parameters for Neuphonic TTS configuration.
@@ -100,7 +119,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
sample_rate: Optional[int] = 22050,
encoding: str = "pcm_linear",
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize the Neuphonic TTS service.
@@ -112,28 +132,35 @@ class NeuphonicTTSService(InterruptibleTTSService):
sample_rate: Audio sample rate in Hz. Defaults to 22050.
encoding: Audio encoding format. Defaults to "pcm_linear".
params: Additional input parameters for TTS configuration.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
params = params or NeuphonicTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_stop_frames=True,
stop_frame_timeout_s=2.0,
sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
language=self.language_to_service_language(params.language),
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
voice=voice_id,
),
**kwargs,
)
params = params or NeuphonicTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = {
"lang_code": self.language_to_service_language(params.language),
"speed": params.speed,
"encoding": encoding,
"sampling_rate": sample_rate,
}
self.set_voice(voice_id)
self._cumulative_time = 0
@@ -160,15 +187,14 @@ class NeuphonicTTSService(InterruptibleTTSService):
"""
return language_to_neuphonic_lang_code(language)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect with new configuration."""
if "voice_id" in settings:
self.set_voice(settings["voice_id"])
await super()._update_settings(settings)
await self._disconnect()
await self._connect()
logger.info(f"Switching TTS to settings: [{self._settings}]")
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect with new configuration."""
changed = await super()._update_settings(delta)
if changed:
await self._disconnect()
await self._connect()
logger.info(f"Switching TTS to settings: [{self._settings}]")
return changed
async def start(self, frame: StartFrame):
"""Start the Neuphonic TTS service.
@@ -266,8 +292,11 @@ class NeuphonicTTSService(InterruptibleTTSService):
logger.debug("Connecting to Neuphonic")
tts_config = {
**self._settings,
"voice_id": self._voice_id,
"lang_code": self._settings.language,
"speed": self._settings.speed,
"encoding": self._settings.encoding,
"sampling_rate": self._settings.sampling_rate,
"voice_id": self._settings.voice,
}
query_params = []
@@ -275,7 +304,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
if value is not None:
query_params.append(f"{key}={value}")
url = f"{self._url}/speak/{self._settings['lang_code']}"
url = f"{self._url}/speak/{self._settings.language}"
if query_params:
url += f"?{'&'.join(query_params)}"
@@ -384,6 +413,8 @@ class NeuphonicHttpTTSService(TTSService):
HTTP-based communication over WebSocket connections.
"""
_settings: NeuphonicTTSSettings
class InputParams(BaseModel):
"""Input parameters for Neuphonic HTTP TTS configuration.
@@ -419,17 +450,24 @@ class NeuphonicHttpTTSService(TTSService):
params: Additional input parameters for TTS configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NeuphonicHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(params.language) or "en",
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = url.rstrip("/")
self._lang_code = self.language_to_service_language(params.language) or "en"
self._speed = params.speed
self._encoding = encoding
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -513,7 +551,7 @@ class NeuphonicHttpTTSService(TTSService):
"""
logger.debug(f"Generating TTS: [{text}]")
url = f"{self._base_url}/sse/speak/{self._lang_code}"
url = f"{self._base_url}/sse/speak/{self._settings.language}"
headers = {
"X-API-KEY": self._api_key,
@@ -522,14 +560,14 @@ class NeuphonicHttpTTSService(TTSService):
payload = {
"text": text,
"lang_code": self._lang_code,
"encoding": self._encoding,
"lang_code": self._settings.language,
"encoding": self._settings.encoding,
"sampling_rate": self.sample_rate,
"speed": self._speed,
"speed": self._settings.speed,
}
if self._voice_id:
payload["voice_id"] = self._voice_id
if self._settings.voice:
payload["voice_id"] = self._settings.voice
try:
await self.start_ttfb_metrics()

208
src/pipecat/services/nvidia/stt.py
View File

@@ -8,7 +8,8 @@
import asyncio
from concurrent.futures import CancelledError as FuturesCancelledError
from typing import AsyncGenerator, List, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, List, Mapping, Optional
from loguru import logger
from pydantic import BaseModel
@@ -22,6 +23,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import NVIDIA_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService, STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -89,6 +91,32 @@ def language_to_nvidia_riva_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class NvidiaSTTSettings(STTSettings):
"""Settings for the NVIDIA Riva streaming STT service."""
pass
@dataclass
class NvidiaSegmentedSTTSettings(STTSettings):
"""Settings for the NVIDIA Riva segmented STT service.
Parameters:
profanity_filter: Whether to filter profanity from results.
automatic_punctuation: Whether to add automatic punctuation.
verbatim_transcripts: Whether to return verbatim transcripts.
boosted_lm_words: List of words to boost in language model.
boosted_lm_score: Score boost for specified words.
"""
profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
verbatim_transcripts: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
boosted_lm_words: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
boosted_lm_score: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NvidiaSTTService(STTService):
"""Real-time speech-to-text service using NVIDIA Riva streaming ASR.
@@ -97,6 +125,8 @@ class NvidiaSTTService(STTService):
processing for low-latency applications.
"""
_settings: NvidiaSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for NVIDIA Riva STT service.
@@ -134,19 +164,21 @@ class NvidiaSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSTTSettings(
model=model_function_map.get("model_name"),
language=params.language,
),
**kwargs,
)
self._server = server
self._api_key = api_key
self._use_ssl = use_ssl
self._profanity_filter = False
self._automatic_punctuation = True
self._no_verbatim_transcripts = False
self._language_code = params.language
self._boosted_lm_words = None
self._boosted_lm_score = 4.0
self._start_history = -1
self._start_threshold = -1.0
self._stop_history = -1
@@ -156,17 +188,6 @@ class NvidiaSTTService(STTService):
self._custom_configuration = ""
self._function_id = model_function_map.get("function_id")
self._settings = {
"language": str(params.language),
"profanity_filter": self._profanity_filter,
"automatic_punctuation": self._automatic_punctuation,
"verbatim_transcripts": not self._no_verbatim_transcripts,
"boosted_lm_words": self._boosted_lm_words,
"boosted_lm_score": self._boosted_lm_score,
}
self.set_model_name(model_function_map.get("model_name"))
self._asr_service = None
self._queue = None
self._config = None
@@ -186,22 +207,18 @@ class NvidiaSTTService(STTService):
config = riva.client.StreamingRecognitionConfig(
config=riva.client.RecognitionConfig(
encoding=riva.client.AudioEncoding.LINEAR_PCM,
language_code=self._language_code,
language_code=self._settings.language,
model="",
max_alternatives=1,
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=not self._no_verbatim_transcripts,
profanity_filter=False,
enable_automatic_punctuation=True,
verbatim_transcripts=True,
sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
interim_results=True,
)
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
)
riva.client.add_endpoint_parameters_to_config(
config,
self._start_history,
@@ -226,18 +243,31 @@ class NvidiaSTTService(STTService):
async def set_model(self, model: str):
"""Set the ASR model for transcription.
.. deprecated:: 0.0.104
Model cannot be changed after initialization for NVIDIA Riva streaming STT.
Set model and function id in the constructor instead, e.g.::
NvidiaSTTService(
api_key=...,
model_function_map={"function_id": "<UUID>", "model_name": "<model_name>"},
)
Args:
model: Model name to set.
Note:
Model cannot be changed after initialization. Use model_function_map
parameter in constructor instead.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated. Model cannot be changed after initialization"
" for NVIDIA Riva streaming STT. Set model and function id in the"
" constructor instead, e.g.:"
" NvidiaSTTService(api_key=..., model_function_map="
"{'function_id': '<UUID>', 'model_name': '<model_name>'})",
DeprecationWarning,
stacklevel=2,
)
async def start(self, frame: StartFrame):
"""Start the NVIDIA Riva STT service and initialize streaming configuration.
@@ -254,7 +284,7 @@ class NvidiaSTTService(STTService):
if not self._thread_task:
self._thread_task = self.create_task(self._thread_task_handler())
logger.debug(f"Initialized NvidiaSTTService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaSTTService with model: {self._settings.model}")
async def stop(self, frame: EndFrame):
"""Stop the NVIDIA Riva STT service and clean up resources.
@@ -318,14 +348,14 @@ class NvidiaSTTService(STTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language_code,
self._settings.language,
result=result,
)
)
await self._handle_transcription(
transcript=transcript,
is_final=result.is_final,
language=self._language_code,
language=self._settings.language,
)
else:
await self.push_frame(
@@ -333,7 +363,7 @@ class NvidiaSTTService(STTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language_code,
self._settings.language,
result=result,
)
)
@@ -386,6 +416,8 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
audio buffering and speech detection.
"""
_settings: NvidiaSegmentedSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for NVIDIA Riva segmented STT service.
@@ -433,30 +465,29 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSegmentedSTTService.InputParams()
# Set model name
self.set_model_name(model_function_map.get("model_name"))
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSegmentedSTTSettings(
model=model_function_map.get("model_name"),
language=self.language_to_service_language(params.language or Language.EN_US)
or "en-US",
profanity_filter=params.profanity_filter,
automatic_punctuation=params.automatic_punctuation,
verbatim_transcripts=params.verbatim_transcripts,
boosted_lm_words=params.boosted_lm_words,
boosted_lm_score=params.boosted_lm_score,
),
**kwargs,
)
# Initialize NVIDIA Riva settings
self._api_key = api_key
self._server = server
self._use_ssl = use_ssl
self._function_id = model_function_map.get("function_id")
self._model_name = model_function_map.get("model_name")
# Store the language as a Language enum and as a string
self._language_enum = params.language or Language.EN_US
self._language = self.language_to_service_language(self._language_enum) or "en-US"
# Configure transcription parameters
self._profanity_filter = params.profanity_filter
self._automatic_punctuation = params.automatic_punctuation
self._verbatim_transcripts = params.verbatim_transcripts
self._boosted_lm_words = params.boosted_lm_words
self._boosted_lm_score = params.boosted_lm_score
# Voice activity detection thresholds (use NVIDIA Riva defaults)
self._start_history = -1
@@ -467,10 +498,8 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
self._stop_threshold_eou = -1.0
self._custom_configuration = ""
# Create NVIDIA Riva client
self._config = None
self._asr_service = None
self._settings = {"language": self._language_enum}
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to NVIDIA Riva's language code.
@@ -498,21 +527,25 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
auth = riva.client.Auth(None, self._use_ssl, self._server, metadata)
self._asr_service = riva.client.ASRService(auth)
def _get_language_code(self) -> str:
"""Get the current NVIDIA Riva language code string."""
return self._settings.language or "en-US"
def _create_recognition_config(self):
"""Create the NVIDIA Riva ASR recognition configuration."""
# Create base configuration
config = riva.client.RecognitionConfig(
language_code=self._language, # Now using the string, not a tuple
language_code=self._get_language_code(),
max_alternatives=1,
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=self._verbatim_transcripts,
profanity_filter=self._settings.profanity_filter,
enable_automatic_punctuation=self._settings.automatic_punctuation,
verbatim_transcripts=self._settings.verbatim_transcripts,
)
# Add word boosting if specified
if self._boosted_lm_words:
if self._settings.boosted_lm_words:
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
config, self._settings.boosted_lm_words, self._settings.boosted_lm_score
)
# Add voice activity detection parameters
@@ -540,22 +573,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
"""
return True
async def set_model(self, model: str):
"""Set the ASR model for transcription.
Args:
model: Model name to set.
Note:
Model cannot be changed after initialization. Use model_function_map
parameter in constructor instead.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
async def start(self, frame: StartFrame):
"""Initialize the service when the pipeline starts.
@@ -565,22 +582,23 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
await super().start(frame)
self._initialize_client()
self._config = self._create_recognition_config()
logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self._settings.model}")
async def set_language(self, language: Language):
"""Set the language for the STT service.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and sync internal state.
Args:
language: Target language for transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._language_enum = language
self._language = self.language_to_service_language(language) or "en-US"
self._settings["language"] = language
delta: A :class:`STTSettings` (or ``NvidiaSegmentedSTTSettings``) delta.
# Update configuration with new language
if self._config:
self._config.language_code = self._language
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if changed:
self._config = self._create_recognition_config()
return changed
@traced_stt
async def _handle_transcription(
@@ -633,11 +651,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
text,
self._user_id,
time_now_iso8601(),
self._language_enum,
self._settings.language,
)
transcription_found = True
await self._handle_transcription(text, True, self._language_enum)
await self._handle_transcription(text, True, self._settings.language)
if not transcription_found:
logger.debug(f"{self}: No transcription results found in NVIDIA Riva response")

85
src/pipecat/services/nvidia/tts.py
View File

@@ -12,7 +12,8 @@ gRPC API for high-quality speech synthesis.
import asyncio
import os
from typing import AsyncGenerator, AsyncIterator, Generator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, AsyncIterator, Generator, Mapping, Optional
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -30,6 +31,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
@@ -42,6 +44,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class NvidiaTTSSettings(TTSSettings):
"""Settings for NVIDIA Riva TTS service.
Parameters:
quality: Audio quality setting (0-100).
"""
quality: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NvidiaTTSService(TTSService):
"""NVIDIA Riva text-to-speech service.
@@ -50,6 +63,8 @@ class NvidiaTTSService(TTSService):
configurable quality settings.
"""
_settings: NvidiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Riva TTS configuration.
@@ -88,36 +103,66 @@ class NvidiaTTSService(TTSService):
use_ssl: Whether to use SSL for the NVIDIA Riva server. Defaults to True.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NvidiaTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NvidiaTTSSettings(
model=model_function_map.get("model_name"),
voice=voice_id,
language=params.language,
quality=params.quality,
),
**kwargs,
)
self._server = server
self._api_key = api_key
self._voice_id = voice_id
self._language_code = params.language
self._quality = params.quality
self._function_id = model_function_map.get("function_id")
self._use_ssl = use_ssl
self.set_model_name(model_function_map.get("model_name"))
self.set_voice(voice_id)
self._service = None
self._config = None
async def set_model(self, model: str):
"""Attempt to set the TTS model.
"""Set the TTS model.
Note: Model cannot be changed after initialization for Riva service.
.. deprecated:: 0.0.104
Model cannot be changed after initialization for NVIDIA Riva TTS.
Set model and function id in the constructor instead, e.g.::
NvidiaTTSService(
api_key=...,
model_function_map={"function_id": "<UUID>", "model_name": "<model_name>"},
)
Args:
model: The model name to set (operation not supported).
model: The model name to set.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated. Model cannot be changed after initialization"
" for NVIDIA Riva TTS. Set model and function id in the constructor"
" instead, e.g.: NvidiaTTSService(api_key=..., model_function_map="
"{'function_id': '<UUID>', 'model_name': '<model_name>'})",
DeprecationWarning,
stacklevel=2,
)
async def _update_settings(self, delta: NvidiaTTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: reconnect gRPC client to apply changed settings.
self._warn_unhandled_updated_settings(changed)
return changed
def _initialize_client(self):
if self._service is not None:
@@ -150,7 +195,7 @@ class NvidiaTTSService(TTSService):
await super().start(frame)
self._initialize_client()
self._config = self._create_synthesis_config()
logger.debug(f"Initialized NvidiaTTSService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaTTSService with model: {self._settings.model}")
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -167,11 +212,11 @@ class NvidiaTTSService(TTSService):
def read_audio_responses() -> Generator[rtts.SynthesizeSpeechResponse, None, None]:
responses = self._service.synthesize_online(
text,
self._voice_id,
self._language_code,
self._settings.voice,
self._settings.language,
sample_rate_hz=self.sample_rate,
zero_shot_audio_prompt_file=None,
zero_shot_quality=self._quality,
zero_shot_quality=self._settings.quality,
custom_dictionary={},
)
return responses

75
src/pipecat/services/openai/base_llm.py
View File

@@ -10,7 +10,8 @@ import asyncio
import base64
import json
from contextlib import asynccontextmanager
from typing import Any, Dict, List, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Mapping, Optional
import httpx
from loguru import logger
@@ -32,7 +33,6 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -42,9 +42,24 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
from pipecat.services.settings import LLMSettings, _NotGiven
from pipecat.utils.tracing.service_decorators import traced_llm
@dataclass
class OpenAILLMSettings(LLMSettings):
"""Settings for OpenAI-compatible LLM services.
Parameters:
max_completion_tokens: Maximum completion tokens to generate.
service_tier: Service tier to use (e.g., "auto", "flex", "priority").
"""
max_completion_tokens: int | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
service_tier: str | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
class BaseOpenAILLMService(LLMService):
"""Base class for all services that use the AsyncOpenAI client.
@@ -55,6 +70,8 @@ class BaseOpenAILLMService(LLMService):
configurations.
"""
_settings: OpenAILLMSettings
class InputParams(BaseModel):
"""Input parameters for OpenAI model configuration.
@@ -116,24 +133,28 @@ class BaseOpenAILLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs)
params = params or BaseOpenAILLMService.InputParams()
self._settings = {
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,
"seed": params.seed,
"temperature": params.temperature,
"top_p": params.top_p,
"max_tokens": params.max_tokens,
"max_completion_tokens": params.max_completion_tokens,
"service_tier": params.service_tier,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
super().__init__(
settings=OpenAILLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
presence_penalty=params.presence_penalty,
seed=params.seed,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
max_tokens=params.max_tokens,
max_completion_tokens=params.max_completion_tokens,
service_tier=params.service_tier,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self.set_model_name(model)
self._full_model_name: str = ""
self._client = self.create_client(
api_key=api_key,
@@ -247,23 +268,23 @@ class BaseOpenAILLMService(LLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"service_tier": self._settings["service_tier"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"seed": self._settings.seed,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
"max_completion_tokens": self._settings.max_completion_tokens,
"service_tier": self._settings.service_tier,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params
async def run_inference(
@@ -517,8 +538,6 @@ class BaseOpenAILLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = OpenAILLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

16
src/pipecat/services/openai/image.py
View File

@@ -11,6 +11,7 @@ for creating images from text prompts.
"""
import io
from dataclasses import dataclass
from typing import AsyncGenerator, Literal, Optional
import aiohttp
@@ -24,6 +25,16 @@ from pipecat.frames.frames import (
URLImageRawFrame,
)
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class OpenAIImageGenSettings(ImageGenSettings):
"""Settings for the OpenAI image generation service.
Parameters:
model: DALL-E model identifier.
"""
class OpenAIImageGenService(ImageGenService):
@@ -52,8 +63,7 @@ class OpenAIImageGenService(ImageGenService):
image_size: Target size for generated images.
model: DALL-E model to use for generation. Defaults to "dall-e-3".
"""
super().__init__()
self.set_model_name(model)
super().__init__(settings=OpenAIImageGenSettings(model=model))
self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
self._aiohttp_session = aiohttp_session
@@ -70,7 +80,7 @@ class OpenAIImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
image = await self._client.images.generate(
prompt=prompt, model=self.model_name, n=1, size=self._image_size
prompt=prompt, model=self._settings.model, n=1, size=self._image_size
)
image_url = image.data[0].url

104
src/pipecat/services/openai/realtime/llm.py
View File

@@ -10,8 +10,8 @@ import base64
import io
import json
import time
from dataclasses import dataclass
from typing import Optional
from dataclasses import dataclass, field
from typing import Any, Optional
from loguru import logger
from PIL import Image
@@ -59,6 +59,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt
@@ -90,6 +91,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class OpenAIRealtimeLLMSettings(LLMSettings):
"""Settings for OpenAI Realtime LLM services.
Parameters:
session_properties: OpenAI Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class OpenAIRealtimeLLMService(LLMService):
"""OpenAI Realtime LLM service providing real-time audio and text communication.
@@ -98,6 +112,8 @@ class OpenAIRealtimeLLMService(LLMService):
management, and real-time transcription.
"""
_settings: OpenAIRealtimeLLMSettings
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
adapter_class = OpenAIRealtimeLLMAdapter
@@ -105,7 +121,7 @@ class OpenAIRealtimeLLMService(LLMService):
self,
*,
api_key: str,
model: str = "gpt-realtime",
model: str = "gpt-realtime-1.5",
base_url: str = "wss://api.openai.com/v1/realtime",
session_properties: Optional[events.SessionProperties] = None,
start_audio_paused: bool = False,
@@ -155,16 +171,26 @@ class OpenAIRealtimeLLMService(LLMService):
# Build WebSocket URL with model query parameter
# Source: https://platform.openai.com/docs/guides/realtime-websocket
full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs)
super().__init__(
base_url=full_url,
settings=OpenAIRealtimeLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = full_url
self.set_model_name(model)
# Initialize session_properties
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused
self._video_frame_detail = video_frame_detail
@@ -227,12 +253,12 @@ class OpenAIRealtimeLLMService(LLMService):
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
modalities = self._session_properties.output_modalities or ["audio", "text"]
modalities = self._settings.session_properties.output_modalities or ["audio", "text"]
return modality in modalities
def _get_enabled_modalities(self) -> list[str]:
"""Get the list of enabled modalities."""
modalities = self._session_properties.output_modalities or ["audio", "text"]
modalities = self._settings.session_properties.output_modalities or ["audio", "text"]
# API only supports single modality responses: either ["text"] or ["audio"]
if "audio" in modalities:
return ["audio"]
@@ -305,9 +331,9 @@ class OpenAIRealtimeLLMService(LLMService):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
turn_detection_disabled = (
self._session_properties.audio
and self._session_properties.audio.input
and self._session_properties.audio.input.turn_detection is False
self._settings.session_properties.audio
and self._settings.session_properties.audio.input
and self._settings.session_properties.audio.input.turn_detection is False
)
if turn_detection_disabled:
await self.send_client_event(events.InputAudioBufferClearEvent())
@@ -327,9 +353,9 @@ class OpenAIRealtimeLLMService(LLMService):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
turn_detection_disabled = (
self._session_properties.audio
and self._session_properties.audio.input
and self._session_properties.audio.input.turn_detection is False
self._settings.session_properties.audio
and self._settings.session_properties.audio.input
and self._settings.session_properties.audio.input.turn_detection is False
)
if turn_detection_disabled:
await self.send_client_event(events.InputAudioBufferCommitEvent())
@@ -397,6 +423,16 @@ class OpenAIRealtimeLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.delta path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.delta is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -424,11 +460,8 @@ class OpenAIRealtimeLLMService(LLMService):
await self._handle_bot_stopped_speaking()
elif isinstance(frame, LLMMessagesAppendFrame):
await self._handle_messages_append(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
await self.push_frame(frame, direction)
@@ -513,8 +546,16 @@ class OpenAIRealtimeLLMService(LLMService):
# treat a send-side error as fatal.
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
settings = self._session_properties
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update if needed."""
changed = await super()._update_settings(delta)
if "session_properties" in changed:
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"})
return changed
async def _send_session_update(self):
settings = self._settings.session_properties
adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
@@ -577,15 +618,18 @@ class OpenAIRealtimeLLMService(LLMService):
await self._handle_evt_function_call_arguments_done(evt)
elif evt.type == "error":
if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt):
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
if evt.error.code == "response_cancel_not_active":
logger.debug(f"{self} {evt.error.message}")
else:
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
@traced_openai_realtime(operation="llm_setup")
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
# to configure the session properties.
await self._update_settings()
await self._send_session_update()
async def _handle_evt_session_updated(self, evt):
# If this is our first context frame, run the LLM
@@ -795,7 +839,7 @@ class OpenAIRealtimeLLMService(LLMService):
async def _handle_evt_speech_started(self, evt):
await self._truncate_current_audio_response()
await self.broadcast_frame(UserStartedSpeakingFrame)
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
async def _handle_evt_speech_stopped(self, evt):
await self.start_ttfb_metrics()
@@ -868,7 +912,7 @@ class OpenAIRealtimeLLMService(LLMService):
await self.send_client_event(evt)
# Send new settings if needed
await self._update_settings()
await self._send_session_update()
# We're done configuring the LLM for this session
self._llm_needs_conversation_setup = False

73
src/pipecat/services/openai/stt.py
View File

@@ -16,7 +16,8 @@ Provides two STT services:
import base64
import json
from typing import AsyncGenerator, Literal, Optional, Union
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Literal, Optional, Union
from loguru import logger
@@ -34,6 +35,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription
@@ -98,24 +100,24 @@ class OpenAISTTService(BaseWhisperSTTService):
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"language": self._language,
"model": self._settings.model,
"language": self._settings.language,
}
if self._include_prob_metrics:
# GPT-4o-transcribe models only support logprobs (not verbose_json)
if self.model_name in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"):
if self._settings.model in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"):
kwargs["response_format"] = "json"
kwargs["include"] = ["logprobs"]
else:
# Whisper models support verbose_json
kwargs["response_format"] = "verbose_json"
if self._prompt is not None:
kwargs["prompt"] = self._prompt
if self._settings.prompt is not None:
kwargs["prompt"] = self._settings.prompt
if self._temperature is not None:
kwargs["temperature"] = self._temperature
if self._settings.temperature is not None:
kwargs["temperature"] = self._settings.temperature
return await self._client.audio.transcriptions.create(**kwargs)
@@ -123,6 +125,17 @@ class OpenAISTTService(BaseWhisperSTTService):
_OPENAI_SAMPLE_RATE = 24000
@dataclass
class OpenAIRealtimeSTTSettings(STTSettings):
"""Settings for the OpenAI Realtime STT service.
Parameters:
prompt: Optional prompt text to guide transcription style.
"""
prompt: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class OpenAIRealtimeSTTService(WebsocketSTTService):
"""OpenAI Realtime Speech-to-Text service using WebSocket transcription sessions.
@@ -156,6 +169,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
)
"""
_settings: OpenAIRealtimeSTTSettings
def __init__(
self,
*,
@@ -206,14 +221,17 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=OpenAIRealtimeSTTSettings(
model=model,
language=language,
prompt=prompt,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self.set_model_name(model)
self._language_code = self._language_to_code(language) if language else None
self._prompt = prompt
self._turn_detection = turn_detection
self._noise_reduction = noise_reduction
@@ -248,19 +266,31 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the language for speech recognition.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and send session update if needed.
If the session is already active, sends an updated configuration
to the server.
Keeps ``_language_code`` and ``_prompt`` in sync with settings
and sends a ``session.update`` to the server when the session is active.
Args:
language: The language to use for speech recognition.
delta: A :class:`STTSettings` (or ``OpenAIRealtimeSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
self._language_code = self._language_to_code(language)
changed = await super()._update_settings(delta)
if not changed:
return changed
if "prompt" in changed and isinstance(self._settings, OpenAIRealtimeSTTSettings):
self._prompt = self._settings.prompt
if self._session_ready:
await self._send_session_update()
return changed
async def start(self, frame: StartFrame):
"""Start the service and establish WebSocket connection.
@@ -405,10 +435,13 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
async def _send_session_update(self):
"""Send ``session.update`` to configure the transcription session."""
transcription: dict = {"model": self.model_name}
transcription: dict = {"model": self._settings.model}
if self._language_code:
transcription["language"] = self._language_code
language_code = (
self._language_to_code(self._settings.language) if self._settings.language else None
)
if language_code:
transcription["language"] = language_code
if self._prompt:
transcription["prompt"] = self._prompt
@@ -606,7 +639,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
logger.debug("Server VAD: speech started")
await self.broadcast_frame(UserStartedSpeakingFrame)
if self._should_interrupt:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
await self.start_processing_metrics()
async def _handle_speech_stopped(self, evt: dict):

60
src/pipecat/services/openai/tts.py
View File

@@ -10,6 +10,7 @@ This module provides integration with OpenAI's text-to-speech API for
generating high-quality synthetic speech from text input.
"""
from dataclasses import dataclass, field
from typing import AsyncGenerator, Dict, Literal, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -60,6 +62,19 @@ VALID_VOICES: Dict[str, ValidVoice] = {
}
@dataclass
class OpenAITTSSettings(TTSSettings):
"""Settings for OpenAI TTS service.
Parameters:
instructions: Instructions to guide voice synthesis behavior.
speed: Voice speed control (0.25 to 4.0, default 1.0).
"""
instructions: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class OpenAITTSService(TTSService):
"""OpenAI Text-to-Speech service that generates audio from text.
@@ -68,6 +83,8 @@ class OpenAITTSService(TTSService):
speech synthesis with streaming audio output.
"""
_settings: OpenAITTSSettings
OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz
class InputParams(BaseModel):
@@ -115,12 +132,6 @@ class OpenAITTSService(TTSService):
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
self.set_model_name(model)
self.set_voice(voice)
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
if instructions or speed:
import warnings
@@ -132,10 +143,18 @@ class OpenAITTSService(TTSService):
stacklevel=2,
)
self._settings = {
"instructions": params.instructions if params else instructions,
"speed": params.speed if params else speed,
}
super().__init__(
sample_rate=sample_rate,
settings=OpenAITTSSettings(
model=model,
voice=voice,
instructions=params.instructions if params else instructions,
speed=params.speed if params else speed,
),
**kwargs,
)
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -145,15 +164,6 @@ class OpenAITTSService(TTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model to use.
Args:
model: The model name to use for text-to-speech synthesis.
"""
logger.info(f"Switching TTS model to: [{model}]")
self.set_model_name(model)
async def start(self, frame: StartFrame):
"""Start the OpenAI TTS service.
@@ -185,16 +195,16 @@ class OpenAITTSService(TTSService):
# Setup API parameters
create_params = {
"input": text,
"model": self.model_name,
"voice": VALID_VOICES[self._voice_id],
"model": self._settings.model,
"voice": VALID_VOICES[self._settings.voice],
"response_format": "pcm",
}
if self._settings["instructions"]:
create_params["instructions"] = self._settings["instructions"]
if self._settings.instructions:
create_params["instructions"] = self._settings.instructions
if self._settings["speed"]:
create_params["speed"] = self._settings["speed"]
if self._settings.speed:
create_params["speed"] = self._settings.speed
async with self._client.audio.speech.with_streaming_response.create(
**create_params

90
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -10,7 +10,7 @@ import base64
import json
import time
import warnings
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Optional
from loguru import logger
@@ -54,6 +54,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai.llm import OpenAIContextAggregatorPair
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt
@@ -91,6 +92,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class OpenAIRealtimeBetaLLMSettings(LLMSettings):
"""Settings for OpenAI Realtime Beta LLM services.
Parameters:
session_properties: OpenAI Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class OpenAIRealtimeBetaLLMService(LLMService):
"""OpenAI Realtime Beta LLM service providing real-time audio and text communication.
@@ -103,6 +117,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
management, and real-time transcription.
"""
_settings: OpenAIRealtimeBetaLLMSettings
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
adapter_class = OpenAIRealtimeLLMAdapter
@@ -140,15 +156,26 @@ class OpenAIRealtimeBetaLLMService(LLMService):
)
full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs)
super().__init__(
base_url=full_url,
settings=OpenAIRealtimeBetaLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = full_url
self.set_model_name(model)
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._send_transcription_frames = send_transcription_frames
self._websocket = None
@@ -187,12 +214,12 @@ class OpenAIRealtimeBetaLLMService(LLMService):
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
modalities = self._session_properties.modalities or ["audio", "text"]
modalities = self._settings.session_properties.modalities or ["audio", "text"]
return modality in modalities
def _get_enabled_modalities(self) -> list[str]:
"""Get the list of enabled modalities."""
return self._session_properties.modalities or ["audio", "text"]
return self._settings.session_properties.modalities or ["audio", "text"]
async def retrieve_conversation_item(self, item_id: str):
"""Retrieve a conversation item by ID from the server.
@@ -259,7 +286,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_interruption(self):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
if self._session_properties.turn_detection is False:
if self._settings.session_properties.turn_detection is False:
await self.send_client_event(events.InputAudioBufferClearEvent())
await self.send_client_event(events.ResponseCancelEvent())
await self._truncate_current_audio_response()
@@ -276,7 +303,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_user_stopped_speaking(self, frame):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
if self._session_properties.turn_detection is False:
if self._settings.session_properties.turn_detection is False:
await self.send_client_event(events.InputAudioBufferCommitEvent())
await self.send_client_event(events.ResponseCreateEvent())
@@ -342,6 +369,16 @@ class OpenAIRealtimeBetaLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.delta path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.delta is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -377,11 +414,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._handle_messages_append(frame)
elif isinstance(frame, RealtimeMessagesUpdateFrame):
self._context = frame.context
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
elif isinstance(frame, RealtimeFunctionCallResultFrame):
await self._handle_function_call_result(frame.result_frame)
@@ -456,8 +490,15 @@ class OpenAIRealtimeBetaLLMService(LLMService):
# treat a send-side error as fatal.
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
settings = self._session_properties
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update if needed."""
changed = await super()._update_settings(delta)
if "session_properties" in changed:
await self._send_session_update()
return changed
async def _send_session_update(self):
settings = self._settings.session_properties
# tools given in the context override the tools in the session properties
if self._context and self._context.tools:
settings.tools = self._context.tools
@@ -503,15 +544,18 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._handle_evt_audio_transcript_delta(evt)
elif evt.type == "error":
if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt):
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
if evt.error.code == "response_cancel_not_active":
logger.debug(f"{self} {evt.error.message}")
else:
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
@traced_openai_realtime(operation="llm_setup")
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
# to configure the session properties.
await self._update_settings()
await self._send_session_update()
async def _handle_evt_session_updated(self, evt):
# If this is our first context frame, run the LLM
@@ -665,7 +709,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_evt_speech_started(self, evt):
await self._truncate_current_audio_response()
await self.broadcast_frame(UserStartedSpeakingFrame)
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
async def _handle_evt_speech_stopped(self, evt):
await self.start_ttfb_metrics()
@@ -750,7 +794,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self._context.llm_needs_initial_messages = False
if self._context.llm_needs_settings_update:
await self._update_settings()
await self._send_session_update()
self._context.llm_needs_settings_update = False
logger.debug(f"Creating response: {self._context.get_messages_for_logging()}")

3
src/pipecat/services/openrouter/llm.py
View File

@@ -72,8 +72,7 @@ class OpenRouterLLMService(OpenAILLMService):
Transformed parameters ready for the API call.
"""
params = super().build_chat_completion_params(params_from_context)
model = getattr(self, "model_name", getattr(self, "model", "")).lower()
if "gemini" in model:
if "gemini" in self._settings.model.lower():
messages = params.get("messages", [])
if not messages:
return params

24
src/pipecat/services/perplexity/llm.py
View File

@@ -11,8 +11,6 @@ an OpenAI-compatible interface. It handles Perplexity's unique token usage
reporting patterns while maintaining compatibility with the Pipecat framework.
"""
from openai import NOT_GIVEN
from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -66,22 +64,22 @@ class PerplexityLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"messages": params_from_context["messages"],
}
# Add OpenAI-compatible parameters if they're set
if self._settings["frequency_penalty"] is not NOT_GIVEN:
params["frequency_penalty"] = self._settings["frequency_penalty"]
if self._settings["presence_penalty"] is not NOT_GIVEN:
params["presence_penalty"] = self._settings["presence_penalty"]
if self._settings["temperature"] is not NOT_GIVEN:
params["temperature"] = self._settings["temperature"]
if self._settings["top_p"] is not NOT_GIVEN:
params["top_p"] = self._settings["top_p"]
if self._settings["max_tokens"] is not NOT_GIVEN:
params["max_tokens"] = self._settings["max_tokens"]
if self._settings.frequency_penalty is not None:
params["frequency_penalty"] = self._settings.frequency_penalty
if self._settings.presence_penalty is not None:
params["presence_penalty"] = self._settings.presence_penalty
if self._settings.temperature is not None:
params["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
params["top_p"] = self._settings.top_p
if self._settings.max_tokens is not None:
params["max_tokens"] = self._settings.max_tokens
return params

49
src/pipecat/services/piper/tts.py
View File

@@ -7,8 +7,9 @@
"""Piper TTS service implementation."""
import asyncio
from dataclasses import dataclass
from pathlib import Path
from typing import AsyncGenerator, AsyncIterator, Optional
from typing import Any, AsyncGenerator, AsyncIterator, Optional
import aiohttp
from loguru import logger
@@ -19,6 +20,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import TTSSettings
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -31,6 +33,13 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class PiperTTSSettings(TTSSettings):
"""Settings for Piper TTS service."""
pass
class PiperTTSService(TTSService):
"""Piper TTS service implementation.
@@ -39,6 +48,8 @@ class PiperTTSService(TTSService):
match the configured sample rate.
"""
_settings: PiperTTSSettings
def __init__(
self,
*,
@@ -58,9 +69,10 @@ class PiperTTSService(TTSService):
use_cuda: Use CUDA for GPU-accelerated inference.
**kwargs: Additional arguments passed to the parent `TTSService`.
"""
super().__init__(**kwargs)
self._voice_id = voice_id
super().__init__(
settings=PiperTTSSettings(model=None, voice=voice_id, language=None),
**kwargs,
)
download_dir = download_dir or Path.cwd()
@@ -85,6 +97,18 @@ class PiperTTSService(TTSService):
"""
return True
async def _update_settings(self, delta: PiperTTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: voice changes would require re-downloading and loading the model.
self._warn_unhandled_updated_settings(changed)
return changed
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Piper.
@@ -143,6 +167,13 @@ class PiperTTSService(TTSService):
# $ uv pip install "piper-tts[http]"
# $ uv run python -m piper.http_server -m en_US-ryan-high
#
@dataclass
class PiperHttpTTSSettings(TTSSettings):
"""Settings for Piper HTTP TTS service."""
pass
class PiperHttpTTSService(TTSService):
"""Piper HTTP TTS service implementation.
@@ -151,6 +182,8 @@ class PiperHttpTTSService(TTSService):
rates and automatic WAV header removal.
"""
_settings: PiperHttpTTSSettings
def __init__(
self,
*,
@@ -167,7 +200,10 @@ class PiperHttpTTSService(TTSService):
voice_id: Piper voice model identifier (e.g. `en_US-ryan-high`).
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(**kwargs)
super().__init__(
settings=PiperHttpTTSSettings(model=None, voice=voice_id, language=None),
**kwargs,
)
if base_url.endswith("/"):
logger.warning("Base URL ends with a slash, this is not allowed.")
@@ -175,7 +211,6 @@ class PiperHttpTTSService(TTSService):
self._base_url = base_url
self._session = aiohttp_session
self._model_id = voice_id
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -205,7 +240,7 @@ class PiperHttpTTSService(TTSService):
data = {
"text": text,
"voice": self._model_id,
"voice": self._settings.voice,
}
async with self._session.post(self._base_url, json=data, headers=headers) as response:

13
src/pipecat/services/playht/__init__.py
View File

@@ -1,13 +0,0 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import sys
from pipecat.services import DeprecatedModuleProxy
from .tts import *
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "playht", "playht.tts")

651
src/pipecat/services/playht/tts.py
View File

@@ -1,651 +0,0 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""PlayHT text-to-speech service implementations.
This module provides integration with PlayHT's text-to-speech API
supporting both WebSocket streaming and HTTP-based synthesis.
"""
import io
import json
import struct
import uuid
import warnings
from typing import AsyncGenerator, Optional
import aiohttp
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use PlayHTTTSService, you need to `pip install pipecat-ai[playht]`.")
raise Exception(f"Missing module: {e}")
def language_to_playht_language(language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding PlayHT language code, or None if not supported.
"""
LANGUAGE_MAP = {
Language.AF: "afrikans",
Language.AM: "amharic",
Language.AR: "arabic",
Language.BN: "bengali",
Language.BG: "bulgarian",
Language.CA: "catalan",
Language.CS: "czech",
Language.DA: "danish",
Language.DE: "german",
Language.EL: "greek",
Language.EN: "english",
Language.ES: "spanish",
Language.FR: "french",
Language.GL: "galician",
Language.HE: "hebrew",
Language.HI: "hindi",
Language.HR: "croatian",
Language.HU: "hungarian",
Language.ID: "indonesian",
Language.IT: "italian",
Language.JA: "japanese",
Language.KO: "korean",
Language.MS: "malay",
Language.NL: "dutch",
Language.PL: "polish",
Language.PT: "portuguese",
Language.RU: "russian",
Language.SQ: "albanian",
Language.SR: "serbian",
Language.SV: "swedish",
Language.TH: "thai",
Language.TL: "tagalog",
Language.TR: "turkish",
Language.UK: "ukrainian",
Language.UR: "urdu",
Language.XH: "xhosa",
Language.ZH: "mandarin",
}
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
class PlayHTTTSService(InterruptibleTTSService):
"""PlayHT WebSocket-based text-to-speech service.
.. deprecated:: 0.0.88
This class is deprecated and will be removed in a future version.
PlayHT is shutting down their API on December 31st, 2025.
Provides real-time text-to-speech synthesis using PlayHT's WebSocket API.
Supports streaming audio generation with configurable voice engines and
language settings.
"""
class InputParams(BaseModel):
"""Input parameters for PlayHT TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
seed: Random seed for voice consistency.
"""
language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0
seed: Optional[int] = None
def __init__(
self,
*,
api_key: str,
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini",
sample_rate: Optional[int] = None,
output_format: str = "wav",
params: Optional[InputParams] = None,
**kwargs,
):
"""Initialize the PlayHT WebSocket TTS service.
Args:
api_key: PlayHT API key for authentication.
user_id: PlayHT user ID for authentication.
voice_url: URL of the voice to use for synthesis.
voice_engine: Voice engine to use. Defaults to "Play3.0-mini".
sample_rate: Audio sample rate. If None, uses default.
output_format: Audio output format. Defaults to "wav".
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
super().__init__(
pause_frame_processing=True,
sample_rate=sample_rate,
**kwargs,
)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"PlayHT is shutting down their API on December 31st, 2025. "
"'PlayHTTTSService' is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
params = params or PlayHTTTSService.InputParams()
self._api_key = api_key
self._user_id = user_id
self._websocket_url = None
self._receive_task = None
self._context_id = None
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "english",
"output_format": output_format,
"voice_engine": voice_engine,
"speed": params.speed,
"seed": params.seed,
}
self.set_model_name(voice_engine)
self.set_voice(voice_url)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as PlayHT service supports metrics generation.
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT service language format.
Args:
language: The language to convert.
Returns:
The PlayHT-specific language code, or None if not supported.
"""
return language_to_playht_language(language)
async def start(self, frame: StartFrame):
"""Start the PlayHT TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
await self._connect()
async def stop(self, frame: EndFrame):
"""Stop the PlayHT TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
"""Cancel the PlayHT TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame)
await self._disconnect()
async def _connect(self):
"""Connect to PlayHT WebSocket and start receive task."""
await super()._connect()
await self._connect_websocket()
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self):
"""Disconnect from PlayHT WebSocket and clean up tasks."""
await super()._disconnect()
if self._receive_task:
await self.cancel_task(self._receive_task)
self._receive_task = None
await self._disconnect_websocket()
async def _connect_websocket(self):
"""Connect to PlayHT websocket."""
try:
if self._websocket and self._websocket.state is State.OPEN:
return
logger.debug("Connecting to PlayHT")
if not self._websocket_url:
await self._get_websocket_url()
if not isinstance(self._websocket_url, str):
raise ValueError("WebSocket URL is not a string")
self._websocket = await websocket_connect(self._websocket_url)
await self._call_event_handler("on_connected")
except ValueError as e:
logger.error(f"{self} initialization error: {e}")
self._websocket = None
await self._call_event_handler("on_connection_error", f"{e}")
except Exception as e:
await self.push_error(error_msg=f"Error connecting: {e}", exception=e)
self._websocket = None
await self._call_event_handler("on_connection_error", f"{e}")
async def _disconnect_websocket(self):
"""Disconnect from PlayHT websocket."""
try:
await self.stop_all_metrics()
if self._websocket:
logger.debug("Disconnecting from PlayHT")
await self._websocket.close()
except Exception as e:
await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e)
finally:
self._context_id = None
self._websocket = None
await self._call_event_handler("on_disconnected")
async def _get_websocket_url(self):
"""Retrieve WebSocket URL from PlayHT API."""
async with aiohttp.ClientSession() as session:
async with session.post(
"https://api.play.ht/api/v4/websocket-auth",
headers={
"Authorization": f"Bearer {self._api_key}",
"X-User-Id": self._user_id,
"Content-Type": "application/json",
},
) as response:
if response.status in (200, 201):
data = await response.json()
# Handle the new response format with multiple URLs
if "websocket_urls" in data:
# Select URL based on voice_engine
if self._settings["voice_engine"] in data["websocket_urls"]:
self._websocket_url = data["websocket_urls"][
self._settings["voice_engine"]
]
else:
raise ValueError(
f"Unsupported voice engine: {self._settings['voice_engine']}"
)
else:
raise ValueError("Invalid response: missing websocket_urls")
else:
raise Exception(f"Failed to get WebSocket URL: {response.status}")
def _get_websocket(self):
"""Get the WebSocket connection if available."""
if self._websocket:
return self._websocket
raise Exception("Websocket not connected")
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request in case we don't have one already in progress.
Returns:
A unique string identifier for the TTS context.
"""
# If a context ID does not exist, create a new one.
# If an ID exists, continue using the current ID.
# When interruptions happen, user speech results in
# an interruption, which resets the context ID.
if not self._context_id:
return str(uuid.uuid4())
return self._context_id
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by stopping metrics and clearing request ID."""
await super()._handle_interruption(frame, direction)
await self.stop_all_metrics()
self._context_id = None
async def _receive_messages(self):
"""Receive messages from PlayHT websocket."""
async for message in self._get_websocket():
if isinstance(message, bytes):
# Skip the WAV header message
if message.startswith(b"RIFF"):
continue
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(message, self.sample_rate, 1, context_id=self._context_id)
await self.push_frame(frame)
else:
logger.debug(f"Received text message: {message}")
try:
msg = json.loads(message)
if msg.get("type") == "start":
# Handle start of stream
logger.debug(f"Started processing request: {msg.get('request_id')}")
elif msg.get("type") == "end":
# Handle end of stream
if "request_id" in msg and msg["request_id"] == self._context_id:
await self.push_frame(TTSStoppedFrame(context_id=self._context_id))
self._context_id = None
elif "error" in msg:
await self.push_error(error_msg=f"Error: {msg['error']}")
except json.JSONDecodeError:
logger.error(f"Invalid JSON message: {message}")
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text using PlayHT's WebSocket API.
Args:
text: The text to synthesize into speech.
context_id: The context ID for tracking audio frames.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]")
try:
# Reconnect if the websocket is closed
if not self._websocket or self._websocket.state is State.CLOSED:
await self._connect()
if not self._context_id:
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
self._context_id = context_id
tts_command = {
"text": text,
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"speed": self._settings["speed"],
"seed": self._settings["seed"],
"request_id": self._context_id,
}
try:
await self._get_websocket().send(json.dumps(tts_command))
await self.start_tts_usage_metrics(text)
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
yield TTSStoppedFrame(context_id=context_id)
await self._disconnect()
await self._connect()
return
# The actual audio frames will be handled in _receive_task_handler
yield None
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
class PlayHTHttpTTSService(TTSService):
"""PlayHT HTTP-based text-to-speech service.
.. deprecated:: 0.0.88
This class is deprecated and will be removed in a future version.
PlayHT is shutting down their API on December 31st, 2025.
Provides text-to-speech synthesis using PlayHT's HTTP API for simpler,
non-streaming synthesis. Suitable for use cases where streaming is not
required and simpler integration is preferred.
"""
class InputParams(BaseModel):
"""Input parameters for PlayHT HTTP TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
seed: Random seed for voice consistency.
"""
language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0
seed: Optional[int] = None
def __init__(
self,
*,
api_key: str,
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini",
protocol: Optional[str] = None,
output_format: str = "wav",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
**kwargs,
):
"""Initialize the PlayHT HTTP TTS service.
Args:
api_key: PlayHT API key for authentication.
user_id: PlayHT user ID for authentication.
voice_url: URL of the voice to use for synthesis.
voice_engine: Voice engine to use. Defaults to "Play3.0-mini".
protocol: Protocol to use ("http" or "ws").
.. deprecated:: 0.0.80
This parameter no longer has any effect and will be removed in a future version.
Use PlayHTTTSService for WebSocket or PlayHTHttpTTSService for HTTP.
output_format: Audio output format. Defaults to "wav".
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
# Warn about deprecated protocol parameter if explicitly provided
if protocol:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"The 'protocol' parameter is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"PlayHT is shutting down their API on December 31st, 2025. "
"'PlayHTHttpTTSService' is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
params = params or PlayHTHttpTTSService.InputParams()
self._user_id = user_id
self._api_key = api_key
# Check if voice_engine contains protocol information (backward compatibility)
if "-http" in voice_engine:
# Extract the base engine name
voice_engine = voice_engine.replace("-http", "")
elif "-ws" in voice_engine:
# Extract the base engine name
voice_engine = voice_engine.replace("-ws", "")
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "english",
"output_format": output_format,
"voice_engine": voice_engine,
"speed": params.speed,
"seed": params.seed,
}
self.set_model_name(voice_engine)
self.set_voice(voice_url)
async def start(self, frame: StartFrame):
"""Start the PlayHT HTTP TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as PlayHT HTTP service supports metrics generation.
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT service language format.
Args:
language: The language to convert.
Returns:
The PlayHT-specific language code, or None if not supported.
"""
return language_to_playht_language(language)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text using PlayHT's HTTP API.
Args:
text: The text to synthesize into speech.
context_id: The context ID for tracking audio frames.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]")
try:
await self.start_ttfb_metrics()
# Prepare the request payload
payload = {
"text": text,
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
"sample_rate": self.sample_rate,
"language": self._settings["language"],
}
# Add optional parameters if they exist
if self._settings["speed"] is not None:
payload["speed"] = self._settings["speed"]
if self._settings["seed"] is not None:
payload["seed"] = self._settings["seed"]
headers = {
"Authorization": f"Bearer {self._api_key}",
"X-User-Id": self._user_id,
"Content-Type": "application/json",
"Accept": "*/*",
}
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame(context_id=context_id)
async with aiohttp.ClientSession() as session:
async with session.post(
"https://api.play.ht/api/v2/tts/stream",
headers=headers,
json=payload,
) as response:
if response.status not in (200, 201):
error_text = await response.text()
raise Exception(f"PlayHT API error {response.status}: {error_text}")
in_header = True
buffer = b""
CHUNK_SIZE = self.chunk_size
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
if len(chunk) == 0:
continue
# Skip the RIFF header
if in_header:
buffer += chunk
if len(buffer) <= 36:
continue
else:
fh = io.BytesIO(buffer)
fh.seek(36)
(data, size) = struct.unpack("<4sI", fh.read(8))
while data != b"data":
fh.read(size)
(data, size) = struct.unpack("<4sI", fh.read(8))
# Extract audio data after header
audio_data = buffer[fh.tell() :]
if len(audio_data) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio_data, self.sample_rate, 1, context_id=context_id
)
yield frame
in_header = False
elif len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
chunk, self.sample_rate, 1, context_id=context_id
)
yield frame
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
finally:
await self.stop_ttfb_metrics()
yield TTSStoppedFrame(context_id=context_id)

78
src/pipecat/services/resembleai/tts.py
View File

@@ -8,7 +8,8 @@
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, ClassVar, Dict, Optional
from loguru import logger
@@ -17,14 +18,13 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
try:
@@ -36,7 +36,27 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
class ResembleAITTSService(AudioContextWordTTSService):
@dataclass
class ResembleAITTSSettings(TTSSettings):
"""Settings for Resemble AI TTS service.
Parameters:
precision: PCM bit depth (PCM_32, PCM_24, PCM_16, or MULAW).
output_format: Audio format (wav or mp3).
resemble_sample_rate: Audio sample rate sent to the API.
"""
precision: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_format: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
resemble_sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {
"voice_id": "voice",
"sample_rate": "resemble_sample_rate",
}
class ResembleAITTSService(AudioContextTTSService):
"""Resemble AI TTS service with WebSocket streaming and word timestamps.
Provides text-to-speech using Resemble AI's streaming WebSocket API.
@@ -44,6 +64,8 @@ class ResembleAITTSService(AudioContextWordTTSService):
multiple simultaneous synthesis requests with proper interruption support.
"""
_settings: ResembleAITTSSettings
def __init__(
self,
*,
@@ -69,17 +91,20 @@ class ResembleAITTSService(AudioContextWordTTSService):
super().__init__(
sample_rate=sample_rate,
reuse_context_id_within_turn=False,
supports_word_timestamps=True,
settings=ResembleAITTSSettings(
model=None,
voice=voice_id,
language=None,
precision=precision,
output_format=output_format,
resemble_sample_rate=sample_rate,
),
**kwargs,
)
self._api_key = api_key
self._voice_id = voice_id
self._url = url
self._settings = {
"precision": precision,
"output_format": output_format,
"sample_rate": sample_rate,
}
self._websocket = None
self._request_id_counter = 0
@@ -100,8 +125,6 @@ class ResembleAITTSService(AudioContextWordTTSService):
self._jitter_buffer_bytes = 44100 # ~1000ms at 22050Hz to handle 400ms+ network gaps
self._playback_started: dict[str, bool] = {} # Track if we've started playback per request
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -120,13 +143,13 @@ class ResembleAITTSService(AudioContextWordTTSService):
JSON string containing the request payload.
"""
msg = {
"voice_uuid": self._voice_id,
"voice_uuid": self._settings.voice,
"data": text,
"binary_response": False, # Use JSON frames to get timestamps
"request_id": self._request_id_counter, # ResembleAI only accepts number
"output_format": self._settings["output_format"],
"sample_rate": self._settings["sample_rate"],
"precision": self._settings["precision"],
"output_format": self._settings.output_format,
"sample_rate": self._settings.resemble_sample_rate,
"precision": self._settings.precision,
"no_audio_header": True,
}
@@ -140,7 +163,7 @@ class ResembleAITTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.resemble_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -222,16 +245,19 @@ class ResembleAITTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by stopping current synthesis.
Args:
frame: The interruption frame.
direction: The direction of frame processing.
"""
await super()._handle_interruption(frame, direction)
async def on_audio_context_interrupted(self, context_id: str):
"""Stop metrics when the bot is interrupted."""
await self.stop_all_metrics()
async def on_audio_context_completed(self, context_id: str):
"""Stop metrics after the Resemble AI context finishes playing.
No close message is needed: Resemble AI signals completion with an
``audio_end`` message (handled in ``_process_messages``), after which
the server-side context is already closed.
"""
pass
async def flush_audio(self):
"""Flush any pending audio and finalize the current context."""
logger.trace(f"{self}: flushing audio")

485
src/pipecat/services/rime/tts.py
View File

@@ -12,7 +12,8 @@ using Rime's API for streaming and batch audio synthesis.
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Optional
import aiohttp
from loguru import logger
@@ -30,9 +31,11 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import (
AudioContextWordTTSService,
AudioContextTTSService,
InterruptibleTTSService,
TextAggregationMode,
TTSService,
)
from pipecat.transcriptions.language import Language, resolve_language
@@ -68,7 +71,67 @@ def language_to_rime_language(language: Language) -> str:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
class RimeTTSService(AudioContextWordTTSService):
@dataclass
class RimeTTSSettings(TTSSettings):
"""Settings for Rime WS JSON and HTTP TTS services.
Parameters:
audioFormat: Audio output format.
samplingRate: Audio sample rate.
segment: Text segmentation mode ("immediate", "bySentence", "never").
speedAlpha: Speech speed multiplier (mistv2 only).
reduceLatency: Whether to reduce latency at potential quality cost (mistv2 only).
pauseBetweenBrackets: Whether to add pauses between bracketed content (mistv2 only).
phonemizeBetweenBrackets: Whether to phonemize bracketed content (mistv2 only).
noTextNormalization: Whether to disable text normalization (mistv2 only).
saveOovs: Whether to save out-of-vocabulary words (mistv2 only).
inlineSpeedAlpha: Inline speed control markup.
repetition_penalty: Token repetition penalty (arcana only, 1.0-2.0).
temperature: Sampling temperature (arcana only, 0.0-1.0).
top_p: Cumulative probability threshold (arcana only, 0.0-1.0).
"""
audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speedAlpha: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
reduceLatency: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pauseBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
phonemizeBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
noTextNormalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
saveOovs: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
inlineSpeedAlpha: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"}
@dataclass
class RimeNonJsonTTSSettings(TTSSettings):
"""Settings for Rime non-JSON WS TTS service.
Parameters:
audioFormat: Audio output format.
samplingRate: Audio sample rate.
segment: Text segmentation mode ("immediate", "bySentence", "never").
repetition_penalty: Token repetition penalty (1.0-2.0).
temperature: Sampling temperature (0.0-1.0).
top_p: Cumulative probability threshold (0.0-1.0).
"""
audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"}
class RimeTTSService(AudioContextTTSService):
"""Text-to-Speech service using Rime's websocket API.
Uses Rime's websocket JSON API to convert text to speech with word-level timing
@@ -76,16 +139,18 @@ class RimeTTSService(AudioContextWordTTSService):
within a turn.
"""
_settings: RimeTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
segment: Text segmentation mode ("immediate", "bySentence", "never").
speed_alpha: Speech speed multiplier.
repetition_penalty: Token repetition penalty (arcana only).
temperature: Sampling temperature (arcana only).
top_p: Cumulative probability threshold (arcana only).
speed_alpha: Speech speed multiplier (mistv2 only).
reduce_latency: Whether to reduce latency at potential quality cost (mistv2 only).
pause_between_brackets: Whether to add pauses between bracketed content (mistv2 only).
phonemize_between_brackets: Whether to phonemize bracketed content (mistv2 only).
@@ -95,12 +160,12 @@ class RimeTTSService(AudioContextWordTTSService):
language: Optional[Language] = Language.EN
segment: Optional[str] = None
speed_alpha: Optional[float] = None
# Arcana params
repetition_penalty: Optional[float] = None
temperature: Optional[float] = None
top_p: Optional[float] = None
# Mistv2 params
speed_alpha: Optional[float] = None
reduce_latency: Optional[bool] = None
pause_between_brackets: Optional[bool] = None
phonemize_between_brackets: Optional[bool] = None
@@ -117,7 +182,8 @@ class RimeTTSService(AudioContextWordTTSService):
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
text_aggregator: Optional[BaseTextAggregator] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize Rime TTS service.
@@ -134,17 +200,48 @@ class RimeTTSService(AudioContextWordTTSService):
.. deprecated:: 0.0.95
Use an LLMTextProcessor before the TTSService for custom text aggregation.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to parent class.
"""
# Initialize with parent class settings for proper frame handling
params = params or RimeTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
append_trailing_space=True,
sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
voice=voice_id,
audioFormat="pcm",
samplingRate=0, # updated in start()
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
inlineSpeedAlpha=None, # Not applicable here
speedAlpha=params.speed_alpha,
# Arcana params
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
# Mistv2 params
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=params.no_text_normalization,
saveOovs=params.save_oovs,
),
**kwargs,
)
@@ -154,16 +251,13 @@ class RimeTTSService(AudioContextWordTTSService):
# The preferred way of taking advantage of Rime spelling is
# to use an LLMTextProcessor and/or a text_transformer to identify
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("spell(", ")")])
self._params = params or RimeTTSService.InputParams()
self._text_aggregator = SkipTagsAggregator(
[("spell(", ")")], aggregation_type=self._text_aggregation_mode
)
# Store service configuration
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._model = model
self._settings = self._build_settings()
# State tracking
self._receive_task = None
@@ -189,60 +283,49 @@ class RimeTTSService(AudioContextWordTTSService):
"""
return language_to_rime_language(language)
def _build_settings(self) -> dict:
"""Build query params for the WebSocket URL based on the current model and params.
def _build_ws_params(self) -> dict[str, Any]:
"""Build query params for the WebSocket URL from current settings.
Returns:
Dictionary of query parameters. Only explicitly-set values are included.
Dictionary of query parameters for the WebSocket URL.
Only explicitly-set values are included. Boolean mistv2 params
are serialized with ``json.dumps()`` for the wire format.
"""
settings = {
"speaker": self._voice_id,
"modelId": self._model,
"audioFormat": "pcm",
"samplingRate": self.sample_rate or 0,
params: dict[str, Any] = {
"speaker": self._settings.voice,
"modelId": self._settings.model,
"audioFormat": self._settings.audioFormat,
"samplingRate": self._settings.samplingRate,
}
if self._params.language:
settings["lang"] = self.language_to_service_language(self._params.language) or "eng"
if self._params.segment is not None:
settings["segment"] = self._params.segment
if self._settings.language is not None:
params["lang"] = self._settings.language
if self._settings.segment is not None:
params["segment"] = self._settings.segment
if self._settings.speedAlpha is not None:
params["speedAlpha"] = self._settings.speedAlpha
if self._model == "arcana":
if self._params.repetition_penalty is not None:
settings["repetition_penalty"] = self._params.repetition_penalty
if self._params.temperature is not None:
settings["temperature"] = self._params.temperature
if self._params.top_p is not None:
settings["top_p"] = self._params.top_p
if self._settings.model == "arcana":
if self._settings.repetition_penalty is not None:
params["repetition_penalty"] = self._settings.repetition_penalty
if self._settings.temperature is not None:
params["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
params["top_p"] = self._settings.top_p
else: # mistv2/mist
if self._params.speed_alpha is not None:
settings["speedAlpha"] = self._params.speed_alpha
if self._params.reduce_latency is not None:
settings["reduceLatency"] = self._params.reduce_latency
if self._params.pause_between_brackets is not None:
settings["pauseBetweenBrackets"] = json.dumps(self._params.pause_between_brackets)
if self._params.phonemize_between_brackets is not None:
settings["phonemizeBetweenBrackets"] = json.dumps(
self._params.phonemize_between_brackets
if self._settings.reduceLatency is not None:
params["reduceLatency"] = self._settings.reduceLatency
if self._settings.pauseBetweenBrackets is not None:
params["pauseBetweenBrackets"] = json.dumps(self._settings.pauseBetweenBrackets)
if self._settings.phonemizeBetweenBrackets is not None:
params["phonemizeBetweenBrackets"] = json.dumps(
self._settings.phonemizeBetweenBrackets
)
if self._params.no_text_normalization is not None:
settings["noTextNormalization"] = json.dumps(self._params.no_text_normalization)
if self._params.save_oovs is not None:
settings["saveOovs"] = json.dumps(self._params.save_oovs)
if self._settings.noTextNormalization is not None:
params["noTextNormalization"] = json.dumps(self._settings.noTextNormalization)
if self._settings.saveOovs is not None:
params["saveOovs"] = json.dumps(self._settings.saveOovs)
return settings
async def set_model(self, model: str):
"""Update the TTS model and reconnect.
Args:
model: The model name to use for synthesis.
"""
self._model = model
self._settings = self._build_settings()
await super().set_model(model)
if self._websocket:
await self._disconnect()
await self._connect()
return params
# A set of Rime-specific helpers for text transformations
def SPELL(text: str) -> str:
@@ -269,72 +352,20 @@ class RimeTTSService(AudioContextWordTTSService):
self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)])
return f"[{text}]"
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if necessary.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if necessary.
Since all settings are WebSocket URL query parameters,
any setting change requires reconnecting to apply the new values.
"""
prev_settings = self._settings.copy()
await super()._update_settings(settings)
changed = await super()._update_settings(delta)
needs_reconnect = False
if "voice" in settings or "voice_id" in settings:
self._settings["speaker"] = self._voice_id
if prev_settings.get("speaker") != self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
needs_reconnect = True
if "model" in settings:
self._settings = self._build_settings()
needs_reconnect = True
if "language" in settings:
new_lang = self.language_to_service_language(settings["language"])
if new_lang and new_lang != prev_settings.get("lang"):
logger.info(f"Updating language to: [{new_lang}]")
self._settings["lang"] = new_lang
needs_reconnect = True
# Arcana params
for key, settings_key in [
("repetition_penalty", "repetition_penalty"),
("temperature", "temperature"),
("top_p", "top_p"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 params
for key, settings_key in [
("speed_alpha", "speedAlpha"),
("reduce_latency", "reduceLatency"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 boolean params (need json.dumps)
for key, settings_key in [
("pause_between_brackets", "pauseBetweenBrackets"),
("phonemize_between_brackets", "phonemizeBetweenBrackets"),
("no_text_normalization", "noTextNormalization"),
("save_oovs", "saveOovs"),
]:
if key in settings and json.dumps(settings[key]) != prev_settings.get(settings_key):
self._settings[settings_key] = json.dumps(settings[key])
needs_reconnect = True
if "segment" in settings and settings["segment"] != prev_settings.get("segment"):
self._settings["segment"] = settings["segment"]
needs_reconnect = True
if needs_reconnect and self._websocket:
if changed and self._websocket:
await self._disconnect()
await self._connect()
return changed
def _build_msg(self, text: str = "") -> dict:
"""Build JSON message for Rime API."""
msg = {"text": text, "contextId": self.get_active_audio_context_id()}
@@ -358,7 +389,7 @@ class RimeTTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings = self._build_settings()
self._settings.samplingRate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -404,7 +435,8 @@ class RimeTTSService(AudioContextWordTTSService):
if self._websocket and self._websocket.state is State.OPEN:
return
params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None)
ws_params = self._build_ws_params()
params = "&".join(f"{k}={v}" for k, v in ws_params.items() if v is not None)
url = f"{self._url}?{params}"
headers = {"Authorization": f"Bearer {self._api_key}"}
self._websocket = await websocket_connect(url, additional_headers=headers)
@@ -435,14 +467,25 @@ class RimeTTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by clearing current context."""
context_id = self.get_active_audio_context_id()
await super()._handle_interruption(frame, direction)
async def _close_context(self, context_id: str):
"""Clear the Rime speech queue and stop metrics."""
await self.stop_all_metrics()
if context_id:
await self._get_websocket().send(json.dumps(self._build_clear_msg()))
async def on_audio_context_interrupted(self, context_id: str):
"""Clear the Rime speech queue and stop metrics when the bot is interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Clear server-side state and stop metrics after the Rime context finishes playing.
Rime does not send a server-side completion signal (e.g. ``done`` / ``end_of_stream`` /
``audio_end``), so we explicitly send a ``clear`` message to clean up
any residual server-side state once all audio has been delivered.
"""
await self._close_context(context_id)
def _calculate_word_times(self, words: list, starts: list, ends: list) -> list:
"""Calculate word timing pairs with proper spacing and punctuation.
@@ -580,6 +623,8 @@ class RimeHttpTTSService(TTSService):
Suitable for use cases where streaming is not required.
"""
_settings: RimeTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime HTTP TTS service.
@@ -621,27 +666,36 @@ class RimeHttpTTSService(TTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RimeHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
audioFormat="pcm",
samplingRate=0,
segment=None,
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=None,
saveOovs=None,
inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else None,
repetition_penalty=None,
temperature=None,
top_p=None,
voice=voice_id,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = {
"lang": self.language_to_service_language(params.language)
if params.language
else "eng",
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": params.pause_between_brackets,
"phonemizeBetweenBrackets": params.phonemize_between_brackets,
}
self.set_voice(voice_id)
self.set_model_name(model)
if params.inline_speed_alpha:
self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -681,10 +735,18 @@ class RimeHttpTTSService(TTSService):
"Content-Type": "application/json",
}
payload = self._settings.copy()
payload = {
"lang": self._settings.language,
"speedAlpha": self._settings.speedAlpha,
"reduceLatency": self._settings.reduceLatency,
"pauseBetweenBrackets": self._settings.pauseBetweenBrackets,
"phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets,
}
if self._settings.inlineSpeedAlpha is not None:
payload["inlineSpeedAlpha"] = self._settings.inlineSpeedAlpha
payload["text"] = text
payload["speaker"] = self._voice_id
payload["modelId"] = self._model_name
payload["speaker"] = self._settings.voice
payload["modelId"] = self._settings.model
payload["samplingRate"] = self.sample_rate
# Arcana does not support PCM audio
@@ -743,6 +805,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
accepts and returns non-JSON messages.
"""
_settings: RimeNonJsonTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime Non-JSON WebSocket TTS service.
@@ -772,7 +836,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
audio_format: str = "pcm",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize Rime Non-JSON WebSocket TTS service.
@@ -785,41 +850,44 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
audio_format: Audio format to use.
sample_rate: Audio sample rate in Hz.
params: Additional configuration parameters.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead. Set to ``TextAggregationMode.SENTENCE``
to aggregate text into sentences before synthesis, or
``TextAggregationMode.TOKEN`` to stream tokens directly for lower latency.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to parent class.
"""
params = params or RimeNonJsonTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_stop_frames=True,
pause_frame_processing=True,
append_trailing_space=True,
settings=RimeNonJsonTTSSettings(
voice=voice_id,
model=model,
audioFormat=audio_format,
samplingRate=sample_rate,
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
),
**kwargs,
)
params = params or RimeNonJsonTTSService.InputParams()
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._model = model
self._settings = {
"speaker": voice_id,
"modelId": model,
"audioFormat": audio_format,
"samplingRate": sample_rate,
}
if params.language:
self._settings["lang"] = self.language_to_service_language(params.language)
if params.segment is not None:
self._settings["segment"] = params.segment
if params.repetition_penalty is not None:
self._settings["repetition_penalty"] = params.repetition_penalty
if params.temperature is not None:
self._settings["temperature"] = params.temperature
if params.top_p is not None:
self._settings["top_p"] = params.top_p
# Add any extra parameters for future compatibility
if params.extra:
self._settings.update(params.extra)
self._settings.extra.update(params.extra)
self._receive_task = None
self._context_id: Optional[str] = None
@@ -851,7 +919,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["samplingRate"] = self.sample_rate
self._settings.samplingRate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -895,8 +963,26 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
try:
if self._websocket and self._websocket.state is State.OPEN:
return
# Build URL with query parameters (only non-None values)
params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None)
# Build URL with query parameters (only given, non-None values)
settings_dict = {
"speaker": self._settings.voice,
"modelId": self._settings.model,
"audioFormat": self._settings.audioFormat,
"samplingRate": self._settings.samplingRate,
}
if self._settings.language is not None:
settings_dict["lang"] = self._settings.language
if self._settings.segment is not None:
settings_dict["segment"] = self._settings.segment
if self._settings.repetition_penalty is not None:
settings_dict["repetition_penalty"] = self._settings.repetition_penalty
if self._settings.temperature is not None:
settings_dict["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
settings_dict["top_p"] = self._settings.top_p
# Include extras
settings_dict.update(self._settings.extra)
params = "&".join(f"{k}={v}" for k, v in settings_dict.items() if v is not None)
url = f"{self._url}?{params}"
headers = {"Authorization": f"Bearer {self._api_key}"}
self._websocket = await websocket_connect(
@@ -990,68 +1076,17 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if necessary.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if necessary.
Since all settings are WebSocket URL query parameters,
any setting change requires reconnecting to apply the new values.
"""
needs_reconnect = False
changed = await super()._update_settings(delta)
# Track previous values from self._settings only
prev_settings = self._settings.copy()
# Let parent class handle standard settings (voice, model, language)
await super()._update_settings(settings)
# Check if voice changed and update settings dict
if "voice" in settings or "voice_id" in settings:
self._settings["speaker"] = self._voice_id
if prev_settings.get("speaker") != self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
needs_reconnect = True
# Check if model changed and update settings dict
if "model" in settings:
self._settings["modelId"] = self._model
if prev_settings.get("modelId") != self._model:
logger.info(f"Switching TTS model to: [{self._model}]")
needs_reconnect = True
# Handle language explicitly
if "language" in settings:
new_lang = self.language_to_service_language(settings["language"])
if new_lang and new_lang != prev_settings.get("lang"):
logger.info(f"Updating language to: [{new_lang}]")
self._settings["lang"] = new_lang
needs_reconnect = True
# Check other parameters
for key in ["segment", "repetition_penalty", "temperature", "top_p"]:
if key in settings and settings[key] != prev_settings.get(key):
logger.info(f"Updating {key} to: [{settings[key]}]")
self._settings[key] = settings[key]
needs_reconnect = True
# Handle extra parameters
for key, value in settings.items():
if key not in [
"voice",
"voice_id",
"model",
"language",
"segment",
"repetition_penalty",
"temperature",
"top_p",
]:
if value != prev_settings.get(key):
logger.info(f"Updating extra parameter {key} to: [{value}]")
self._settings[key] = value
needs_reconnect = True
# Reconnect if any setting changed
if needs_reconnect:
if changed:
logger.debug("Settings changed, reconnecting WebSocket with new parameters")
await self._disconnect()
await self._connect()
return changed

12
src/pipecat/services/sambanova/llm.py
View File

@@ -84,19 +84,19 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
"max_completion_tokens": self._settings.max_completion_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params
@traced_llm # type: ignore

2
src/pipecat/services/sambanova/stt.py
View File

@@ -72,7 +72,7 @@ class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"model": self._settings.model,
"response_format": "json",
"language": self._language,
}

205
src/pipecat/services/sarvam/stt.py
View File

@@ -12,8 +12,8 @@ can handle multiple audio formats for Indian language speech recognition.
"""
import base64
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Literal, Optional
from loguru import logger
from pydantic import BaseModel
@@ -32,6 +32,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.sarvam._sdk import sdk_headers
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import SARVAM_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -130,6 +131,23 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = {
}
@dataclass
class SarvamSTTSettings(STTSettings):
"""Settings for the Sarvam STT service.
Parameters:
prompt: Optional prompt to guide transcription/translation style.
mode: Mode of operation (transcribe, translate, verbatim, etc.).
vad_signals: Enable VAD signals in response.
high_vad_sensitivity: Enable high VAD sensitivity.
"""
prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
mode: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SarvamSTTService(STTService):
"""Sarvam speech-to-text service.
@@ -148,6 +166,8 @@ class SarvamSTTService(STTService):
...
"""
_settings: SarvamSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Sarvam STT service.
@@ -220,50 +240,41 @@ class SarvamSTTService(STTService):
f"Model '{model}' does not support language parameter (auto-detects language)."
)
# Resolve mode default from model config
mode = params.mode if params.mode is not None else self._config.default_mode
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=keepalive_timeout,
keepalive_interval=keepalive_interval,
settings=SarvamSTTSettings(
model=model,
language=params.language,
prompt=params.prompt,
mode=mode,
vad_signals=params.vad_signals,
high_vad_sensitivity=params.high_vad_sensitivity,
),
**kwargs,
)
self.set_model_name(model)
self._api_key = api_key
self._language_code: Optional[Language] = params.language
# Set language string: use provided language or model's default
if params.language:
self._language_string = language_to_sarvam_language(params.language)
else:
self._language_string = self._config.default_language
self._prompt = params.prompt
# Set mode: use provided mode or model's default
self._mode = params.mode if params.mode is not None else self._config.default_mode
# Store connection parameters
self._vad_signals = params.vad_signals
self._high_vad_sensitivity = params.high_vad_sensitivity
self._input_audio_codec = input_audio_codec
# Initialize Sarvam SDK client
self._sdk_headers = sdk_headers()
# NOTE: We avoid passing non-standard kwargs here because different sarvamai
# versions expose different constructor signatures (static type checkers
# complain otherwise). We instead inject headers best-effort below.
self._sarvam_client = AsyncSarvamAI(api_subscription_key=api_key)
for attr in ("default_headers", "_default_headers", "headers", "_headers"):
d = getattr(self._sarvam_client, attr, None)
if isinstance(d, dict):
d.update(self._sdk_headers)
break
# Pass Pipecat SDK headers directly at client construction time so they are
# merged by the Sarvam SDK's client wrapper and consistently applied to
# WebSocket handshake requests.
self._sarvam_client = AsyncSarvamAI(api_subscription_key=api_key, headers=self._sdk_headers)
self._websocket_context = None
self._socket_client = None
self._receive_task = None
if self._vad_signals:
if params.vad_signals:
self._register_event_handler("on_speech_started")
self._register_event_handler("on_speech_stopped")
self._register_event_handler("on_utterance_end")
@@ -281,6 +292,12 @@ class SarvamSTTService(STTService):
"""
return language_to_sarvam_language(language)
def _get_language_string(self) -> Optional[str]:
"""Resolve the current language setting to a Sarvam language code string."""
if self._settings.language:
return language_to_sarvam_language(self._settings.language)
return self._config.default_language
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -298,50 +315,91 @@ class SarvamSTTService(STTService):
await super().process_frame(frame, direction)
# Only handle VAD frames when not using Sarvam's VAD signals
if not self._vad_signals:
if not self._settings.vad_signals:
if isinstance(frame, VADUserStartedSpeakingFrame):
await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
if self._socket_client:
await self._socket_client.flush()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta, validate, sync state, and reconnect.
Args:
language: The language to use for speech recognition.
delta: A :class:`STTSettings` (or ``SarvamSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
Raises:
ValueError: If called on a model that auto-detects language.
ValueError: If a setting is not supported by the current model.
"""
if not self._config.supports_language:
raise ValueError(
f"Model '{self.model_name}' does not support language parameter "
"(auto-detects language)."
)
# Validate against model capabilities before applying
if is_given(delta.language) and delta.language is not None:
if not self._config.supports_language:
raise ValueError(
f"Model '{self._settings.model}' does not support language parameter "
"(auto-detects language)."
)
logger.info(f"Switching STT language to: [{language}]")
self._language_code = language
self._language_string = language_to_sarvam_language(language)
await self._disconnect()
await self._connect()
if isinstance(delta, SarvamSTTSettings):
if is_given(delta.prompt) and delta.prompt is not None:
if not self._config.supports_prompt:
raise ValueError(
f"Model '{self._settings.model}' does not support prompt parameter."
)
if is_given(delta.mode) and delta.mode is not None:
if not self._config.supports_mode:
raise ValueError(
f"Model '{self._settings.model}' does not support mode parameter."
)
changed = await super()._update_settings(delta)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if not changed:
# return changed
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def set_prompt(self, prompt: Optional[str]):
"""Set the transcription/translation prompt and reconnect.
.. deprecated::
Use ``STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...))`` instead.
Args:
prompt: Prompt text to guide transcription/translation style/context.
Pass None to clear/disable prompt.
Only applicable to models that support prompts.
"""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
f"{self.__class__.__name__}.set_prompt() is deprecated. "
"Use STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...)) instead.",
DeprecationWarning,
stacklevel=2,
)
if not self._config.supports_prompt:
if prompt is not None:
raise ValueError(f"Model '{self.model_name}' does not support prompt parameter.")
raise ValueError(
f"Model '{self._settings.model}' does not support prompt parameter."
)
# If prompt is None and model doesn't support prompts, silently return (no-op)
return
logger.info(f"Updating {self.model_name} prompt.")
self._prompt = prompt
logger.info(f"Updating {self._settings.model} prompt.")
self._settings.prompt = prompt
await self._disconnect()
await self._connect()
@@ -422,51 +480,58 @@ class SarvamSTTService(STTService):
try:
# Build common connection parameters
connect_kwargs = {
"model": self.model_name,
"model": self._settings.model,
"sample_rate": str(self.sample_rate),
}
# Enable flush signal when using Pipecat's VAD (not Sarvam's) so that
# the flush() call on user-stopped-speaking is honored by the server.
if not self._vad_signals:
if not self._settings.vad_signals:
connect_kwargs["flush_signal"] = "true"
# Only send vad parameters when explicitly set (avoid overriding server defaults)
if self._vad_signals is not None:
connect_kwargs["vad_signals"] = "true" if self._vad_signals else "false"
if self._high_vad_sensitivity is not None:
if self._settings.vad_signals is not None:
connect_kwargs["vad_signals"] = "true" if self._settings.vad_signals else "false"
if self._settings.high_vad_sensitivity is not None:
connect_kwargs["high_vad_sensitivity"] = (
"true" if self._high_vad_sensitivity else "false"
"true" if self._settings.high_vad_sensitivity else "false"
)
# Add language_code for models that support it
if self._language_string is not None:
connect_kwargs["language_code"] = self._language_string
language_string = self._get_language_string()
if language_string is not None:
connect_kwargs["language_code"] = language_string
# Add mode for models that support it
if self._config.supports_mode and self._mode is not None:
connect_kwargs["mode"] = self._mode
if self._config.supports_mode and self._settings.mode is not None:
connect_kwargs["mode"] = self._settings.mode
# Prompt support differs across sarvamai versions. Prefer connect-time prompt
# when available and gracefully degrade if the SDK doesn't accept it.
if self._prompt is not None and self._config.supports_prompt:
connect_kwargs["prompt"] = self._prompt
if self._settings.prompt is not None and self._config.supports_prompt:
connect_kwargs["prompt"] = self._settings.prompt
def _connect_with_sdk_headers(connect_fn, **kwargs):
# Different SDK versions may use different kwarg names.
# If prompt is unsupported at connect-time, retry without it.
# Headers are supplied through request_options because this is a
# documented SDK parameter that survives SDK signature changes.
request_options = {"additional_headers": self._sdk_headers}
attempts = [kwargs]
if "prompt" in kwargs:
attempts.append({k: v for k, v in kwargs.items() if k != "prompt"})
last_type_error = None
for attempt_kwargs in attempts:
for header_kw in ("headers", "additional_headers", "extra_headers"):
try:
return connect_fn(**attempt_kwargs, **{header_kw: self._sdk_headers})
except TypeError as e:
last_type_error = e
try:
return connect_fn(
**attempt_kwargs,
request_options=request_options,
)
except TypeError as e:
last_type_error = e
try:
# Fallback for SDK builds that don't expose request_options.
return connect_fn(**attempt_kwargs)
except TypeError as e:
last_type_error = e
@@ -491,10 +556,10 @@ class SarvamSTTService(STTService):
self._socket_client = await self._websocket_context.__aenter__()
# Fallback for SDKs that support runtime prompt updates.
if self._prompt is not None and self._config.supports_prompt:
if self._settings.prompt is not None and self._config.supports_prompt:
prompt_setter = getattr(self._socket_client, "set_prompt", None)
if callable(prompt_setter):
await prompt_setter(self._prompt)
await prompt_setter(self._settings.prompt)
# Register event handler for incoming messages
def _message_handler(message):
@@ -579,7 +644,7 @@ class SarvamSTTService(STTService):
logger.debug("User started speaking")
await self._call_event_handler("on_speech_started")
await self.broadcast_frame(UserStartedSpeakingFrame)
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
elif signal == "END_SPEECH":
logger.debug("User stopped speaking")
@@ -592,10 +657,12 @@ class SarvamSTTService(STTService):
# Prefer language from message (auto-detected for translate models). Fallback to configured.
if language_code:
language = self._map_language_code_to_enum(language_code)
elif self._language_string:
language = self._map_language_code_to_enum(self._language_string)
else:
language = Language.HI_IN
language_string = self._get_language_string()
if language_string:
language = self._map_language_code_to_enum(language_string)
else:
language = Language.HI_IN
# Emit utterance end event
await self._call_event_handler("on_utterance_end")

291
src/pipecat/services/sarvam/tts.py
View File

@@ -40,9 +40,9 @@ See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for full API
import asyncio
import base64
import json
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple
from typing import Any, AsyncGenerator, ClassVar, Dict, List, Optional, Tuple
import aiohttp
from loguru import logger
@@ -62,7 +62,8 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.sarvam._sdk import sdk_headers
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -244,6 +245,80 @@ def language_to_sarvam_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class SarvamHttpTTSSettings(TTSSettings):
"""Settings for Sarvam HTTP TTS service.
Parameters:
language: Sarvam language code.
enable_preprocessing: Whether to enable text preprocessing. Defaults to False.
**Note:** Always enabled for bulbul:v3-beta (cannot be disabled).
pace: Speech pace multiplier. Defaults to 1.0.
- bulbul:v2: Range 0.3 to 3.0
- bulbul:v3-beta: Range 0.5 to 2.0
pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0).
Lower values = more deterministic, higher = more random. Defaults to 0.6.
**Note:** Only supported for bulbul:v3-beta. Ignored for v2.
sample_rate: Audio sample rate.
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sarvam_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class SarvamTTSSettings(TTSSettings):
"""Settings for Sarvam WebSocket TTS service.
Parameters:
language: Sarvam language code (e.g. ``"hi-IN"``). Uses the standard
``TTSSettings.language`` field.
speech_sample_rate: Audio sample rate as string.
enable_preprocessing: Enable text preprocessing. Defaults to False.
**Note:** Always enabled for bulbul:v3-beta.
min_buffer_size: Minimum characters to buffer before generating audio.
Lower values reduce latency but may affect quality. Defaults to 50.
max_chunk_length: Maximum characters processed in a single chunk.
Controls memory usage and processing efficiency. Defaults to 150.
output_audio_codec: Audio codec format. Options: linear16, mulaw, alaw,
opus, flac, aac, wav, mp3. Defaults to "linear16".
output_audio_bitrate: Audio bitrate (32k, 64k, 96k, 128k, 192k).
Defaults to "128k".
pace: Speech pace multiplier. Defaults to 1.0.
- bulbul:v2: Range 0.3 to 3.0
- bulbul:v3-beta: Range 0.5 to 2.0
pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0).
Lower = more deterministic, higher = more random. Defaults to 0.6.
**Note:** Only supported for bulbul:v3-beta. Ignored for v2.
"""
_aliases: ClassVar[Dict[str, str]] = {"target_language_code": "language"}
speech_sample_rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_buffer_size: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_chunk_length: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_audio_codec: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_audio_bitrate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SarvamHttpTTSService(TTSService):
"""Text-to-Speech service using Sarvam AI's API.
@@ -296,6 +371,8 @@ class SarvamHttpTTSService(TTSService):
)
"""
_settings: SarvamHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for Sarvam TTS configuration.
@@ -383,18 +460,12 @@ class SarvamHttpTTSService(TTSService):
if sample_rate is None:
sample_rate = self._config.default_sample_rate
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or SarvamHttpTTSService.InputParams()
# Set default voice based on model if not specified
if voice_id is None:
voice_id = self._config.default_speaker
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Validate and clamp pace to model's valid range
pace = params.pace
pace_min, pace_max = self._config.pace_range
@@ -402,37 +473,49 @@ class SarvamHttpTTSService(TTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = {
"language": (
self.language_to_service_language(params.language) if params.language else "en-IN"
super().__init__(
sample_rate=sample_rate,
settings=SarvamHttpTTSSettings(
language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
),
"enable_preprocessing": (
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
"pace": pace,
"model": model,
}
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Add parameters based on model support
if self._config.supports_pitch:
self._settings["pitch"] = params.pitch
self._settings.pitch = params.pitch
elif params.pitch != 0.0:
logger.warning(f"pitch parameter is ignored for {model}")
if self._config.supports_loudness:
self._settings["loudness"] = params.loudness
self._settings.loudness = params.loudness
elif params.loudness != 1.0:
logger.warning(f"loudness parameter is ignored for {model}")
if self._config.supports_temperature:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
elif params.temperature != 0.6:
logger.warning(f"temperature parameter is ignored for {model}")
self.set_model_name(model)
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -459,7 +542,7 @@ class SarvamHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.sarvam_sample_rate = self.sample_rate
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -480,21 +563,25 @@ class SarvamHttpTTSService(TTSService):
# Build payload with common parameters
payload = {
"text": text,
"target_language_code": self._settings["language"],
"speaker": self._voice_id,
"target_language_code": self._settings.language,
"speaker": self._settings.voice,
"sample_rate": self.sample_rate,
"enable_preprocessing": self._settings["enable_preprocessing"],
"model": self._model_name,
"pace": self._settings.get("pace", 1.0),
"enable_preprocessing": self._settings.enable_preprocessing,
"model": self._settings.model,
"pace": self._settings.pace if self._settings.pace is not None else 1.0,
}
# Add model-specific parameters based on config
if self._config.supports_pitch:
payload["pitch"] = self._settings.get("pitch", 0.0)
payload["pitch"] = self._settings.pitch if self._settings.pitch is not None else 0.0
if self._config.supports_loudness:
payload["loudness"] = self._settings.get("loudness", 1.0)
payload["loudness"] = (
self._settings.loudness if self._settings.loudness is not None else 1.0
)
if self._config.supports_temperature:
payload["temperature"] = self._settings.get("temperature", 0.6)
payload["temperature"] = (
self._settings.temperature if self._settings.temperature is not None else 0.6
)
headers = {
"api-subscription-key": self._api_key,
@@ -605,6 +692,8 @@ class SarvamTTSService(InterruptibleTTSService):
See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for API details.
"""
_settings: SarvamTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Sarvam TTS WebSocket service.
@@ -696,7 +785,8 @@ class SarvamTTSService(InterruptibleTTSService):
model: str = "bulbul:v2",
voice_id: Optional[str] = None,
url: str = "wss://api.sarvam.ai/text-to-speech/ws",
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
**kwargs,
@@ -710,7 +800,12 @@ class SarvamTTSService(InterruptibleTTSService):
- "bulbul:v3-beta": Advanced model with temperature control
voice_id: Speaker voice ID. If None, uses model-appropriate default.
url: WebSocket URL for the TTS backend (default production URL).
aggregate_sentences: Merge multiple sentences into one audio chunk (default True).
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
sample_rate: Output audio sample rate in Hz (8000, 16000, 22050, 24000).
If None, uses model-specific default.
params: Optional input parameters to override defaults.
@@ -729,26 +824,11 @@ class SarvamTTSService(InterruptibleTTSService):
if sample_rate is None:
sample_rate = self._config.default_sample_rate
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or SarvamTTSService.InputParams()
# Set default voice based on model if not specified
if voice_id is None:
voice_id = self._config.default_speaker
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
self.set_model_name(model)
self.set_voice(voice_id)
params = params or SarvamTTSService.InputParams()
# Validate and clamp pace to model's valid range
pace = params.pace
@@ -757,37 +837,57 @@ class SarvamTTSService(InterruptibleTTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = {
"target_language_code": (
self.language_to_service_language(params.language) if params.language else "en-IN"
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
settings=SarvamTTSSettings(
language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
speech_sample_rate=str(sample_rate),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
min_buffer_size=params.min_buffer_size,
max_chunk_length=params.max_chunk_length,
output_audio_codec=params.output_audio_codec,
output_audio_bitrate=params.output_audio_bitrate,
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
),
"speaker": voice_id,
"speech_sample_rate": str(sample_rate),
"enable_preprocessing": (
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
"min_buffer_size": params.min_buffer_size,
"max_chunk_length": params.max_chunk_length,
"output_audio_codec": params.output_audio_codec,
"output_audio_bitrate": params.output_audio_bitrate,
"pace": pace,
"model": model,
}
**kwargs,
)
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
# Add parameters based on model support
if self._config.supports_pitch:
self._settings["pitch"] = params.pitch
self._settings.pitch = params.pitch
elif params.pitch != 0.0:
logger.warning(f"pitch parameter is ignored for {model}")
if self._config.supports_loudness:
self._settings["loudness"] = params.loudness
self._settings.loudness = params.loudness
elif params.loudness != 1.0:
logger.warning(f"loudness parameter is ignored for {model}")
if self._config.supports_temperature:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
elif params.temperature != 0.6:
logger.warning(f"temperature parameter is ignored for {model}")
@@ -823,7 +923,7 @@ class SarvamTTSService(InterruptibleTTSService):
await super().start(frame)
# WebSocket API expects sample rate as string
self._settings["speech_sample_rate"] = str(self.sample_rate)
self._settings.speech_sample_rate = str(self.sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -870,14 +970,15 @@ class SarvamTTSService(InterruptibleTTSService):
if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
await self.flush_audio()
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and resend config if voice changed."""
changed = await super()._update_settings(delta)
if changed:
await self._send_config()
return changed
async def _connect(self):
"""Connect to Sarvam WebSocket and start background tasks."""
await super()._connect()
@@ -912,12 +1013,14 @@ class SarvamTTSService(InterruptibleTTSService):
if self._websocket and self._websocket.state is State.OPEN:
return
ws_additional_headers = {
"api-subscription-key": self._api_key,
**sdk_headers(),
}
self._websocket = await websocket_connect(
self._websocket_url,
additional_headers={
"api-subscription-key": self._api_key,
**sdk_headers(),
},
additional_headers=ws_additional_headers,
)
logger.debug("Connected to Sarvam TTS Websocket")
await self._send_config()
@@ -934,9 +1037,27 @@ class SarvamTTSService(InterruptibleTTSService):
"""Send initial configuration message."""
if not self._websocket:
raise Exception("WebSocket not connected")
self._settings["speaker"] = self._voice_id
logger.debug(f"Config being sent is {self._settings}")
config_message = {"type": "config", "data": self._settings}
# Build config dict for the API
config_data = {
"target_language_code": self._settings.language,
"speaker": self._settings.voice,
"speech_sample_rate": self._settings.speech_sample_rate,
"enable_preprocessing": self._settings.enable_preprocessing,
"min_buffer_size": self._settings.min_buffer_size,
"max_chunk_length": self._settings.max_chunk_length,
"output_audio_codec": self._settings.output_audio_codec,
"output_audio_bitrate": self._settings.output_audio_bitrate,
"pace": self._settings.pace,
"model": self._settings.model,
}
if self._settings.pitch is not None:
config_data["pitch"] = self._settings.pitch
if self._settings.loudness is not None:
config_data["loudness"] = self._settings.loudness
if self._settings.temperature is not None:
config_data["temperature"] = self._settings.temperature
logger.debug(f"Config being sent is {config_data}")
config_message = {"type": "config", "data": config_data}
try:
await self._websocket.send(json.dumps(config_message))

433
src/pipecat/services/settings.py Normal file
View File

@@ -0,0 +1,433 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Settings infrastructure for Pipecat AI services.
Each service type has a settings dataclass (``LLMSettings``, ``TTSSettings``,
``STTSettings``, or a service-specific subclass). The same class is used in
two distinct modes:
**Store mode** — the service's ``self._settings`` object that holds the full
current state. Every field must have a real value; ``NOT_GIVEN`` is never
valid here. Services that don't support an inherited field should set it to
``None``. ``validate_complete()`` (called automatically in
``AIService.start()``) enforces this invariant.
**Delta mode** — a sparse update object carried by an
``*UpdateSettingsFrame``. Only the fields the caller wants to change are set;
all others remain at their default of ``NOT_GIVEN``. ``apply_update()``
merges a delta into a store, skipping any ``NOT_GIVEN`` fields.
Key helpers:
- ``NOT_GIVEN`` / ``is_given()`` — sentinel and check for "field not provided
in this delta".
- ``apply_update(delta)`` — merge a delta into a store, returning changed
fields.
- ``from_mapping(dict)`` — build a delta from a plain dict (for backward
compatibility with dict-based ``*UpdateSettingsFrame``).
- ``validate_complete()`` — assert that a store has no ``NOT_GIVEN`` fields.
- ``extra`` dict — overflow for service-specific keys that don't map to a
declared field.
"""
from __future__ import annotations
import copy
from dataclasses import dataclass, field, fields
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Mapping, Optional, Type, TypeVar
from loguru import logger
from pipecat.transcriptions.language import Language
if TYPE_CHECKING:
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionConfig
# ---------------------------------------------------------------------------
# NOT_GIVEN sentinel
# ---------------------------------------------------------------------------
class _NotGiven:
"""Sentinel meaning "this field was not included in the delta".
``NOT_GIVEN`` is distinct from ``None`` (which is a valid stored value,
typically meaning "this service doesn't support this field"). Every
settings field defaults to ``NOT_GIVEN`` so that delta-mode objects are
sparse by default and ``apply_update`` can skip untouched fields.
``NOT_GIVEN`` must never appear in a store-mode object — see
``validate_complete()``.
"""
_instance: Optional[_NotGiven] = None
def __new__(cls) -> _NotGiven:
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __repr__(self) -> str:
return "NOT_GIVEN"
def __bool__(self) -> bool:
return False
NOT_GIVEN: _NotGiven = _NotGiven()
"""Singleton sentinel meaning "this field was not included in the delta".
Valid only in delta-mode settings objects. Must never appear in a service's
``self._settings`` (store mode) — use ``None`` instead for unsupported fields.
"""
def is_given(value: Any) -> bool:
"""Check whether a delta field was explicitly provided.
Typically used when processing a delta to decide whether a field
should be applied::
if is_given(delta.voice):
# caller wants to change the voice
...
For store-mode objects this always returns ``True`` (since
``validate_complete`` ensures no ``NOT_GIVEN`` fields remain).
Args:
value: The value to check.
Returns:
``True`` if *value* is anything other than ``NOT_GIVEN``.
"""
return not isinstance(value, _NotGiven)
# ---------------------------------------------------------------------------
# Base ServiceSettings
# ---------------------------------------------------------------------------
_S = TypeVar("_S", bound="ServiceSettings")
@dataclass
class ServiceSettings:
"""Base class for runtime-updatable service settings.
These settings capture the subset of a service's configuration that can
be changed **while the pipeline is running** (e.g. switching the model or
changing the voice). They are *not* meant to capture every constructor
parameter — only those that support live updates via
``*UpdateSettingsFrame``.
Every AI service type (LLM, TTS, STT) extends this with its own fields.
Each instance operates in one of two modes (see module docstring):
- **Store mode** (``self._settings``): holds the full current state.
Every field must be a real value — ``NOT_GIVEN`` is never valid.
Use ``None`` for inherited fields the service doesn't support.
Enforced at runtime by ``validate_complete()``.
- **Delta mode** (``*UpdateSettingsFrame``): a sparse update.
Only fields the caller wants to change are set; all others stay at
the default ``NOT_GIVEN`` and are skipped by ``apply_update()``.
Parameters:
model: The model identifier used by the service. Set to ``None``
in store mode if the service has no model concept.
extra: Overflow dict for service-specific keys that don't map to a
declared field.
"""
# -- common fields -------------------------------------------------------
model: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
"""AI model identifier (e.g. ``"gpt-4o"``, ``"eleven_turbo_v2_5"``).
Defaults to ``NOT_GIVEN`` for delta mode. In store mode, set to a
model string or ``None`` if the service has no model concept.
"""
extra: Dict[str, Any] = field(default_factory=dict)
"""Catch-all for service-specific keys that have no declared field."""
# -- class-level configuration -------------------------------------------
_aliases: ClassVar[Dict[str, str]] = {}
"""Map of alternative key names to canonical field names.
For example ``{"voice_id": "voice"}`` lets callers use either spelling.
Subclasses should override this as needed.
"""
# -- public API ----------------------------------------------------------
def given_fields(self) -> Dict[str, Any]:
"""Return a dict of only the fields that are not ``NOT_GIVEN``.
Primarily useful for delta-mode objects to inspect which fields were
set. For a store-mode object this returns all declared fields (since
none should be ``NOT_GIVEN``).
Skips the ``extra`` field itself but merges its entries into the
returned dict at the top level.
Returns:
Dictionary mapping field names to their provided values.
"""
result: Dict[str, Any] = {}
for f in fields(self):
if f.name == "extra":
continue
val = getattr(self, f.name)
if is_given(val):
result[f.name] = val
result.update(self.extra)
return result
def apply_update(self: _S, delta: _S) -> Dict[str, Any]:
"""Merge a delta-mode object into this store-mode object.
Only fields in *delta* that are **given** (i.e. not ``NOT_GIVEN``)
are considered. A field is "changed" if its new value differs from
the current value.
The ``extra`` dicts are merged: keys present in the delta overwrite
keys in the target.
Args:
delta: A delta-mode settings object of the same type.
Returns:
A dict mapping each changed field name to its **pre-update** value.
Use ``changed.keys()`` for the set of names, or index with
``changed["field"]`` to inspect the old value.
Examples::
# store-mode object (all fields given)
current = TTSSettings(voice="alice", language="en")
# delta-mode object (only voice is set)
delta = TTSSettings(voice="bob")
changed = current.apply_update(delta)
# changed == {"voice": "alice"}
# current.voice == "bob", current.language == "en"
"""
changed: Dict[str, Any] = {}
for f in fields(self):
if f.name == "extra":
continue
new_val = getattr(delta, f.name, NOT_GIVEN)
if not is_given(new_val):
continue
old_val = getattr(self, f.name)
if old_val != new_val:
setattr(self, f.name, new_val)
changed[f.name] = old_val
# Merge extra
for key, new_val in delta.extra.items():
old_val = self.extra.get(key, NOT_GIVEN)
if old_val != new_val:
self.extra[key] = new_val
changed[key] = old_val
return changed
@classmethod
def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S:
"""Build a **delta-mode** settings object from a plain dictionary.
This exists for backward compatibility with code that passes plain
dicts via ``*UpdateSettingsFrame(settings={...})``. The returned
object is a delta: only the keys present in *settings* are set;
all other fields remain ``NOT_GIVEN``.
Keys are matched to dataclass fields by name. Keys listed in
``_aliases`` are translated to their canonical name first. Any
remaining unrecognized keys are placed into ``extra``.
Args:
settings: A dictionary of setting names to values.
Returns:
A new delta-mode settings instance.
Examples::
delta = TTSSettings.from_mapping({"voice_id": "alice", "speed": 1.2})
# delta.voice == "alice" (via alias)
# delta.language is NOT_GIVEN (not in the dict)
# delta.extra == {"speed": 1.2}
"""
field_names = {f.name for f in fields(cls)} - {"extra"}
kwargs: Dict[str, Any] = {}
extra: Dict[str, Any] = {}
for key, value in settings.items():
# Resolve aliases first
canonical = cls._aliases.get(key, key)
if canonical in field_names:
kwargs[canonical] = value
else:
extra[key] = value
instance = cls(**kwargs)
instance.extra = extra
return instance
def validate_complete(self) -> None:
"""Check that this is a valid store-mode object (no ``NOT_GIVEN`` fields).
Called automatically by ``AIService.start()`` to catch fields that a
service forgot to initialize in its ``__init__``. Can also be called
manually after constructing a store-mode settings object.
Logs a warning for each uninitialized field. Failure to initialize
all fields may or may not cause runtime issues — it depends on
whether and how the service actually reads the field — but it indicates
a deviation from expectations and should be fixed.
"""
missing = [
f.name
for f in fields(self)
if f.name != "extra" and isinstance(getattr(self, f.name), _NotGiven)
]
if missing:
names = ", ".join(missing)
logger.error(
f"{type(self).__name__}: the following fields are NOT_GIVEN: {names}. "
f"All settings fields should be initialized in the service's "
f"__init__ (use None for unsupported fields)."
)
def copy(self: _S) -> _S:
"""Return a deep copy of this settings instance.
Returns:
A new settings object with the same field values.
"""
return copy.deepcopy(self)
# ---------------------------------------------------------------------------
# Service-specific settings
# ---------------------------------------------------------------------------
@dataclass
class ImageGenSettings(ServiceSettings):
"""Runtime-updatable settings for image generation services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Image generation model identifier.
"""
@dataclass
class VisionSettings(ServiceSettings):
"""Runtime-updatable settings for vision services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Vision model identifier.
"""
@dataclass
class LLMSettings(ServiceSettings):
"""Runtime-updatable settings for LLM services.
Used in both store and delta mode — see ``ServiceSettings``.
These fields are common across LLM providers. Not every provider supports
every field; in store mode, set unsupported fields to ``None`` (e.g. a
service that doesn't support ``seed`` should initialize it as
``seed=None``).
Parameters:
model: LLM model identifier.
temperature: Sampling temperature.
max_tokens: Maximum tokens to generate.
top_p: Nucleus sampling probability.
top_k: Top-k sampling parameter.
frequency_penalty: Frequency penalty.
presence_penalty: Presence penalty.
seed: Random seed for reproducibility.
filter_incomplete_user_turns: Enable LLM-based turn completion detection
to suppress bot responses when the user was cut off mid-thought.
See ``examples/foundational/22-filter-incomplete-turns.py`` and
``UserTurnCompletionLLMServiceMixin``.
user_turn_completion_config: Configuration for turn completion behavior
when ``filter_incomplete_user_turns`` is enabled. Controls timeouts
and prompts for incomplete turns.
"""
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_tokens: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_k: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
frequency_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
presence_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
seed: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
filter_incomplete_user_turns: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
user_turn_completion_config: UserTurnCompletionConfig | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
@dataclass
class TTSSettings(ServiceSettings):
"""Runtime-updatable settings for TTS services.
Used in both store and delta mode — see ``ServiceSettings``.
In store mode, set unsupported fields to ``None`` (e.g. ``language=None``
if the service doesn't expose a language setting).
Parameters:
model: TTS model identifier.
voice: Voice identifier or name.
language: Language for speech synthesis. The union type reflects the
*input* side: callers may pass a ``Language`` enum or a raw string
in a delta. However, the **stored** value (in store mode) is
always a service-specific string or ``None`` —
``TTSService._update_settings`` converts ``Language`` enums via
``language_to_service_language()`` before writing, and
``__init__`` methods do the same at construction time.
"""
voice: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: Language | str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@dataclass
class STTSettings(ServiceSettings):
"""Runtime-updatable settings for STT services.
Used in both store and delta mode — see ``ServiceSettings``.
In store mode, set unsupported fields to ``None`` (e.g. ``language=None``
if the service auto-detects language).
Parameters:
model: STT model identifier.
language: Language for speech recognition. The union type reflects the
*input* side: callers may pass a ``Language`` enum or a raw string
in a delta. However, the **stored** value (in store mode) is
always a service-specific string or ``None`` —
``STTService._update_settings`` converts ``Language`` enums via
``language_to_service_language()`` before writing, and
``__init__`` methods do the same at construction time.
"""
language: Language | str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)

109
src/pipecat/services/soniox/stt.py
View File

@@ -8,7 +8,8 @@
import json
import time
from typing import AsyncGenerator, List, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, List, Optional
from loguru import logger
from pydantic import BaseModel
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import SONIOX_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -134,6 +136,35 @@ def _prepare_language_hints(
return list(set(prepared_languages))
@dataclass
class SonioxSTTSettings(STTSettings):
"""Settings for Soniox STT service.
Parameters:
audio_format: Audio format to use for transcription.
num_channels: Number of channels to use for transcription.
language_hints: List of language hints to use for transcription.
language_hints_strict: If true, strictly enforce language hints.
context: Customization for transcription. String for models with
context_version 1 and SonioxContextObject for models with
context_version 2.
enable_speaker_diarization: Whether to enable speaker diarization.
enable_language_identification: Whether to enable language identification.
client_reference_id: Client reference ID to use for transcription.
"""
audio_format: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
num_channels: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_hints: List[Language] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_hints_strict: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
context: SonioxContextObject | str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_speaker_diarization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_language_identification: bool | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
client_reference_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SonioxSTTService(WebsocketSTTService):
"""Speech-to-Text service using Soniox's WebSocket API.
@@ -144,6 +175,8 @@ class SonioxSTTService(WebsocketSTTService):
For complete API documentation, see: https://soniox.com/docs/speech-to-text/api-reference/websocket-api
"""
_settings: SonioxSTTSettings
def __init__(
self,
*,
@@ -169,19 +202,30 @@ class SonioxSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService.
"""
params = params or SonioxInputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=1,
keepalive_interval=5,
settings=SonioxSTTSettings(
model=params.model,
language=None,
audio_format=params.audio_format,
num_channels=params.num_channels,
language_hints=params.language_hints,
language_hints_strict=params.language_hints_strict,
context=params.context,
enable_speaker_diarization=params.enable_speaker_diarization,
enable_language_identification=params.enable_language_identification,
client_reference_id=params.client_reference_id,
),
**kwargs,
)
params = params or SonioxInputParams()
self._api_key = api_key
self._url = url
self.set_model_name(params.model)
self._params = params
self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._final_transcription_buffer = []
@@ -189,6 +233,14 @@ class SonioxSTTService(WebsocketSTTService):
self._receive_task = None
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Soniox STT supports metrics generation.
"""
return True
async def start(self, frame: StartFrame):
"""Start the Soniox STT websocket connection.
@@ -198,6 +250,31 @@ class SonioxSTTService(WebsocketSTTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, delta: SonioxSTTSettings) -> dict[str, Any]:
"""Apply settings delta.
Settings are stored but not applied to the active connection.
Args:
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def stop(self, frame: EndFrame):
"""Stop the Soniox STT websocket connection.
@@ -233,10 +310,8 @@ class SonioxSTTService(WebsocketSTTService):
Yields:
Frame: None (transcription results come via WebSocket callbacks).
"""
await self.start_processing_metrics()
if self._websocket and self._websocket.state is State.OPEN:
await self._websocket.send(audio)
await self.stop_processing_metrics()
yield None
@@ -311,24 +386,26 @@ class SonioxSTTService(WebsocketSTTService):
# Either one or the other is required.
enable_endpoint_detection = not self._vad_force_turn_endpoint
context = self._params.context
s = self._settings
context = s.context
if isinstance(context, SonioxContextObject):
context = context.model_dump()
# Send the initial configuration message.
config = {
"api_key": self._api_key,
"model": self._model_name,
"audio_format": self._params.audio_format,
"num_channels": self._params.num_channels or 1,
"model": s.model,
"audio_format": s.audio_format,
"num_channels": s.num_channels or 1,
"enable_endpoint_detection": enable_endpoint_detection,
"sample_rate": self.sample_rate,
"language_hints": _prepare_language_hints(self._params.language_hints),
"language_hints_strict": self._params.language_hints_strict,
"language_hints": _prepare_language_hints(s.language_hints),
"language_hints_strict": s.language_hints_strict,
"context": context,
"enable_speaker_diarization": self._params.enable_speaker_diarization,
"enable_language_identification": self._params.enable_language_identification,
"client_reference_id": self._params.client_reference_id,
"enable_speaker_diarization": s.enable_speaker_diarization,
"enable_language_identification": s.enable_language_identification,
"client_reference_id": s.client_reference_id,
}
# Send the configuration message.
@@ -415,6 +492,8 @@ class SonioxSTTService(WebsocketSTTService):
# the rest will be sent as interim tokens (even final tokens).
await send_endpoint_transcript()
else:
if not self._final_transcription_buffer:
await self.start_processing_metrics()
self._final_transcription_buffer.append(token)
else:
non_final_transcription.append(token)

274
src/pipecat/services/speechmatics/stt.py
View File

@@ -8,8 +8,10 @@
import asyncio
import os
import warnings
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator
from typing import Any, AsyncGenerator, ClassVar
from dotenv import load_dotenv
from loguru import logger
@@ -31,6 +33,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import SPEECHMATICS_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -80,6 +83,83 @@ class TurnDetectionMode(str, Enum):
SMART_TURN = "smart_turn"
@dataclass
class SpeechmaticsSTTSettings(STTSettings):
"""Settings for Speechmatics STT service.
See ``SpeechmaticsSTTService.InputParams`` for detailed descriptions of each field.
Parameters:
model: The operating point / model name.
domain: Domain for Speechmatics API.
turn_detection_mode: Endpoint handling mode.
speaker_active_format: Formatter for active speaker ID.
speaker_passive_format: Formatter for passive speaker ID.
focus_speakers: List of speaker IDs to focus on.
ignore_speakers: List of speaker IDs to ignore.
focus_mode: Speaker focus mode for diarization.
known_speakers: List of known speaker labels and identifiers.
additional_vocab: List of additional vocabulary entries.
audio_encoding: Audio encoding format.
operating_point: Operating point for accuracy vs. latency.
max_delay: Maximum delay in seconds for transcription.
end_of_utterance_silence_trigger: Maximum delay for end of utterance trigger.
end_of_utterance_max_delay: Maximum delay for end of utterance.
punctuation_overrides: Punctuation overrides.
include_partials: Include partial segment fragments.
split_sentences: Emit finalized sentences mid-turn.
enable_diarization: Enable speaker diarization.
speaker_sensitivity: Diarization sensitivity.
max_speakers: Maximum number of speakers to detect.
prefer_current_speaker: Prefer current speaker ID.
extra_params: Extra parameters for the STT engine.
"""
domain: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
turn_detection_mode: TurnDetectionMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_active_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_passive_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
focus_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
ignore_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
focus_mode: SpeakerFocusMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
known_speakers: list[SpeakerIdentifier] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
additional_vocab: list[AdditionalVocabEntry] | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
audio_encoding: AudioEncoding | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
operating_point: OperatingPoint | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
end_of_utterance_silence_trigger: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
end_of_utterance_max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
punctuation_overrides: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_partials: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
split_sentences: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_diarization: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_sensitivity: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_speakers: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prefer_current_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
extra_params: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
#: Fields that can be updated on a live connection via the Speechmatics
#: diarization-config API — no reconnect needed.
HOT_FIELDS: ClassVar[frozenset[str]] = frozenset(
{
"focus_speakers",
"ignore_speakers",
"focus_mode",
}
)
#: Fields that are purely local (formatting templates) — no reconnect
#: and no API call needed.
LOCAL_FIELDS: ClassVar[frozenset[str]] = frozenset(
{
"speaker_active_format",
"speaker_passive_format",
}
)
class SpeechmaticsSTTService(STTService):
"""Speechmatics STT service implementation.
@@ -98,6 +178,8 @@ class SpeechmaticsSTTService(STTService):
...
"""
_settings: SpeechmaticsSTTSettings
# Export related classes as class attributes
TurnDetectionMode = TurnDetectionMode
AudioEncoding = AudioEncoding
@@ -316,8 +398,6 @@ class SpeechmaticsSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
# Service parameters
self._api_key: str = api_key or os.getenv("SPEECHMATICS_API_KEY")
self._base_url: str = (
@@ -337,31 +417,62 @@ class SpeechmaticsSTTService(STTService):
# Deprecation check
self._check_deprecated_args(kwargs, params)
# Voice agent
# Output formatting defaults
speaker_active_format = params.speaker_active_format
if speaker_active_format is None:
speaker_active_format = (
"@{speaker_id}: {text}" if params.enable_diarization else "{text}"
)
speaker_passive_format = params.speaker_passive_format or speaker_active_format
# Settings — seeded from InputParams
settings = SpeechmaticsSTTSettings(
model=None, # Will be resolved from operating_point after config is built
language=params.language,
domain=params.domain,
turn_detection_mode=params.turn_detection_mode,
speaker_active_format=speaker_active_format,
speaker_passive_format=speaker_passive_format,
focus_speakers=params.focus_speakers,
ignore_speakers=params.ignore_speakers,
focus_mode=params.focus_mode,
known_speakers=params.known_speakers,
additional_vocab=params.additional_vocab,
audio_encoding=params.audio_encoding,
operating_point=params.operating_point,
max_delay=params.max_delay,
end_of_utterance_silence_trigger=params.end_of_utterance_silence_trigger,
end_of_utterance_max_delay=params.end_of_utterance_max_delay,
punctuation_overrides=params.punctuation_overrides,
include_partials=params.include_partials,
split_sentences=params.split_sentences,
enable_diarization=params.enable_diarization,
speaker_sensitivity=params.speaker_sensitivity,
max_speakers=params.max_speakers,
prefer_current_speaker=params.prefer_current_speaker,
extra_params=params.extra_params,
)
# Build SDK config from settings, then resolve model from operating_point
self._client: VoiceAgentClient | None = None
self._config: VoiceAgentConfig = self._prepare_config(params)
self._config: VoiceAgentConfig = self._build_config(settings)
settings.model = self._config.operating_point.value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
# Outbound frame queue
self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue()
# Output formatting
if params.speaker_active_format is None:
params.speaker_active_format = (
"@{speaker_id}: {text}" if params.enable_diarization else "{text}"
)
# Framework options
self._enable_vad: bool = self._config.end_of_utterance_mode not in [
EndOfUtteranceMode.FIXED,
EndOfUtteranceMode.EXTERNAL,
]
self._speaker_active_format: str = params.speaker_active_format
self._speaker_passive_format: str = (
params.speaker_passive_format or params.speaker_active_format
)
# Model + metrics
self.set_model_name(self._config.operating_point.value)
# Message queue
self._stt_msg_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
@@ -384,6 +495,64 @@ class SpeechmaticsSTTService(STTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, delta: SpeechmaticsSTTSettings) -> dict[str, Any]:
"""Apply settings delta, reconnecting only when necessary.
Fields are classified into three categories (see
``SpeechmaticsSTTSettings``):
* **HOT_FIELDS** – diarization speaker settings that can be pushed
to a live Speechmatics connection without reconnecting.
* **LOCAL_FIELDS** – formatting templates evaluated locally; no
reconnect or API call needed.
* Everything else – baked into ``VoiceAgentConfig`` at connection
time and therefore require a full disconnect / reconnect.
Args:
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
no_reconnect = SpeechmaticsSTTSettings.HOT_FIELDS | SpeechmaticsSTTSettings.LOCAL_FIELDS
needs_reconnect = bool(changed.keys() - no_reconnect)
if needs_reconnect:
logger.debug(f"{self} settings update requires reconnect: {changed.keys()}")
# Connection-level fields changed — rebuild the SDK config
# from the now-updated self._settings, then reconnect.
self._config = self._build_config(self._settings)
await self._disconnect()
await self._connect()
elif changed.keys() & SpeechmaticsSTTSettings.HOT_FIELDS:
logger.debug(f"{self} applying hot settings update: {changed.keys()}")
if self._config.enable_diarization:
# Only hot-updatable fields changed — push to the live session.
self._config.speaker_config.focus_speakers = self._settings.focus_speakers
self._config.speaker_config.ignore_speakers = self._settings.ignore_speakers
self._config.speaker_config.focus_mode = self._settings.focus_mode
if self._client:
self._client.update_diarization_config(self._config.speaker_config)
else:
logger.debug(
f"{self} hot settings updated but diarization not enabled: {changed.keys()}. ignoring."
)
# Diarization not enabled — the new settings will take effect
# if/when diarization is enabled, which does require a reconnect.
elif changed.keys() & SpeechmaticsSTTSettings.LOCAL_FIELDS:
logger.debug(
f"{self} local settings update, no special action required: {changed.keys()}"
)
# Only local fields changed — no need to push to the STT engine,
# the new settings will take effect immediately.
return changed
async def stop(self, frame: EndFrame):
"""Called when the session ends."""
await super().stop(frame)
@@ -494,28 +663,39 @@ class SpeechmaticsSTTService(STTService):
# CONFIGURATION
# ============================================================================
def _prepare_config(self, params: InputParams) -> VoiceAgentConfig:
"""Parse the InputParams into VoiceAgentConfig."""
# Preset
config = VoiceAgentConfigPreset.load(params.turn_detection_mode.value)
def _build_config(self, settings: SpeechmaticsSTTSettings) -> VoiceAgentConfig:
"""Build a ``VoiceAgentConfig`` from the given settings.
Used both at init time (with explicit settings, before
``super().__init__`` has run) and before reconnecting so the
connection always reflects the latest settings.
Args:
settings: Settings to build from.
"""
s = settings
# Preset from turn detection mode
config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value)
# Language + domain
config.language = self._language_to_speechmatics_language(params.language)
config.domain = params.domain
config.output_locale = self._locale_to_speechmatics_locale(config.language, params.language)
language = s.language
config.language = self._language_to_speechmatics_language(language)
config.domain = s.domain if s.domain is not None else None
config.output_locale = self._locale_to_speechmatics_locale(config.language, language)
# Speaker config
config.speaker_config = SpeakerFocusConfig(
focus_speakers=params.focus_speakers,
ignore_speakers=params.ignore_speakers,
focus_mode=params.focus_mode,
focus_speakers=s.focus_speakers if s.focus_speakers is not None else [],
ignore_speakers=s.ignore_speakers if s.ignore_speakers is not None else [],
focus_mode=s.focus_mode if s.focus_mode is not None else SpeakerFocusMode.RETAIN,
)
config.known_speakers = params.known_speakers
config.known_speakers = s.known_speakers if s.known_speakers is not None else []
# Custom dictionary
config.additional_vocab = params.additional_vocab
config.additional_vocab = s.additional_vocab if s.additional_vocab is not None else []
# Advanced parameters
# Advanced parameters — only set if not None
for param in [
"operating_point",
"max_delay",
@@ -529,21 +709,20 @@ class SpeechmaticsSTTService(STTService):
"max_speakers",
"prefer_current_speaker",
]:
if getattr(params, param) is not None:
setattr(config, param, getattr(params, param))
val = getattr(s, param)
if val is not None:
setattr(config, param, val)
# Extra parameters
if isinstance(params.extra_params, dict):
for key, value in params.extra_params.items():
if isinstance(s.extra_params, dict):
for key, value in s.extra_params.items():
if hasattr(config, key):
setattr(config, key, value)
# Enable sentences
config.speech_segment_config = SpeechSegmentConfig(
emit_sentences=params.split_sentences or False
)
split = s.split_sentences if s.split_sentences is not None else False
config.speech_segment_config = SpeechSegmentConfig(emit_sentences=split or False)
# Return the complete config
return config
def update_params(
@@ -552,12 +731,23 @@ class SpeechmaticsSTTService(STTService):
) -> None:
"""Updates the speaker configuration.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with
``SpeechmaticsSTTSettings(...)`` instead.
This can update the speakers to listen to or ignore during an in-flight
transcription. Only available if diarization is enabled.
Args:
params: Update parameters for the service.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"update_params() is deprecated. Use STTUpdateSettingsFrame with "
"SpeechmaticsSTTSettings(...) instead.",
DeprecationWarning,
)
# Check possible
if not self._config.enable_diarization:
raise ValueError("Diarization is not enabled")
@@ -646,7 +836,7 @@ class SpeechmaticsSTTService(STTService):
# await self.start_processing_metrics()
await self.broadcast_frame(UserStartedSpeakingFrame)
if self._should_interrupt:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
async def _handle_end_of_turn(self, message: dict[str, Any]) -> None:
"""Handle EndOfTurn events.
@@ -727,9 +917,9 @@ class SpeechmaticsSTTService(STTService):
def attr_from_segment(segment: dict[str, Any]) -> dict[str, Any]:
# Formats the output text based on the speaker and defined formats from the config.
text = (
self._speaker_active_format
self._settings.speaker_active_format
if segment.get("is_active", True)
else self._speaker_passive_format
else self._settings.speaker_passive_format
).format(
**{
"speaker_id": segment.get("speaker_id", "UU"),

40
src/pipecat/services/speechmatics/tts.py
View File

@@ -7,7 +7,8 @@
"""Speechmatics TTS service integration."""
import asyncio
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from urllib.parse import urlencode
import aiohttp
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.network import exponential_backoff_time
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -35,6 +37,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class SpeechmaticsTTSSettings(TTSSettings):
"""Settings for Speechmatics TTS service.
Parameters:
max_retries: Maximum number of retries for HTTP requests.
"""
max_retries: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SpeechmaticsTTSService(TTSService):
"""Speechmatics TTS service implementation.
@@ -42,6 +55,8 @@ class SpeechmaticsTTSService(TTSService):
It converts text to speech and returns raw PCM audio data for real-time playback.
"""
_settings: SpeechmaticsTTSSettings
SPEECHMATICS_SAMPLE_RATE = 16000
class InputParams(BaseModel):
@@ -80,7 +95,18 @@ class SpeechmaticsTTSService(TTSService):
f"Speechmatics TTS only supports {self.SPEECHMATICS_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or SpeechmaticsTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=SpeechmaticsTTSSettings(
model=None,
voice=voice_id,
language=None,
max_retries=params.max_retries,
),
**kwargs,
)
# Service parameters
self._api_key: str = api_key
@@ -91,12 +117,6 @@ class SpeechmaticsTTSService(TTSService):
if not self._api_key:
raise ValueError("Missing Speechmatics API key")
# Default parameters
self._params = params or SpeechmaticsTTSService.InputParams()
# Set voice from constructor parameter
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -131,7 +151,7 @@ class SpeechmaticsTTSService(TTSService):
}
# Complete HTTP URL
url = _get_endpoint_url(self._base_url, self._voice_id, self.sample_rate)
url = _get_endpoint_url(self._base_url, self._settings.voice, self.sample_rate)
try:
# Start TTS TTFB metrics
@@ -159,7 +179,7 @@ class SpeechmaticsTTSService(TTSService):
attempt += 1
# Check if we've exceeded the maximum number of attempts
if attempt >= self._params.max_retries:
if attempt >= self._settings.max_retries:
raise ValueError()
# Report error frame

123
src/pipecat/services/stt_service.py
View File

@@ -9,9 +9,10 @@
import asyncio
import io
import time
import warnings
import wave
from abc import abstractmethod
from typing import Any, AsyncGenerator, Dict, Mapping, Optional
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from websockets.protocol import State
@@ -32,6 +33,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import STTSettings, is_given
from pipecat.services.stt_latency import DEFAULT_TTFS_P99
from pipecat.services.websocket_service import WebsocketService
from pipecat.transcriptions.language import Language
@@ -73,6 +75,8 @@ class STTService(AIService):
logger.error(f"STT connection error: {error}")
"""
_settings: STTSettings
def __init__(
self,
*,
@@ -82,6 +86,7 @@ class STTService(AIService):
ttfs_p99_latency: Optional[float] = None,
keepalive_timeout: Optional[float] = None,
keepalive_interval: float = 5.0,
settings: Optional[STTSettings] = None,
**kwargs,
):
"""Initialize the STT service.
@@ -105,13 +110,20 @@ class STTService(AIService):
connection alive. None disables keepalive. Useful for services that
close idle connections (e.g. behind a ServiceSwitcher).
keepalive_interval: Seconds between idle checks when keepalive is enabled.
settings: The runtime-updatable settings for the STT service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or STTSettings(),
**kwargs,
)
self._audio_passthrough = audio_passthrough
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._settings: Dict[str, Any] = {}
self._muted: bool = False
self._user_id: str = ""
self._ttfs_p99_latency = ttfs_p99_latency
@@ -122,6 +134,7 @@ class STTService(AIService):
self._user_speaking: bool = False
self._finalize_pending: bool = False
self._finalize_requested: bool = False
self._last_transcript_time: float = 0
# Keepalive state
self._keepalive_timeout = keepalive_timeout
@@ -179,18 +192,53 @@ class STTService(AIService):
async def set_model(self, model: str):
"""Set the speech recognition model.
.. deprecated:: 0.0.104
Use ``STTUpdateSettingsFrame(model=...)`` instead.
Args:
model: The name of the model to use for speech recognition.
"""
self.set_model_name(model)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated, use 'STTUpdateSettingsFrame(model=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching STT model to: [{model}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(model=model))
async def set_language(self, language: Language):
"""Set the language for speech recognition.
.. deprecated:: 0.0.104
Use ``STTUpdateSettingsFrame(language=...)`` instead.
Args:
language: The language to use for speech recognition.
"""
pass
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_language' is deprecated, use 'STTUpdateSettingsFrame(language=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching STT language to: [{language}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(language=language))
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a language to the service-specific language format.
Args:
language: The language to convert.
Returns:
The service-specific language identifier, or None if not supported.
"""
return Language(language)
@abstractmethod
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
@@ -222,20 +270,29 @@ class STTService(AIService):
await self._cancel_ttfb_timeout()
await self._cancel_keepalive_task()
async def _update_settings(self, settings: Mapping[str, Any]):
logger.info(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating STT setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
await self.set_language(value)
elif key == "language":
await self.set_language(value)
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for STT service: {key}")
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply an STT settings delta.
Handles ``model`` (via parent). Translates ``Language`` enum values
before applying so the stored value is a service-specific string.
Concrete services should override this method and handle language
changes (including any reconnect logic) based on the returned
changed-field dict.
Args:
delta: An STT settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
# Translate language *before* applying so the stored value is canonical
if is_given(delta.language) and isinstance(delta.language, Language):
converted = self.language_to_service_language(delta.language)
if converted is not None:
delta.language = converted
changed = await super()._update_settings(delta)
return changed
async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
"""Process an audio frame for speech recognition.
@@ -300,7 +357,20 @@ class STTService(AIService):
await self._handle_vad_user_stopped_speaking(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, STTUpdateSettingsFrame):
await self._update_settings(frame.settings)
if frame.delta is not None:
await self._update_settings(frame.delta)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via STTUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.104, use STTUpdateSettingsFrame(delta=STTSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
delta = type(self._settings).from_mapping(frame.settings)
await self._update_settings(delta)
elif isinstance(frame, STTMuteFrame):
self._muted = frame.mute
logger.debug(f"STT service {'muted' if frame.mute else 'unmuted'}")
@@ -323,6 +393,9 @@ class STTService(AIService):
direction: The direction to push the frame.
"""
if isinstance(frame, TranscriptionFrame):
# Store the transcript time for TTFB calculation
self._last_transcript_time = time.time()
# Set finalized from pending state and auto-reset
if self._finalize_pending:
frame.finalized = True
@@ -376,6 +449,7 @@ class STTService(AIService):
self._user_speaking = True
self._finalize_requested = False
self._finalize_pending = False
self._last_transcript_time = 0
async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame):
"""Handle VAD user stopped speaking frame.
@@ -405,14 +479,17 @@ class STTService(AIService):
)
async def _ttfb_timeout_handler(self):
"""Wait for timeout then report TTFB.
"""Wait for timeout then report TTFB using the last transcript timestamp.
This timeout allows the final transcription to arrive before we calculate
and report TTFB. If no transcription arrived, no TTFB is reported.
and report TTFB. Uses _last_transcript_time as the end time so we measure
to when the transcript actually arrived, not when the timeout fired.
If no transcription arrived, no TTFB is reported.
"""
try:
await asyncio.sleep(self._stt_ttfb_timeout)
await self.stop_ttfb_metrics()
if self._last_transcript_time > 0:
await self.stop_ttfb_metrics(end_time=self._last_transcript_time)
except asyncio.CancelledError:
# Task was cancelled (new utterance or interruption), which is expected behavior
pass

5
src/pipecat/services/tavus/video.py
View File

@@ -94,6 +94,7 @@ class TavusVideoService(AIService):
"""
await super().setup(setup)
callbacks = TavusCallbacks(
on_joined=self._on_joined,
on_participant_joined=self._on_participant_joined,
on_participant_left=self._on_participant_left,
)
@@ -119,6 +120,10 @@ class TavusVideoService(AIService):
await self._client.cleanup()
self._client = None
async def _on_joined(self, data):
"""Handle bot joined the Daily room."""
logger.info("Tavus bot joined Daily room")
async def _on_participant_left(self, participant, reason):
"""Handle participant leaving the session."""
participant_id = participant["id"]

Some files were not shown because too many files have changed in this diff Show More