wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'pipecat-ai:main' into add-support-for-private-endpoint-azure-stt

Browse Source
This commit is contained in:
Radek Sedlák
2026-03-01 17:55:45 +01:00
committed by GitHub
parent 5ea2d47d39 903dc6c1a9
commit 6fc718947d
307 changed files with 22779 additions and 5003 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/pipecat/audio/filters/krisp_viva_filter.py
View File

@@ -39,7 +39,11 @@ class KrispVivaFilter(BaseAudioFilter):
"""
def __init__(
self, model_path: str = None, frame_duration: int = 10, noise_suppression_level: int = 100
self,
model_path: str = None,
frame_duration: int = 10,
noise_suppression_level: int = 100,
api_key: str = "",
) -> None:
"""Initialize the Krisp noise reduction filter.
@@ -48,6 +52,8 @@ class KrispVivaFilter(BaseAudioFilter):
If None, uses KRISP_VIVA_FILTER_MODEL_PATH environment variable.
frame_duration: Frame duration in milliseconds.
noise_suppression_level: Noise suppression level.
api_key: Krisp SDK API key. If empty, falls back to
the KRISP_VIVA_API_KEY environment variable.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_FILTER_MODEL_PATH is not set.
@@ -57,6 +63,8 @@ class KrispVivaFilter(BaseAudioFilter):
"""
super().__init__()
self._api_key = api_key
try:
# Set model path, checking environment if not specified
if model_path:
@@ -132,7 +140,7 @@ class KrispVivaFilter(BaseAudioFilter):
"""
try:
# Acquire SDK reference (will initialize on first call)
KrispVivaSDKManager.acquire()
KrispVivaSDKManager.acquire(api_key=self._api_key)
self._session = self._create_session(sample_rate, self._frame_duration_ms)
except Exception as e:
logger.error(f"Failed to start Krisp session: {e}", exc_info=True)

26
src/pipecat/audio/krisp_instance.py
View File

@@ -7,6 +7,7 @@
"""Krisp Instance manager for pipecat audio."""
import atexit
import os
from threading import Lock
from loguru import logger
@@ -88,17 +89,26 @@ class KrispVivaSDKManager:
_lock = Lock()
_reference_count = 0
@staticmethod
def _license_callback(error, error_message):
"""Callback for Krisp SDK licensing errors."""
logger.error(f"Krisp licensing error: {error} - {error_message}")
@staticmethod
def _log_callback(log_message, log_level):
"""Thread-safe callback for Krisp SDK logging."""
logger.info(f"[{log_level}] {log_message}")
@classmethod
def acquire(cls):
def acquire(cls, api_key: str = ""):
"""Acquire a reference to the SDK (initializes if needed).
Call this when creating a filter instance.
Args:
api_key: Krisp SDK API key. If empty, falls back to the
KRISP_VIVA_API_KEY environment variable.
Raises:
Exception: If SDK initialization fails (propagated from krisp_audio)
"""
@@ -106,7 +116,19 @@ class KrispVivaSDKManager:
# Initialize SDK on first acquire
if cls._reference_count == 0:
try:
krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
key = api_key or os.environ.get("KRISP_VIVA_API_KEY", "")
try:
# New SDK signature (requires license key)
krisp_audio.globalInit(
"",
key,
cls._license_callback,
cls._log_callback,
krisp_audio.LogLevel.Off,
)
except TypeError:
# Old SDK signature (no license key)
krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
cls._initialized = True

39
src/pipecat/audio/turn/krisp_viva_turn.py
View File

@@ -15,6 +15,7 @@ passed directly to the constructor.
"""
import os
import time
from typing import Optional, Tuple
import numpy as np
@@ -26,7 +27,7 @@ from pipecat.audio.krisp_instance import (
int_to_krisp_sample_rate,
)
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData
from pipecat.metrics.metrics import MetricsData, TurnMetricsData
try:
import krisp_audio
@@ -63,6 +64,7 @@ class KrispVivaTurn(BaseTurnAnalyzer):
model_path: Optional[str] = None,
sample_rate: Optional[int] = None,
params: Optional[KrispTurnParams] = None,
api_key: str = "",
) -> None:
"""Initialize the Krisp turn analyzer.
@@ -72,6 +74,8 @@ class KrispVivaTurn(BaseTurnAnalyzer):
sample_rate: Optional initial sample rate for audio processing.
If provided, this will be used as the fixed sample rate.
params: Configuration parameters for turn analysis behavior.
api_key: Krisp SDK API key. If empty, falls back to
the KRISP_VIVA_API_KEY environment variable.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_TURN_MODEL_PATH is not set.
@@ -83,7 +87,7 @@ class KrispVivaTurn(BaseTurnAnalyzer):
# Acquire SDK reference (will initialize on first call)
try:
KrispVivaSDKManager.acquire()
KrispVivaSDKManager.acquire(api_key=api_key)
self._sdk_acquired = True
except Exception as e:
self._sdk_acquired = False
@@ -115,6 +119,9 @@ class KrispVivaTurn(BaseTurnAnalyzer):
self._last_probability = None
self._frame_probabilities = []
self._last_state = EndOfTurnState.INCOMPLETE
self._speech_stopped_time: Optional[float] = None
self._e2e_processing_time_ms: Optional[float] = None
self._last_metrics: Optional[TurnMetricsData] = None
# Create session with provided sample rate or default to 16000 Hz
# This preloads the model to improve latency when set_sample_rate is called later
@@ -288,7 +295,14 @@ class KrispVivaTurn(BaseTurnAnalyzer):
# Track speech start time
if not self._speech_triggered:
logger.trace("Speech detected, turn analysis started")
self._e2e_processing_time_ms = None
self._speech_triggered = True
# Reset speech stopped time when speech resumes
self._speech_stopped_time = None
else:
# Record the moment speech transitions to non-speech
if self._speech_triggered and self._speech_stopped_time is None:
self._speech_stopped_time = time.perf_counter()
# Note: We don't immediately mark as complete on silence detection.
# Instead, we wait for the model's probability check below to confirm
# end-of-turn based on the threshold.
@@ -308,6 +322,18 @@ class KrispVivaTurn(BaseTurnAnalyzer):
# Only mark as complete if we've detected speech and the model
# confirms with sufficient confidence
if self._speech_triggered and prob >= self._params.threshold:
# Calculate e2e processing time: time from speech stop to threshold crossing
if self._speech_stopped_time is not None:
self._e2e_processing_time_ms = (
time.perf_counter() - self._speech_stopped_time
) * 1000
self._last_metrics = TurnMetricsData(
processor="KrispVivaTurn",
is_complete=True,
probability=prob,
e2e_processing_time_ms=self._e2e_processing_time_ms,
)
logger.debug(f"Krisp turn complete")
state = EndOfTurnState.COMPLETE
self.clear()
break
@@ -329,12 +355,15 @@ class KrispVivaTurn(BaseTurnAnalyzer):
Tuple containing the end-of-turn state and optional metrics data.
Returns the last state determined by append_audio().
"""
# For real-time processing, the state is determined in append_audio
# Return the last state that was computed
return self._last_state, None
# For real-time processing, the state is determined in append_audio.
# Consume metrics so they aren't pushed twice.
metrics = self._last_metrics
self._last_metrics = None
return self._last_state, metrics
def clear(self):
"""Reset the turn analyzer to its initial state."""
self._speech_triggered = False
self._audio_buffer.clear()
self._last_state = EndOfTurnState.INCOMPLETE
self._speech_stopped_time = None

13
src/pipecat/audio/turn/smart_turn/base_smart_turn.py
View File

@@ -21,7 +21,7 @@ import numpy as np
from loguru import logger
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData
from pipecat.metrics.metrics import MetricsData, TurnMetricsData
# Default timing parameters
STOP_SECS = 3
@@ -222,18 +222,11 @@ class BaseSmartTurn(BaseTurnAnalyzer):
# Calculate processing time
e2e_processing_time_ms = (end_time - start_time) * 1000
# Extract metrics from the nested structure
metrics = result.get("metrics", {})
inference_time = metrics.get("inference_time", 0)
total_time = metrics.get("total_time", 0)
# Prepare the result data
result_data = SmartTurnMetricsData(
result_data = TurnMetricsData(
processor="BaseSmartTurn",
is_complete=result["prediction"] == 1,
probability=result["probability"],
inference_time_ms=inference_time * 1000,
server_total_time_ms=total_time * 1000,
e2e_processing_time_ms=e2e_processing_time_ms,
)
@@ -241,8 +234,6 @@ class BaseSmartTurn(BaseTurnAnalyzer):
f"Prediction: {'Complete' if result_data.is_complete else 'Incomplete'}"
)
logger.trace(f"Probability of complete: {result_data.probability:.4f}")
logger.trace(f"Inference time: {result_data.inference_time_ms:.2f}ms")
logger.trace(f"Server total time: {result_data.server_total_time_ms:.2f}ms")
logger.trace(f"E2E processing time: {result_data.e2e_processing_time_ms:.2f}ms")
except SmartTurnTimeoutException:
logger.debug(

12
src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py
View File

@@ -13,21 +13,13 @@ local end-of-turn detection without requiring network connectivity.
from typing import Any, Dict, Optional
import numpy as np
import onnxruntime as ort
from loguru import logger
from transformers import WhisperFeatureExtractor
from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn
from pipecat.utils.env import env_truthy
try:
import onnxruntime as ort
from transformers import WhisperFeatureExtractor
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use LocalSmartTurnAnalyzerV3, you need to `pip install pipecat-ai[local-smart-turn-v3]`."
)
raise Exception(f"Missing module: {e}")
class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
"""Local turn analyzer using the smart-turn-v3 ONNX model.

76
src/pipecat/frames/frames.py
View File

@@ -14,7 +14,6 @@ and LLM processing.
import asyncio
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import (
TYPE_CHECKING,
Any,
@@ -36,12 +35,15 @@ from pipecat.audio.turn.base_turn_analyzer import BaseTurnParams
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import AggregationType
from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
if TYPE_CHECKING:
from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.services.settings import ServiceSettings
from pipecat.utils.context.llm_context_summarization import LLMContextSummaryConfig
from pipecat.utils.tracing.tracing_context import TracingContext
@@ -123,6 +125,9 @@ class Frame:
id: Unique identifier for the frame instance.
name: Human-readable name combining class name and instance count.
pts: Presentation timestamp in nanoseconds.
broadcast_sibling_id: ID of the paired frame when this frame was
broadcast in both directions. Set automatically by
``broadcast_frame()`` and ``broadcast_frame_instance()``.
metadata: Dictionary for arbitrary frame metadata.
transport_source: Name of the transport source that created this frame.
transport_destination: Name of the transport destination for this frame.
@@ -131,6 +136,7 @@ class Frame:
id: int = field(init=False)
name: str = field(init=False)
pts: Optional[int] = field(init=False)
broadcast_sibling_id: Optional[int] = field(init=False)
metadata: Dict[str, Any] = field(init=False)
transport_source: Optional[str] = field(init=False)
transport_destination: Optional[str] = field(init=False)
@@ -139,6 +145,7 @@ class Frame:
self.id: int = obj_id()
self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self.pts: Optional[int] = None
self.broadcast_sibling_id: Optional[int] = None
self.metadata: Dict[str, Any] = {}
self.transport_source: Optional[str] = None
self.transport_destination: Optional[str] = None
@@ -387,16 +394,6 @@ class LLMTextFrame(TextFrame):
self.includes_inter_frame_spaces = True
class AggregationType(str, Enum):
"""Built-in aggregation strings."""
SENTENCE = "sentence"
WORD = "word"
def __str__(self):
return self.value
@dataclass
class AggregatedTextFrame(TextFrame):
"""Text frame representing an aggregation of TextFrames.
@@ -1994,6 +1991,32 @@ class LLMFullResponseEndFrame(ControlFrame):
self.skip_tts = None
@dataclass
class LLMAssistantPushAggregationFrame(ControlFrame):
"""Frame that forces the LLM assistant aggregator to push its current aggregation to context.
When received by ``LLMAssistantAggregator``, any text that has been accumulated
in the aggregation buffer is immediately committed to the conversation context as
an assistant message, without waiting for an ``LLMFullResponseEndFrame``.
"""
@dataclass
class LLMSummarizeContextFrame(ControlFrame):
"""Frame requesting on-demand context summarization.
Push this frame into the pipeline to trigger a manual context summarization.
Parameters:
config: Optional per-request override for summary generation settings
(prompt, token budget, messages to keep). If ``None``, the
summarizer's default :class:`~pipecat.utils.context.llm_context_summarization.LLMContextSummaryConfig`
is used.
"""
config: Optional["LLMContextSummaryConfig"] = None
@dataclass
class LLMContextSummaryRequestFrame(ControlFrame):
"""Frame requesting context summarization from an LLM service.
@@ -2013,6 +2036,8 @@ class LLMContextSummaryRequestFrame(ControlFrame):
the summary text.
summarization_prompt: System prompt instructing the LLM how to generate
the summary.
summarization_timeout: Maximum time in seconds for the LLM to generate a
summary. When None, a default timeout of 120s is applied.
"""
request_id: str
@@ -2020,6 +2045,7 @@ class LLMContextSummaryRequestFrame(ControlFrame):
min_messages_to_keep: int
target_context_tokens: int
summarization_prompt: str
summarization_timeout: Optional[float] = None
@dataclass
@@ -2112,16 +2138,24 @@ class TTSStoppedFrame(ControlFrame):
@dataclass
class ServiceUpdateSettingsFrame(ControlFrame):
class ServiceUpdateSettingsFrame(ControlFrame, UninterruptibleFrame):
"""Base frame for updating service settings.
A control frame containing a request to update service settings.
Supports both a ``settings`` dict (for backward compatibility) and a
``delta`` object. When both are provided, ``delta`` takes precedence.
Parameters:
settings: Dictionary of setting name to value mappings.
.. deprecated:: 0.0.104
Use ``delta`` with a typed settings object instead.
delta: :class:`~pipecat.services.settings.ServiceSettings` delta-mode
object describing the fields to change.
"""
settings: Mapping[str, Any]
settings: Mapping[str, Any] = field(default_factory=dict)
delta: Optional["ServiceSettings"] = None
@dataclass
@@ -2145,6 +2179,20 @@ class STTUpdateSettingsFrame(ServiceUpdateSettingsFrame):
pass
@dataclass
class UserIdleTimeoutUpdateFrame(SystemFrame):
"""Frame for updating the user idle timeout at runtime.
Setting timeout to 0 disables idle detection. Setting a positive value
enables it.
Parameters:
timeout: The new idle timeout in seconds. 0 disables idle detection.
"""
timeout: float
@dataclass
class VADParamsUpdateFrame(ControlFrame):
"""Frame for updating VAD parameters.

43
src/pipecat/metrics/metrics.py
View File

@@ -41,6 +41,10 @@ class TTFBMetricsData(MetricsData):
class ProcessingMetricsData(MetricsData):
"""General processing time metrics data.
.. deprecated:: 0.0.104
Processing metrics are deprecated and will be removed in a future version.
Use TTFB metrics instead.
Parameters:
value: Processing time measurement in seconds.
"""
@@ -87,19 +91,44 @@ class TTSUsageMetricsData(MetricsData):
value: int
class SmartTurnMetricsData(MetricsData):
"""Metrics data for smart turn predictions.
class TextAggregationMetricsData(MetricsData):
"""Text aggregation time metrics data.
Measures the time from the first LLM token to the first complete sentence,
representing the latency cost of sentence aggregation in the TTS pipeline.
Parameters:
value: Aggregation time in seconds.
"""
value: float
class TurnMetricsData(MetricsData):
"""Metrics data for turn detection predictions.
Parameters:
is_complete: Whether the turn is predicted to be complete.
probability: Confidence probability of the turn completion prediction.
inference_time_ms: Time taken for inference in milliseconds.
server_total_time_ms: Total server processing time in milliseconds.
e2e_processing_time_ms: End-to-end processing time in milliseconds.
e2e_processing_time_ms: End-to-end processing time in milliseconds,
measured from VAD speech-to-silence transition to turn completion.
"""
is_complete: bool
probability: float
inference_time_ms: float
server_total_time_ms: float
e2e_processing_time_ms: float
class SmartTurnMetricsData(TurnMetricsData):
"""Metrics data for smart turn predictions.
.. deprecated:: 0.0.104
Use :class:`TurnMetricsData` instead. This class will be removed in a future version.
Parameters:
inference_time_ms: Time taken for inference in milliseconds.
server_total_time_ms: Total server processing time in milliseconds.
"""
inference_time_ms: float = 0.0
server_total_time_ms: float = 0.0

33
src/pipecat/observers/loggers/metrics_log_observer.py
View File

@@ -24,6 +24,7 @@ from pipecat.metrics.metrics import (
SmartTurnMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
TurnMetricsData,
)
from pipecat.observers.base_observer import BaseObserver, FramePushed
@@ -37,7 +38,7 @@ class MetricsLogObserver(BaseObserver):
- ProcessingMetricsData (General processing time)
- LLMUsageMetricsData (Token usage statistics)
- TTSUsageMetricsData (Text-to-Speech character counts)
- SmartTurnMetricsData (Turn prediction metrics)
- TurnMetricsData (Turn prediction metrics)
This allows developers to track performance metrics, token usage,
and other statistics throughout the pipeline.
@@ -70,6 +71,17 @@ class MetricsLogObserver(BaseObserver):
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
# Normalize deprecated types in include_metrics
if include_metrics and SmartTurnMetricsData in include_metrics:
import warnings
warnings.warn(
"SmartTurnMetricsData is deprecated in include_metrics, "
"use TurnMetricsData instead.",
DeprecationWarning,
stacklevel=2,
)
include_metrics = (include_metrics - {SmartTurnMetricsData}) | {TurnMetricsData}
self._include_metrics = include_metrics
self._frames_seen = set()
@@ -144,8 +156,8 @@ class MetricsLogObserver(BaseObserver):
logger.debug(
f"📊 {processor_info} TTS USAGE{model_info}: {metrics_data.value} characters at {time_sec:.3f}s"
)
elif isinstance(metrics_data, SmartTurnMetricsData):
self._log_smart_turn(metrics_data, processor_info, model_info, time_sec)
elif isinstance(metrics_data, TurnMetricsData):
self._log_turn(metrics_data, processor_info, model_info, time_sec)
else:
# Generic fallback for unknown metrics types
logger.debug(
@@ -191,28 +203,27 @@ class MetricsLogObserver(BaseObserver):
f"📊 {processor_info} LLM TOKEN USAGE{model_info}: {usage_str} at {time_sec:.2f}s"
)
def _log_smart_turn(
def _log_turn(
self,
metrics_data: SmartTurnMetricsData,
metrics_data: TurnMetricsData,
processor_info: str,
model_info: str,
time_sec: float,
):
"""Log smart turn prediction metrics.
"""Log turn prediction metrics.
Args:
metrics_data: The smart turn metrics data.
metrics_data: The turn metrics data.
processor_info: Formatted processor name string.
model_info: Formatted model name string.
time_sec: Timestamp in seconds.
"""
complete_str = "COMPLETE" if metrics_data.is_complete else "INCOMPLETE"
e2e_str = f"{metrics_data.e2e_processing_time_ms:.1f}ms"
logger.debug(
f"📊 {processor_info} SMART TURN{model_info}: {complete_str} "
f"📊 {processor_info} TURN{model_info}: {complete_str} "
f"(probability: {metrics_data.probability:.2%}, "
f"inference: {metrics_data.inference_time_ms:.1f}ms, "
f"server: {metrics_data.server_total_time_ms:.1f}ms, "
f"e2e: {metrics_data.e2e_processing_time_ms:.1f}ms) "
f"e2e: {e2e_str}) "
f"at {time_sec:.2f}s"
)

42
src/pipecat/pipeline/task.py
View File

@@ -330,6 +330,7 @@ class PipelineTask(BasePipelineTask):
# RTVI support
self._rtvi = None
prepend_rtvi = False
external_rtvi = self._find_processor(pipeline, RTVIProcessor)
external_observer_found = any(isinstance(o, RTVIObserver) for o in observers)
@@ -352,6 +353,7 @@ class PipelineTask(BasePipelineTask):
elif enable_rtvi:
self._rtvi = rtvi_processor or RTVIProcessor()
observers.append(self._rtvi.create_rtvi_observer(params=rtvi_observer_params))
prepend_rtvi = True
if self._rtvi:
# Automatically call RTVIProcessor.set_bot_ready()
@@ -387,9 +389,12 @@ class PipelineTask(BasePipelineTask):
# source allows us to receive and react to upstream frames, and the sink
# allows us to receive and react to downstream frames.
source = PipelineSource(self._source_push_frame, name=f"{self}::Source")
sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
processors = [self._rtvi, pipeline] if self._rtvi else [pipeline]
self._pipeline = Pipeline(processors, source=source, sink=sink)
self._sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
# Only prepend the RTVIProcessor if we created it ourselves. When the
# user already placed it inside their pipeline we must not insert it
# again or it will appear twice in the frame chain.
processors = [self._rtvi, pipeline] if prepend_rtvi else [pipeline]
self._pipeline = Pipeline(processors, source=source, sink=self._sink)
# The task observer acts as a proxy to the provided observers. This way,
# we only need to pass a single observer (using the StartFrame) which
@@ -620,26 +625,43 @@ class PipelineTask(BasePipelineTask):
self._finished = True
logger.debug(f"Pipeline task {self} has finished")
async def queue_frame(self, frame: Frame):
"""Queue a single frame to be pushed down the pipeline.
async def queue_frame(
self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
):
"""Queue a single frame to be pushed through the pipeline.
Downstream frames are pushed from the beginning of the pipeline.
Upstream frames are pushed from the end of the pipeline.
Args:
frame: The frame to be processed.
direction: The direction to push the frame. Defaults to downstream.
"""
await self._push_queue.put(frame)
if direction == FrameDirection.DOWNSTREAM:
await self._push_queue.put(frame)
else:
await self._sink.queue_frame(frame, direction)
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
"""Queues multiple frames to be pushed down the pipeline.
async def queue_frames(
self,
frames: Iterable[Frame] | AsyncIterable[Frame],
direction: FrameDirection = FrameDirection.DOWNSTREAM,
):
"""Queue multiple frames to be pushed through the pipeline.
Downstream frames are pushed from the beginning of the pipeline.
Upstream frames are pushed from the end of the pipeline.
Args:
frames: An iterable or async iterable of frames to be processed.
direction: The direction to push the frames. Defaults to downstream.
"""
if isinstance(frames, AsyncIterable):
async for frame in frames:
await self.queue_frame(frame)
await self.queue_frame(frame, direction)
elif isinstance(frames, Iterable):
for frame in frames:
await self.queue_frame(frame)
await self.queue_frame(frame, direction)
async def _cancel(self, *, reason: Optional[str] = None):
"""Internal cancellation logic for the pipeline task.

232
src/pipecat/processors/aggregators/llm_context_summarizer.py
View File

@@ -6,8 +6,10 @@
"""This module defines a summarizer for managing LLM context summarization."""
import asyncio
import uuid
from typing import Optional
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from loguru import logger
@@ -17,28 +19,68 @@ from pipecat.frames.frames import (
LLMContextSummaryRequestFrame,
LLMContextSummaryResultFrame,
LLMFullResponseStartFrame,
LLMSummarizeContextFrame,
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject
from pipecat.utils.context.llm_context_summarization import (
LLMContextSummarizationConfig,
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMAutoContextSummarizationConfig,
LLMContextSummarizationUtil,
LLMContextSummaryConfig,
)
if TYPE_CHECKING:
from pipecat.services.llm_service import LLMService
@dataclass
class SummaryAppliedEvent:
"""Event data emitted when context summarization completes successfully.
Parameters:
original_message_count: Number of messages before summarization.
new_message_count: Number of messages after summarization.
summarized_message_count: Number of messages that were compressed
into the summary.
preserved_message_count: Number of recent messages preserved
uncompressed.
"""
original_message_count: int
new_message_count: int
summarized_message_count: int
preserved_message_count: int
class LLMContextSummarizer(BaseObject):
"""Summarizer for managing LLM context summarization.
This class manages automatic context summarization when token or message
limits are reached. It monitors the LLM context size, triggers
summarization requests, and applies the results to compress conversation history.
This class manages context summarization, either automatically when token or
message limits are reached, or on-demand when an ``LLMSummarizeContextFrame``
is received. It monitors the LLM context size, triggers summarization requests,
and applies the results to compress conversation history.
When ``auto_trigger=True`` (the default), summarization is triggered
automatically based on the configured thresholds in
``LLMAutoContextSummarizationConfig``. When ``auto_trigger=False``,
threshold checks are skipped and summarization only happens when an
``LLMSummarizeContextFrame`` is explicitly pushed into the pipeline.
Both modes can coexist: set ``auto_trigger=True`` and also push
``LLMSummarizeContextFrame`` at any time to force an immediate summarization
(subject to the ``_summarization_in_progress`` guard).
Event handlers available:
- on_request_summarization: Emitted when summarization should be triggered.
The aggregator should broadcast this frame to the LLM service.
- on_summary_applied: Emitted after a summary has been successfully applied
to the context. Receives a SummaryAppliedEvent with metrics about the
compression.
Example::
@summarizer.event_handler("on_request_summarization")
@@ -49,24 +91,36 @@ class LLMContextSummarizer(BaseObject):
context=frame.context,
...
)
@summarizer.event_handler("on_summary_applied")
async def on_summary_applied(summarizer, event: SummaryAppliedEvent):
logger.info(f"Compressed {event.original_message_count} -> {event.new_message_count} messages")
"""
def __init__(
self,
*,
context: LLMContext,
config: Optional[LLMContextSummarizationConfig] = None,
config: Optional[LLMAutoContextSummarizationConfig] = None,
auto_trigger: bool = True,
):
"""Initialize the context summarizer.
Args:
context: The LLM context to monitor and summarize.
config: Configuration for summarization behavior. If None, uses default config.
config: Auto-summarization configuration controlling both trigger
thresholds and default summary generation parameters. If None,
uses default ``LLMAutoContextSummarizationConfig`` values.
auto_trigger: Whether to automatically trigger summarization when
thresholds are reached. When False, summarization only happens
when an ``LLMSummarizeContextFrame`` is pushed into the pipeline.
Defaults to True.
"""
super().__init__()
self._context = context
self._config = config or LLMContextSummarizationConfig()
self._auto_config = config or LLMAutoContextSummarizationConfig()
self._auto_trigger = auto_trigger
self._task_manager: Optional[BaseTaskManager] = None
@@ -74,6 +128,7 @@ class LLMContextSummarizer(BaseObject):
self._pending_summary_request_id: Optional[str] = None
self._register_event_handler("on_request_summarization", sync=True)
self._register_event_handler("on_summary_applied")
@property
def task_manager(self) -> BaseTaskManager:
@@ -103,6 +158,8 @@ class LLMContextSummarizer(BaseObject):
"""
if isinstance(frame, LLMFullResponseStartFrame):
await self._handle_llm_response_start(frame)
elif isinstance(frame, LLMSummarizeContextFrame):
await self._handle_manual_summarization_request(frame)
elif isinstance(frame, LLMContextSummaryResultFrame):
await self._handle_summary_result(frame)
elif isinstance(frame, InterruptionFrame):
@@ -117,12 +174,24 @@ class LLMContextSummarizer(BaseObject):
if self._should_summarize():
await self._request_summarization()
async def _handle_interruption(self):
"""Handle interruption by canceling summarization in progress.
async def _handle_manual_summarization_request(self, frame: LLMSummarizeContextFrame):
"""Handle an explicit on-demand summarization request.
Reuses the same ``_request_summarization()`` code path as auto mode,
so bookkeeping (``_summarization_in_progress``,
``_pending_summary_request_id``) is always updated correctly.
Args:
frame: The interruption frame.
frame: The manual summarization request frame, optionally carrying
a per-request :class:`~pipecat.utils.context.llm_context_summarization.LLMContextSummaryConfig`.
"""
if self._summarization_in_progress:
logger.debug(f"{self}: Summarization already in progress, ignoring manual request")
return
await self._request_summarization(config_override=frame.config)
async def _handle_interruption(self):
"""Handle interruption by canceling summarization in progress."""
# Reset summarization state to allow new requests. This is necessary because
# the request frame (LLMContextSummaryRequestFrame) may have been cancelled
# during interruption. We preserve _pending_summary_request_id to handle the
@@ -145,13 +214,17 @@ class LLMContextSummarizer(BaseObject):
Returns:
True if all conditions are met:
- ``auto_trigger`` is enabled
- No summarization currently in progress
- AND either:
- Token count exceeds max_context_tokens
- OR message count exceeds max_unsummarized_messages since last summary
- Token count exceeds ``max_context_tokens``
- OR message count exceeds ``max_unsummarized_messages`` since last summary
"""
logger.trace(f"{self}: Checking if context summarization is needed")
if not self._auto_trigger:
return False
if self._summarization_in_progress:
logger.debug(f"{self}: Summarization already in progress")
return False
@@ -161,20 +234,20 @@ class LLMContextSummarizer(BaseObject):
num_messages = len(self._context.messages)
# Check if we've reached the token limit
token_limit = self._config.max_context_tokens
token_limit = self._auto_config.max_context_tokens
token_limit_exceeded = total_tokens >= token_limit
# Check if we've exceeded max unsummarized messages
messages_since_summary = len(self._context.messages) - 1
message_threshold_exceeded = (
messages_since_summary >= self._config.max_unsummarized_messages
messages_since_summary >= self._auto_config.max_unsummarized_messages
)
logger.trace(
f"{self}: Context has {num_messages} messages, "
f"~{total_tokens} tokens (limit: {token_limit}), "
f"{messages_since_summary} messages since last summary "
f"(message threshold: {self._config.max_unsummarized_messages})"
f"(message threshold: {self._auto_config.max_unsummarized_messages})"
)
# Trigger if either limit is exceeded
@@ -189,21 +262,30 @@ class LLMContextSummarizer(BaseObject):
reason.append(f"~{total_tokens} tokens (>={token_limit} limit)")
if message_threshold_exceeded:
reason.append(
f"{messages_since_summary} messages (>={self._config.max_unsummarized_messages} threshold)"
f"{messages_since_summary} messages (>={self._auto_config.max_unsummarized_messages} threshold)"
)
logger.debug(f"{self}: ✓ Summarization needed - {', '.join(reason)}")
return True
async def _request_summarization(self):
async def _request_summarization(
self, config_override: Optional[LLMContextSummaryConfig] = None
):
"""Request context summarization from LLM service.
Creates a summarization request frame and emits it via event handler.
Creates a summarization request frame and either handles it directly
using a dedicated LLM (if configured) or emits it via event handler
for the pipeline's primary LLM.
Tracks the request ID to match async responses and prevent race conditions.
Args:
config_override: Optional per-request summary configuration. If provided,
overrides the default summary generation settings from
``self._auto_config.summary_config``.
"""
# Generate unique request ID
request_id = str(uuid.uuid4())
min_keep = self._config.min_messages_after_summary
summary_config = config_override or self._auto_config.summary_config
# Mark summarization in progress
self._summarization_in_progress = True
@@ -215,13 +297,66 @@ class LLMContextSummarizer(BaseObject):
request_frame = LLMContextSummaryRequestFrame(
request_id=request_id,
context=self._context,
min_messages_to_keep=min_keep,
target_context_tokens=self._config.target_context_tokens,
summarization_prompt=self._config.summary_prompt,
min_messages_to_keep=summary_config.min_messages_after_summary,
target_context_tokens=summary_config.target_context_tokens,
summarization_prompt=summary_config.summary_prompt,
summarization_timeout=summary_config.summarization_timeout,
)
# Emit event for aggregator to broadcast
await self._call_event_handler("on_request_summarization", request_frame)
if summary_config.llm:
# Use dedicated LLM directly — no need to involve the pipeline
self.task_manager.create_task(
self._generate_summary_with_dedicated_llm(summary_config.llm, request_frame),
f"{self}-dedicated-llm-summary",
)
else:
# Emit event for aggregator to broadcast to the pipeline LLM
await self._call_event_handler("on_request_summarization", request_frame)
async def _generate_summary_with_dedicated_llm(
self, llm: "LLMService", frame: LLMContextSummaryRequestFrame
):
"""Generate summary using a dedicated LLM service.
Calls the dedicated LLM's _generate_summary directly and feeds the
result back through _handle_summary_result, bypassing the pipeline.
Args:
llm: The dedicated LLM service to use for summarization.
frame: The summarization request frame.
"""
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try:
summary, last_index = await asyncio.wait_for(
llm._generate_summary(frame),
timeout=timeout,
)
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary=summary,
last_summarized_index=last_index,
)
except asyncio.TimeoutError:
error = f"Context summarization timed out after {timeout}s"
logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary="",
last_summarized_index=-1,
error=error,
)
except Exception as e:
error = f"Error generating context summary: {e}"
logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary="",
last_summarized_index=-1,
error=error,
)
await self._handle_summary_result(result_frame)
async def _handle_summary_result(self, frame: LLMContextSummaryResultFrame):
"""Handle context summarization result from LLM service.
@@ -234,7 +369,9 @@ class LLMContextSummarizer(BaseObject):
"""
logger.debug(f"{self}: Received summary result (request_id={frame.request_id})")
# Check if this is the result we're waiting for
# Check if this is the result we're waiting for. Both auto and manual
# summarization set _pending_summary_request_id via _request_summarization(),
# so this check always applies.
if frame.request_id != self._pending_summary_request_id:
logger.debug(f"{self}: Ignoring stale summary result (request_id={frame.request_id})")
return
@@ -271,7 +408,7 @@ class LLMContextSummarizer(BaseObject):
if last_summarized_index >= len(self._context.messages):
return False
min_keep = self._config.min_messages_after_summary
min_keep = self._auto_config.summary_config.min_messages_after_summary
remaining = len(self._context.messages) - 1 - last_summarized_index
if remaining < min_keep:
return False
@@ -288,16 +425,29 @@ class LLMContextSummarizer(BaseObject):
summary: The generated summary text.
last_summarized_index: Index of the last message that was summarized.
"""
config = self._auto_config.summary_config
messages = self._context.messages
# Find the first system message to preserve
first_system_msg = next((msg for msg in messages if msg.get("role") == "system"), None)
# Find the first system message to preserve. LLMSpecificMessage instances are excluded
# because they are not dict-like and never represent a system message; they hold
# service-specific metadata (e.g. thinking blocks) that is always paired with a
# standard message.
first_system_msg = next(
(
msg
for msg in messages
if not isinstance(msg, LLMSpecificMessage) and msg.get("role") == "system"
),
None,
)
# Get recent messages to keep
recent_messages = messages[last_summarized_index + 1 :]
# Create summary message as an assistant message
summary_message = {"role": "assistant", "content": f"Conversation summary: {summary}"}
# Create summary message as a user message (the summary is context
# provided *to* the assistant, not something the assistant said)
summary_content = config.summary_message_template.format(summary=summary)
summary_message = {"role": "user", "content": summary_content}
# Reconstruct context
new_messages = []
@@ -307,9 +457,23 @@ class LLMContextSummarizer(BaseObject):
new_messages.extend(recent_messages)
# Update context
original_message_count = len(messages)
num_system_preserved = 1 if first_system_msg else 0
self._context.set_messages(new_messages)
# Messages actually summarized = index range minus the preserved system message
summarized_count = last_summarized_index + 1 - num_system_preserved
logger.info(
f"{self}: Applied context summary, compressed {last_summarized_index + 1} messages "
f"into summary. Context now has {len(new_messages)} messages (was {len(messages)})"
f"{self}: Applied context summary, compressed {summarized_count} messages "
f"into summary. Context now has {len(new_messages)} messages (was {original_message_count})"
)
# Emit event for observability
event = SummaryAppliedEvent(
original_message_count=original_message_count,
new_message_count=len(new_messages),
summarized_message_count=summarized_count,
preserved_message_count=len(recent_messages) + num_system_preserved,
)
await self._call_event_handler("on_summary_applied", event)

135
src/pipecat/processors/aggregators/llm_response_universal.py
View File

@@ -35,6 +35,7 @@ from pipecat.frames.frames import (
InputAudioRawFrame,
InterimTranscriptionFrame,
InterruptionFrame,
LLMAssistantPushAggregationFrame,
LLMContextAssistantTimestampFrame,
LLMContextFrame,
LLMContextSummaryRequestFrame,
@@ -78,7 +79,10 @@ from pipecat.turns.user_stop import BaseUserTurnStopStrategy, UserTurnStoppedPar
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionConfig
from pipecat.turns.user_turn_controller import UserTurnController
from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies, UserTurnStrategies
from pipecat.utils.context.llm_context_summarization import LLMContextSummarizationConfig
from pipecat.utils.context.llm_context_summarization import (
LLMAutoContextSummarizationConfig,
LLMContextSummarizationConfig,
)
from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text
from pipecat.utils.time import time_now_iso8601
@@ -92,9 +96,9 @@ class LLMUserAggregatorParams:
user_mute_strategies: List of user mute strategies.
user_turn_stop_timeout: Time in seconds to wait before considering the
user's turn finished.
user_idle_timeout: Optional timeout in seconds for detecting user idle state.
If set, the aggregator will emit an `on_user_turn_idle` event when the user
has been idle (not speaking) for this duration. Set to None to disable
user_idle_timeout: Timeout in seconds for detecting user idle state.
The aggregator will emit an `on_user_turn_idle` event when the user
has been idle (not speaking) for this duration. Set to 0 to disable
idle detection.
vad_analyzer: Voice Activity Detection analyzer instance.
filter_incomplete_user_turns: Whether to filter out incomplete user turns.
@@ -109,7 +113,7 @@ class LLMUserAggregatorParams:
user_turn_strategies: Optional[UserTurnStrategies] = None
user_mute_strategies: List[BaseUserMuteStrategy] = field(default_factory=list)
user_turn_stop_timeout: float = 5.0
user_idle_timeout: Optional[float] = None
user_idle_timeout: float = 0
vad_analyzer: Optional[VADAnalyzer] = None
filter_incomplete_user_turns: bool = False
user_turn_completion_config: Optional[UserTurnCompletionConfig] = None
@@ -124,18 +128,54 @@ class LLMAssistantAggregatorParams:
in text frames by adding spaces between tokens. This parameter is
ignored when used with the newer LLMAssistantAggregator, which
handles word spacing automatically.
enable_context_summarization: Enable automatic context summarization when token
limits are reached (disabled by default). When enabled, older conversation
messages are automatically compressed into summaries to manage context size.
context_summarization_config: Configuration for context summarization behavior.
Controls thresholds, message preservation, and summarization prompts. If None
and summarization is enabled, uses default configuration values.
enable_auto_context_summarization: Enable automatic context summarization when token
or message-count limits are reached (disabled by default). When enabled,
older conversation messages are automatically compressed into summaries to
manage context size.
auto_context_summarization_config: Configuration for automatic context
summarization. Controls trigger thresholds, message preservation, and
summarization prompts. If None, uses default
``LLMAutoContextSummarizationConfig`` values.
"""
expect_stripped_words: bool = True
enable_context_summarization: bool = False
enable_auto_context_summarization: bool = False
auto_context_summarization_config: Optional[LLMAutoContextSummarizationConfig] = None
# ---------------------------------------------------------------------------
# Deprecated field names — kept for backward compatibility.
# Use enable_auto_context_summarization and auto_context_summarization_config instead.
# ---------------------------------------------------------------------------
enable_context_summarization: Optional[bool] = None
context_summarization_config: Optional[LLMContextSummarizationConfig] = None
def __post_init__(self):
if self.enable_context_summarization is not None:
warnings.warn(
"LLMAssistantAggregatorParams.enable_context_summarization is deprecated. "
"Use enable_auto_context_summarization instead.",
DeprecationWarning,
stacklevel=2,
)
self.enable_auto_context_summarization = self.enable_context_summarization
self.enable_context_summarization = None
if self.context_summarization_config is not None:
warnings.warn(
"LLMAssistantAggregatorParams.context_summarization_config is deprecated. "
"Use auto_context_summarization_config (LLMAutoContextSummarizationConfig) instead.",
DeprecationWarning,
stacklevel=2,
)
if isinstance(self.context_summarization_config, LLMContextSummarizationConfig):
self.auto_context_summarization_config = (
self.context_summarization_config.to_auto_config()
)
else:
# Accept LLMAutoContextSummarizationConfig passed to the deprecated field
self.auto_context_summarization_config = self.context_summarization_config # type: ignore[assignment]
self.context_summarization_config = None
@dataclass
class UserTurnStoppedMessage:
@@ -404,15 +444,10 @@ class LLMUserAggregator(LLMContextAggregator):
"on_user_turn_stop_timeout", self._on_user_turn_stop_timeout
)
# Optional user idle controller
self._user_idle_controller: Optional[UserIdleController] = None
if self._params.user_idle_timeout:
self._user_idle_controller = UserIdleController(
user_idle_timeout=self._params.user_idle_timeout
)
self._user_idle_controller.add_event_handler(
"on_user_turn_idle", self._on_user_turn_idle
)
self._user_idle_controller = UserIdleController(
user_idle_timeout=self._params.user_idle_timeout
)
self._user_idle_controller.add_event_handler("on_user_turn_idle", self._on_user_turn_idle)
# VAD controller
self._vad_controller: Optional[VADController] = None
@@ -466,6 +501,10 @@ class LLMUserAggregator(LLMContextAggregator):
await self.push_frame(frame, direction)
elif isinstance(frame, TranscriptionFrame):
await self._handle_transcription(frame)
elif isinstance(frame, (InterimTranscriptionFrame, TranslationFrame)):
# Interim transcriptions and translations are consumed here
# and not pushed downstream, same as final TranscriptionFrame.
pass
elif isinstance(frame, LLMRunFrame):
await self._handle_llm_run(frame)
elif isinstance(frame, LLMMessagesAppendFrame):
@@ -489,8 +528,7 @@ class LLMUserAggregator(LLMContextAggregator):
await self._user_turn_controller.process_frame(frame)
if self._user_idle_controller:
await self._user_idle_controller.process_frame(frame)
await self._user_idle_controller.process_frame(frame)
async def push_aggregation(self) -> str:
"""Push the current aggregation."""
@@ -507,8 +545,7 @@ class LLMUserAggregator(LLMContextAggregator):
async def _start(self, frame: StartFrame):
await self._user_turn_controller.setup(self.task_manager)
if self._user_idle_controller:
await self._user_idle_controller.setup(self.task_manager)
await self._user_idle_controller.setup(self.task_manager)
for s in self._params.user_mute_strategies:
await s.setup(self.task_manager)
@@ -541,20 +578,16 @@ class LLMUserAggregator(LLMContextAggregator):
async def _cleanup(self):
await self._user_turn_controller.cleanup()
if self._user_idle_controller:
await self._user_idle_controller.cleanup()
await self._user_idle_controller.cleanup()
for s in self._params.user_mute_strategies:
await s.cleanup()
async def _maybe_mute_frame(self, frame: Frame):
# Control frames must flow unconditionally — never feed them to mute
# strategies. Without this guard, strategies like
# MuteUntilFirstBotCompleteUserMuteStrategy fire on_user_mute_started
# and broadcast UserMuteStartedFrame before StartFrame is pushed
# downstream, causing downstream processors to receive frames before
# StartFrame and log errors.
# Lifecycle frames should never be muted and should not trigger mute
# state changes. Evaluating mute strategies on StartFrame would
# broadcast UserMuteStartedFrame before StartFrame reaches downstream
# processors.
if isinstance(frame, (StartFrame, EndFrame, CancelFrame)):
return False
@@ -698,6 +731,8 @@ class LLMUserAggregator(LLMContextAggregator):
if params.enable_user_speaking_frames:
await self.broadcast_frame(UserStartedSpeakingFrame)
await self._user_idle_controller.process_frame(UserStartedSpeakingFrame())
if params.enable_interruptions and self._allow_interruptions:
await self.push_interruption_task_frame_and_wait()
@@ -714,6 +749,8 @@ class LLMUserAggregator(LLMContextAggregator):
if params.enable_user_speaking_frames:
await self.broadcast_frame(UserStoppedSpeakingFrame)
await self._user_idle_controller.process_frame(UserStoppedSpeakingFrame())
await self._maybe_emit_user_turn_stopped(strategy)
async def _on_user_turn_stop_timeout(self, controller):
@@ -827,16 +864,18 @@ class LLMAssistantAggregator(LLMContextAggregator):
self._thought_aggregation: List[TextPartForConcatenation] = []
self._thought_start_time: str = ""
# Context summarization
self._summarizer: Optional[LLMContextSummarizer] = None
if self._params.enable_context_summarization:
self._summarizer = LLMContextSummarizer(
context=self._context,
config=self._params.context_summarization_config,
)
self._summarizer.add_event_handler(
"on_request_summarization", self._on_request_summarization
)
# Context summarization — always create the summarizer so that manually
# pushed LLMSummarizeContextFrame frames are always handled.
# Auto-triggering based on thresholds is only enabled when
# enable_auto_context_summarization is True.
self._summarizer: Optional[LLMContextSummarizer] = LLMContextSummarizer(
context=self._context,
config=self._params.auto_context_summarization_config,
auto_trigger=self._params.enable_auto_context_summarization,
)
self._summarizer.add_event_handler(
"on_request_summarization", self._on_request_summarization
)
self._register_event_handler("on_assistant_turn_started")
self._register_event_handler("on_assistant_turn_stopped")
@@ -882,6 +921,8 @@ class LLMAssistantAggregator(LLMContextAggregator):
elif isinstance(frame, (EndFrame, CancelFrame)):
await self._handle_end_or_cancel(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, LLMAssistantPushAggregationFrame):
await self.push_aggregation()
elif isinstance(frame, LLMFullResponseStartFrame):
await self._handle_llm_start(frame)
elif isinstance(frame, LLMFullResponseEndFrame):
@@ -1264,8 +1305,8 @@ class LLMContextAggregatorPair:
self,
context: LLMContext,
*,
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
user_params: Optional[LLMUserAggregatorParams] = None,
assistant_params: Optional[LLMAssistantAggregatorParams] = None,
):
"""Initialize the LLM context aggregator pair.
@@ -1274,6 +1315,8 @@ class LLMContextAggregatorPair:
user_params: Parameters for the user context aggregator.
assistant_params: Parameters for the assistant context aggregator.
"""
user_params = user_params or LLMUserAggregatorParams()
assistant_params = assistant_params or LLMAssistantAggregatorParams()
self._user = LLMUserAggregator(context, params=user_params)
self._assistant = LLMAssistantAggregator(context, params=assistant_params)

129
src/pipecat/processors/frame_processor.py
View File

@@ -52,8 +52,6 @@ from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMet
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject
INTERRUPTION_COMPLETION_TIMEOUT = 2.0
class FrameDirection(Enum):
"""Direction of frame flow in the processing pipeline.
@@ -419,27 +417,65 @@ class FrameProcessor(BaseObject):
"""
self._metrics.set_core_metrics_data(data)
async def start_ttfb_metrics(self):
"""Start time-to-first-byte metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb)
async def start_ttfb_metrics(self, *, start_time: Optional[float] = None):
"""Start time-to-first-byte metrics collection.
async def stop_ttfb_metrics(self):
"""Stop time-to-first-byte metrics collection and push results."""
Args:
start_time: Optional timestamp to use as the start time. If None,
uses the current time.
"""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_ttfb_metrics()
await self._metrics.start_ttfb_metrics(
start_time=start_time, report_only_initial_ttfb=self._report_only_initial_ttfb
)
async def stop_ttfb_metrics(self, *, end_time: Optional[float] = None):
"""Stop time-to-first-byte metrics collection and push results.
Args:
end_time: Optional timestamp to use as the end time. If None, uses
the current time.
"""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_ttfb_metrics(end_time=end_time)
if frame:
await self.push_frame(frame)
async def start_processing_metrics(self):
"""Start processing metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_processing_metrics()
_processing_metrics_warned = False
async def stop_processing_metrics(self):
"""Stop processing metrics collection and push results."""
async def start_processing_metrics(self, *, start_time: Optional[float] = None):
"""Start processing metrics collection.
.. deprecated:: 0.0.104
Processing metrics are deprecated and will be removed in a future version.
Use TTFB metrics instead.
Args:
start_time: Optional timestamp to use as the start time. If None,
uses the current time.
"""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_processing_metrics()
if not FrameProcessor._processing_metrics_warned:
FrameProcessor._processing_metrics_warned = True
logger.warning(
"Processing metrics are deprecated and will be removed in a future version. "
"Use TTFB metrics instead."
)
await self._metrics.start_processing_metrics(start_time=start_time)
async def stop_processing_metrics(self, *, end_time: Optional[float] = None):
"""Stop processing metrics collection and push results.
.. deprecated:: 0.0.104
Processing metrics are deprecated and will be removed in a future version.
Use TTFB metrics instead.
Args:
end_time: Optional timestamp to use as the end time. If None, uses
the current time.
"""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_processing_metrics(end_time=end_time)
if frame:
await self.push_frame(frame)
@@ -465,10 +501,23 @@ class FrameProcessor(BaseObject):
if frame:
await self.push_frame(frame)
async def start_text_aggregation_metrics(self):
"""Start text aggregation time metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_text_aggregation_metrics()
async def stop_text_aggregation_metrics(self):
"""Stop text aggregation time metrics collection and push results."""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_text_aggregation_metrics()
if frame:
await self.push_frame(frame)
async def stop_all_metrics(self):
"""Stop all active metrics collection."""
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
await self.stop_text_aggregation_metrics()
def create_task(self, coroutine: Coroutine, name: Optional[str] = None) -> asyncio.Task:
"""Create a new task managed by this processor.
@@ -741,7 +790,7 @@ class FrameProcessor(BaseObject):
await self._call_event_handler("on_after_push_frame", frame)
async def push_interruption_task_frame_and_wait(self):
async def push_interruption_task_frame_and_wait(self, *, timeout: float = 5.0):
"""Push an interruption task frame upstream and wait for the interruption.
This function sends an `InterruptionTaskFrame` upstream to the
@@ -750,9 +799,11 @@ class FrameProcessor(BaseObject):
attached to both frames so the caller can wait until the interruption
has fully traversed the pipeline. The event is set when the
`InterruptionFrame` reaches the pipeline sink. If the frame does
not complete within `INTERRUPTION_COMPLETION_TIMEOUT` seconds, a
warning is logged periodically until it completes.
not complete within the given timeout, a warning is logged and the
event is forcibly set so the caller is unblocked.
Args:
timeout: Maximum seconds to wait for the interruption to complete.
"""
self._wait_for_interruption = True
@@ -760,19 +811,20 @@ class FrameProcessor(BaseObject):
await self.push_frame(InterruptionTaskFrame(event=event), FrameDirection.UPSTREAM)
# Wait for the `InterruptionFrame` to complete and log a warning
# periodically if it takes too long.
# Wait for the `InterruptionFrame` to complete and log a warning if it
# takes too long. If it does take too long make sure we unblock it,
# otherwise we will hang here forever.
while not event.is_set():
try:
await asyncio.wait_for(event.wait(), timeout=INTERRUPTION_COMPLETION_TIMEOUT)
await asyncio.wait_for(event.wait(), timeout=timeout)
except asyncio.TimeoutError:
logger.warning(
f"{self}: InterruptionFrame has not completed after"
f" {INTERRUPTION_COMPLETION_TIMEOUT}s. Make sure"
" InterruptionFrame.complete() is being called (e.g. if the"
" frame is being blocked or consumed before reaching the"
" pipeline sink)."
f" {timeout}s. Make sure InterruptionFrame.complete()"
" is being called (e.g. if the frame is being blocked"
" or consumed before reaching the pipeline sink)."
)
event.set()
self._wait_for_interruption = False
@@ -787,8 +839,12 @@ class FrameProcessor(BaseObject):
frame_cls: The class of the frame to be broadcasted.
**kwargs: Keyword arguments to be passed to the frame's constructor.
"""
await self.push_frame(frame_cls(**kwargs))
await self.push_frame(frame_cls(**kwargs), FrameDirection.UPSTREAM)
downstream_frame = frame_cls(**kwargs)
upstream_frame = frame_cls(**kwargs)
downstream_frame.broadcast_sibling_id = upstream_frame.id
upstream_frame.broadcast_sibling_id = downstream_frame.id
await self.push_frame(downstream_frame)
await self.push_frame(upstream_frame, FrameDirection.UPSTREAM)
async def broadcast_frame_instance(self, frame: Frame):
"""Broadcasts a frame instance upstream and downstream.
@@ -812,15 +868,18 @@ class FrameProcessor(BaseObject):
if not f.init and f.name not in ("id", "name")
}
new_frame = frame_cls(**init_fields)
downstream_frame = frame_cls(**init_fields)
for k, v in extra_fields.items():
setattr(new_frame, k, v)
await self.push_frame(new_frame)
setattr(downstream_frame, k, v)
new_frame = frame_cls(**init_fields)
upstream_frame = frame_cls(**init_fields)
for k, v in extra_fields.items():
setattr(new_frame, k, v)
await self.push_frame(new_frame, FrameDirection.UPSTREAM)
setattr(upstream_frame, k, v)
downstream_frame.broadcast_sibling_id = upstream_frame.id
upstream_frame.broadcast_sibling_id = downstream_frame.id
await self.push_frame(downstream_frame)
await self.push_frame(upstream_frame, FrameDirection.UPSTREAM)
async def __start(self, frame: StartFrame):
"""Handle the start frame to initialize processor state.
@@ -906,7 +965,7 @@ class FrameProcessor(BaseObject):
try:
timestamp = self._clock.get_time() if self._clock else 0
if direction == FrameDirection.DOWNSTREAM and self._next:
logger.trace(f"Pushing {frame} from {self} to {self._next}")
logger.trace(f"Pushing {frame} downstream from {self} to {self._next}")
if self._observer:
data = FramePushed(

44
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -25,6 +25,7 @@ from typing import (
Literal,
Mapping,
Optional,
Set,
Tuple,
Union,
)
@@ -1026,6 +1027,11 @@ class RTVIObserverParams:
metrics_enabled: Indicates if metrics messages should be sent.
system_logs_enabled: Indicates if system logs should be sent.
errors_enabled: [Deprecated] Indicates if errors messages should be sent.
ignored_sources: List of frame processors whose frames should be silently ignored
by this observer. Useful for suppressing RTVI messages from secondary pipeline
branches (e.g. a silent evaluation LLM) that should not be visible to clients.
Sources can also be added and removed dynamically via ``add_ignored_source()``
and ``remove_ignored_source()``.
skip_aggregator_types: List of aggregation types to skip sending as tts/output messages.
Note: if using this to avoid sending secure information, be sure to also disable
bot_llm_enabled to avoid leaking through LLM messages.
@@ -1065,6 +1071,7 @@ class RTVIObserverParams:
metrics_enabled: bool = True
system_logs_enabled: bool = False
errors_enabled: Optional[bool] = None
ignored_sources: List[FrameProcessor] = field(default_factory=list)
skip_aggregator_types: Optional[List[AggregationType | str]] = None
bot_output_transforms: Optional[
List[
@@ -1110,6 +1117,7 @@ class RTVIObserver(BaseObserver):
self._rtvi = rtvi
self._params = params or RTVIObserverParams()
self._ignored_sources: Set[FrameProcessor] = set(self._params.ignored_sources)
self._frames_seen = set()
self._bot_transcription = ""
@@ -1170,6 +1178,31 @@ class RTVIObserver(BaseObserver):
if not (agg_type == aggregation_type and func == transform_function)
]
def add_ignored_source(self, source: FrameProcessor):
"""Ignore all frames pushed by the given processor.
Any frame whose source matches ``source`` will be silently skipped,
preventing RTVI messages from being emitted for activity in that
processor. Useful for suppressing events from secondary pipeline
branches (e.g. a silent evaluation LLM) that should not be visible
to clients.
Args:
source: The frame processor to ignore.
"""
self._ignored_sources.add(source)
def remove_ignored_source(self, source: FrameProcessor):
"""Stop ignoring frames pushed by the given processor.
Reverses a previous call to ``add_ignored_source()``. If ``source``
was not previously ignored this is a no-op.
Args:
source: The frame processor to stop ignoring.
"""
self._ignored_sources.discard(source)
def _get_function_call_report_level(self, function_name: str) -> RTVIFunctionCallReportLevel:
"""Get the report level for a specific function call.
@@ -1220,10 +1253,13 @@ class RTVIObserver(BaseObserver):
frame = data.frame
direction = data.direction
# Only process downstream frames. Some frames are broadcast in both
# directions (e.g. UserStartedSpeakingFrame, FunctionCallResultFrame),
# and we only want to send one RTVI message per event.
if direction != FrameDirection.DOWNSTREAM:
# Frames from explicitly ignored sources are always skipped.
if self._ignored_sources and src in self._ignored_sources:
return
# For broadcast frames (pushed in both directions), only process
# the downstream copy to avoid sending duplicate RTVI messages.
if frame.broadcast_sibling_id is not None and direction != FrameDirection.DOWNSTREAM:
return
# If we have already seen this frame, let's skip it.

74
src/pipecat/processors/metrics/frame_processor_metrics.py
View File

@@ -17,6 +17,7 @@ from pipecat.metrics.metrics import (
LLMUsageMetricsData,
MetricsData,
ProcessingMetricsData,
TextAggregationMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
)
@@ -43,6 +44,7 @@ class FrameProcessorMetrics(BaseObject):
self._task_manager = None
self._start_ttfb_time = 0
self._start_processing_time = 0
self._start_text_aggregation_time = 0
self._last_ttfb_time = 0
self._should_report_ttfb = True
@@ -107,49 +109,78 @@ class FrameProcessorMetrics(BaseObject):
"""
self._core_metrics_data = MetricsData(processor=name)
async def start_ttfb_metrics(self, report_only_initial_ttfb):
async def start_ttfb_metrics(
self, *, start_time: Optional[float] = None, report_only_initial_ttfb: bool
):
"""Start measuring time-to-first-byte (TTFB).
Args:
start_time: Optional timestamp to use as the start time. If None,
uses the current time.
report_only_initial_ttfb: Whether to report only the first TTFB measurement.
"""
if self._should_report_ttfb:
self._start_ttfb_time = time.time()
self._start_ttfb_time = start_time or time.time()
self._last_ttfb_time = 0
self._should_report_ttfb = not report_only_initial_ttfb
async def stop_ttfb_metrics(self):
async def stop_ttfb_metrics(self, *, end_time: Optional[float] = None):
"""Stop TTFB measurement and generate metrics frame.
Args:
end_time: Optional timestamp to use as the end time. If None, uses
the current time.
Returns:
MetricsFrame containing TTFB data, or None if not measuring.
"""
if self._start_ttfb_time == 0:
return None
self._last_ttfb_time = time.time() - self._start_ttfb_time
logger.debug(f"{self._processor_name()} TTFB: {self._last_ttfb_time}")
end_time = end_time or time.time()
self._last_ttfb_time = end_time - self._start_ttfb_time
logger.debug(f"{self._processor_name()} TTFB: {self._last_ttfb_time:.3f}s")
ttfb = TTFBMetricsData(
processor=self._processor_name(), value=self._last_ttfb_time, model=self._model_name()
)
self._start_ttfb_time = 0
return MetricsFrame(data=[ttfb])
async def start_processing_metrics(self):
"""Start measuring processing time."""
self._start_processing_time = time.time()
async def start_processing_metrics(self, *, start_time: Optional[float] = None):
"""Start measuring processing time.
async def stop_processing_metrics(self):
.. deprecated:: 0.0.104
Processing metrics are deprecated and will be removed in a future version.
Use TTFB metrics instead.
Args:
start_time: Optional timestamp to use as the start time. If None,
uses the current time.
"""
self._start_processing_time = start_time or time.time()
async def stop_processing_metrics(self, *, end_time: Optional[float] = None):
"""Stop processing time measurement and generate metrics frame.
.. deprecated:: 0.0.104
Processing metrics are deprecated and will be removed in a future version.
Use TTFB metrics instead.
Args:
end_time: Optional timestamp to use as the end time. If None, uses
the current time.
Returns:
MetricsFrame containing processing duration data, or None if not measuring.
"""
if self._start_processing_time == 0:
return None
value = time.time() - self._start_processing_time
logger.debug(f"{self._processor_name()} processing time: {value}")
end_time = end_time or time.time()
value = end_time - self._start_processing_time
logger.debug(f"{self._processor_name()} processing time: {value:.3f}s")
processing = ProcessingMetricsData(
processor=self._processor_name(), value=value, model=self._model_name()
)
@@ -190,3 +221,24 @@ class FrameProcessorMetrics(BaseObject):
)
logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
return MetricsFrame(data=[characters])
async def start_text_aggregation_metrics(self):
"""Start measuring text aggregation time (first token to first sentence)."""
self._start_text_aggregation_time = time.time()
async def stop_text_aggregation_metrics(self):
"""Stop text aggregation measurement and generate metrics frame.
Returns:
MetricsFrame containing text aggregation time, or None if not measuring.
"""
if self._start_text_aggregation_time == 0:
return None
value = time.time() - self._start_text_aggregation_time
logger.debug(f"{self._processor_name()} text aggregation time: {value}")
aggregation = TextAggregationMetricsData(
processor=self._processor_name(), value=value, model=self._model_name()
)
self._start_text_aggregation_time = 0
return MetricsFrame(data=[aggregation])

31
src/pipecat/processors/metrics/sentry.py
View File

@@ -7,6 +7,7 @@
"""Sentry integration for frame processor metrics."""
import asyncio
from typing import Optional
from loguru import logger
@@ -70,13 +71,18 @@ class SentryMetrics(FrameProcessorMetrics):
logger.trace(f"{self} Flushing Sentry metrics")
sentry_sdk.flush(timeout=5.0)
async def start_ttfb_metrics(self, report_only_initial_ttfb):
async def start_ttfb_metrics(
self, *, start_time: Optional[float] = None, report_only_initial_ttfb: bool
):
"""Start tracking time-to-first-byte metrics.
Args:
start_time: Optional start timestamp override.
report_only_initial_ttfb: Whether to report only the initial TTFB measurement.
"""
await super().start_ttfb_metrics(report_only_initial_ttfb)
await super().start_ttfb_metrics(
start_time=start_time, report_only_initial_ttfb=report_only_initial_ttfb
)
if self._should_report_ttfb and self._sentry_available:
self._ttfb_metrics_tx = sentry_sdk.start_transaction(
@@ -87,23 +93,25 @@ class SentryMetrics(FrameProcessorMetrics):
f"{self} Sentry transaction started (ID: {self._ttfb_metrics_tx.span_id} Name: {self._ttfb_metrics_tx.name})"
)
async def stop_ttfb_metrics(self):
async def stop_ttfb_metrics(self, *, end_time: Optional[float] = None):
"""Stop tracking time-to-first-byte metrics.
Queues the TTFB transaction for completion and transmission to Sentry.
Args:
end_time: Optional end timestamp override.
"""
await super().stop_ttfb_metrics()
await super().stop_ttfb_metrics(end_time=end_time)
if self._sentry_available and self._ttfb_metrics_tx:
await self._sentry_queue.put(self._ttfb_metrics_tx)
self._ttfb_metrics_tx = None
async def start_processing_metrics(self):
async def start_processing_metrics(self, *, start_time: Optional[float] = None):
"""Start tracking frame processing metrics.
Creates a new Sentry transaction to track processing performance.
Args:
start_time: Optional start timestamp override.
"""
await super().start_processing_metrics()
await super().start_processing_metrics(start_time=start_time)
if self._sentry_available:
self._processing_metrics_tx = sentry_sdk.start_transaction(
@@ -114,12 +122,13 @@ class SentryMetrics(FrameProcessorMetrics):
f"{self} Sentry transaction started (ID: {self._processing_metrics_tx.span_id} Name: {self._processing_metrics_tx.name})"
)
async def stop_processing_metrics(self):
async def stop_processing_metrics(self, *, end_time: Optional[float] = None):
"""Stop tracking frame processing metrics.
Queues the processing transaction for completion and transmission to Sentry.
Args:
end_time: Optional end timestamp override.
"""
await super().stop_processing_metrics()
await super().stop_processing_metrics(end_time=end_time)
if self._sentry_available and self._processing_metrics_tx:
await self._sentry_queue.put(self._processing_metrics_tx)

1
src/pipecat/serializers/genesys.py
View File

@@ -642,7 +642,6 @@ class GenesysAudioHookSerializer(FrameSerializer):
"""
# Binary data = audio
if isinstance(data, bytes):
logger.debug(f"[AUDIO IN] Received {len(data)} bytes from Genesys")
return await self._deserialize_audio(data)
# Text data = JSON control message

101
src/pipecat/services/ai_service.py
View File

@@ -10,7 +10,7 @@ Provides the foundation for all AI services in the Pipecat framework, including
model management, settings handling, and frame processing lifecycle methods.
"""
from typing import Any, AsyncGenerator, Dict, Mapping
from typing import Any, AsyncGenerator, Dict
from loguru import logger
@@ -23,6 +23,7 @@ from pipecat.frames.frames import (
)
from pipecat.metrics.metrics import MetricsData
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.settings import ServiceSettings
class AIService(FrameProcessor):
@@ -34,36 +35,38 @@ class AIService(FrameProcessor):
this base infrastructure.
"""
def __init__(self, **kwargs):
def __init__(self, settings: ServiceSettings | None = None, **kwargs):
"""Initialize the AI service.
Args:
settings: The runtime-updatable settings for the AI service.
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
self._model_name: str = ""
self._settings: Dict[str, Any] = {}
self._settings: ServiceSettings = (
settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ServiceSettings()
)
self._sync_model_name_to_metrics()
self._session_properties: Dict[str, Any] = {}
self._tracing_enabled: bool = False
self._tracing_context = None
@property
def model_name(self) -> str:
"""Get the current model name.
def _sync_model_name_to_metrics(self):
"""Sync the current AI model name (in `self._settings.model`) for usage in metrics.
Returns:
The name of the AI model being used.
"""
return self._model_name
def set_model_name(self, model: str):
"""Set the AI model name and update metrics.
We don't store model name here because there's already a single source
of truth for it in `self._settings.model`. This method is just for
syncing the model name to the metrics data.
Args:
model: The name of the AI model to use.
"""
self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
self.set_core_metrics_data(
MetricsData(processor=self.name, model=self._settings.model or "")
)
async def start(self, frame: StartFrame):
"""Start the AI service.
@@ -74,6 +77,7 @@ class AIService(FrameProcessor):
Args:
frame: The start frame containing initialization parameters.
"""
self._settings.validate_complete()
self._tracing_enabled = frame.enable_tracing
self._tracing_context = frame.tracing_context
@@ -99,44 +103,45 @@ class AIService(FrameProcessor):
"""
pass
async def _update_settings(self, settings: Mapping[str, Any]):
from pipecat.services.openai.realtime.events import SessionProperties
async def _update_settings(self, delta: ServiceSettings) -> Dict[str, Any]:
"""Apply a settings delta and return the changed fields.
for key, value in settings.items():
logger.debug("Update request for:", key, value)
The delta is applied to ``_settings`` and a dict mapping each changed
field name to its **pre-update** value is returned. The ``model``
field is handled specially: when it changes, ``set_model_name`` is
called.
if key in self._settings:
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._settings[key] = value
elif key in SessionProperties.model_fields:
logger.debug("Attempting to update", key, value)
Concrete services should override this method (calling ``super()``)
to react to specific changed fields (e.g. reconnect on voice change).
try:
from pipecat.services.openai.realtime.events import TurnDetection
Args:
delta: A delta-mode settings object.
if isinstance(self._session_properties, SessionProperties):
current_properties = self._session_properties
else:
current_properties = SessionProperties(**self._session_properties)
Returns:
Dict mapping changed field names to their previous values.
"""
changed = self._settings.apply_update(delta)
if key == "turn_detection" and isinstance(value, dict):
turn_detection = TurnDetection(**value)
setattr(current_properties, key, turn_detection)
else:
setattr(current_properties, key, value)
if "model" in changed:
self._sync_model_name_to_metrics()
validated_properties = SessionProperties.model_validate(
current_properties.model_dump()
)
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._session_properties = validated_properties.model_dump()
except Exception as e:
logger.warning(f"Unexpected error updating session property {key}: {e}")
elif key == "model":
logger.info(f"Updating LLM setting {key} to: [{value}]")
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for {self.name} service: {key}")
if changed:
logger.info(f"{self.name}: updated settings fields: {set(changed)}")
return changed
def _warn_unhandled_updated_settings(self, unhandled):
"""Log a warning for settings changes that won't take effect at runtime.
Convenience helper for ``_update_settings`` overrides. Accepts any
iterable of field names (a ``dict``, ``set``, ``dict_keys``, etc.).
Args:
unhandled: Field names that changed but are not applied.
"""
if unhandled:
fields = ", ".join(sorted(unhandled))
logger.warning(f"{self.name}: runtime update of [{fields}] is not currently supported")
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and handle service lifecycle.

168
src/pipecat/services/anthropic/llm.py
View File

@@ -16,8 +16,8 @@ import copy
import io
import json
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, Union
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Literal, Optional, Union
import httpx
from loguru import logger
@@ -42,7 +42,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
UserImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -59,6 +58,8 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
from pipecat.services.settings import LLMSettings, _NotGiven, is_given
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -69,6 +70,50 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
class AnthropicThinkingConfig(BaseModel):
"""Configuration for extended thinking.
Parameters:
type: Type of thinking mode (currently only "enabled" or "disabled").
budget_tokens: Maximum number of tokens for thinking.
With today's models, the minimum is 1024.
Only allowed if type is "enabled".
"""
# Why `| str` here? To not break compatibility in case Anthropic adds
# more types in the future.
type: Literal["enabled", "disabled"] | str
# Why not enforce minimnum of 1024 here? To not break compatibility in
# case Anthropic changes this requirement in the future.
budget_tokens: int
@dataclass
class AnthropicLLMSettings(LLMSettings):
"""Settings for Anthropic LLM services.
Parameters:
enable_prompt_caching: Whether to enable prompt caching.
thinking: Extended thinking configuration.
"""
enable_prompt_caching: bool | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
thinking: AnthropicThinkingConfig | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
@classmethod
def from_mapping(cls, settings):
"""Convert a plain dict to settings, coercing thinking dicts.
For backward compatibility, a ``thinking`` value that is a plain dict
is converted to a :class:`AnthropicThinkingConfig`.
"""
instance = super().from_mapping(settings)
if is_given(instance.thinking) and isinstance(instance.thinking, dict):
instance.thinking = AnthropicThinkingConfig(**instance.thinking)
return instance
@dataclass
class AnthropicContextAggregatorPair:
"""Pair of context aggregators for Anthropic conversations.
@@ -115,26 +160,13 @@ class AnthropicLLMService(LLMService):
Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex.
"""
_settings: AnthropicLLMSettings
# Overriding the default adapter to use the Anthropic one.
adapter_class = AnthropicLLMAdapter
class ThinkingConfig(BaseModel):
"""Configuration for extended thinking.
Parameters:
type: Type of thinking mode (currently only "enabled" or "disabled").
budget_tokens: Maximum number of tokens for thinking.
With today's models, the minimum is 1024.
Only allowed if type is "enabled".
"""
# Why `| str` here? To not break compatibility in case Anthropic adds
# more types in the future.
type: Literal["enabled", "disabled"] | str
# Why not enforce minimnum of 1024 here? To not break compatibility in
# case Anthropic changes this requirement in the future.
budget_tokens: int
# Backward compatibility: ThinkingConfig used to be defined inline here.
ThinkingConfig = AnthropicThinkingConfig
class InputParams(BaseModel):
"""Input parameters for Anthropic model inference.
@@ -163,9 +195,7 @@ class AnthropicLLMService(LLMService):
temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
top_k: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0)
top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
thinking: Optional["AnthropicLLMService.ThinkingConfig"] = Field(
default_factory=lambda: NOT_GIVEN
)
thinking: Optional[AnthropicThinkingConfig] = Field(default_factory=lambda: NOT_GIVEN)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def model_post_init(self, __context):
@@ -184,7 +214,7 @@ class AnthropicLLMService(LLMService):
self,
*,
api_key: str,
model: str = "claude-sonnet-4-5-20250929",
model: str = "claude-sonnet-4-6",
params: Optional[InputParams] = None,
client=None,
retry_timeout_secs: Optional[float] = 5.0,
@@ -195,38 +225,46 @@ class AnthropicLLMService(LLMService):
Args:
api_key: Anthropic API key for authentication.
model: Model name to use. Defaults to "claude-sonnet-4-5-20250929".
model: Model name to use. Defaults to "claude-sonnet-4-6".
params: Optional model parameters for inference.
client: Optional custom Anthropic client instance.
retry_timeout_secs: Request timeout in seconds for retry logic.
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AnthropicLLMService.InputParams()
super().__init__(
settings=AnthropicLLMSettings(
model=model,
max_tokens=params.max_tokens,
enable_prompt_caching=(
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._client = client or AsyncAnthropic(
api_key=api_key
) # if the client is provided, use it and remove it, otherwise create a new one
self.set_model_name(model)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = {
"max_tokens": params.max_tokens,
"enable_prompt_caching": (
params.enable_prompt_caching
if params.enable_prompt_caching is not None
else (
params.enable_prompt_caching_beta
if params.enable_prompt_caching_beta is not None
else False
)
),
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"thinking": params.thinking,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
def can_generate_metrics(self) -> bool:
"""Check if this service can generate usage metrics.
@@ -280,7 +318,7 @@ class AnthropicLLMService(LLMService):
if isinstance(context, LLMContext):
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
invocation_params = adapter.get_llm_invocation_params(
context, enable_prompt_caching=self._settings["enable_prompt_caching"]
context, enable_prompt_caching=self._settings.enable_prompt_caching
)
messages = invocation_params["messages"]
system = invocation_params["system"]
@@ -293,21 +331,21 @@ class AnthropicLLMService(LLMService):
# Build params using the same method as streaming completions
params = {
"model": self.model_name,
"max_tokens": max_tokens if max_tokens is not None else self._settings["max_tokens"],
"model": self._settings.model,
"max_tokens": max_tokens if max_tokens is not None else self._settings.max_tokens,
"stream": False,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"temperature": self._settings.temperature,
"top_k": self._settings.top_k,
"top_p": self._settings.top_p,
"messages": messages,
"system": system,
"tools": tools,
"betas": ["interleaved-thinking-2025-05-14"],
}
if self._settings["thinking"]:
params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
if self._settings.thinking:
params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
params.update(self._settings["extra"])
params.update(self._settings.extra)
# LLM completion
response = await self._client.beta.messages.create(**params)
@@ -358,14 +396,14 @@ class AnthropicLLMService(LLMService):
if isinstance(context, LLMContext):
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
params = adapter.get_llm_invocation_params(
context, enable_prompt_caching=self._settings["enable_prompt_caching"]
context, enable_prompt_caching=self._settings.enable_prompt_caching
)
return params
# Anthropic-specific context
messages = (
context.get_messages_with_cache_control_markers()
if self._settings["enable_prompt_caching"]
if self._settings.enable_prompt_caching
else context.messages
)
return AnthropicLLMInvocationParams(
@@ -407,22 +445,22 @@ class AnthropicLLMService(LLMService):
await self.start_ttfb_metrics()
params = {
"model": self.model_name,
"max_tokens": self._settings["max_tokens"],
"model": self._settings.model,
"max_tokens": self._settings.max_tokens,
"stream": True,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"temperature": self._settings.temperature,
"top_k": self._settings.top_k,
"top_p": self._settings.top_p,
}
# Add thinking parameter if set
if self._settings["thinking"]:
params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
if self._settings.thinking:
params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
# Messages, system, tools
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
# "Interleaved thinking" needed to allow thinking between sequences
# of function calls, when extended thinking is enabled.
@@ -576,11 +614,9 @@ class AnthropicLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = AnthropicLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMEnablePromptCachingFrame):
logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
self._settings["enable_prompt_caching"] = frame.enable
self._settings.enable_prompt_caching = frame.enable
else:
await self.push_frame(frame, direction)

71
src/pipecat/services/assemblyai/stt.py
View File

@@ -12,6 +12,7 @@ WebSocket API for streaming audio transcription.
import asyncio
import json
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional
from urllib.parse import urlencode
@@ -29,6 +30,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import ASSEMBLYAI_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -52,6 +54,21 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AssemblyAISTTSettings(STTSettings):
"""Settings for the AssemblyAI STT service.
See :class:`AssemblyAIConnectionParams` for detailed parameter descriptions.
Parameters:
connection_params: Connection configuration parameters.
"""
connection_params: AssemblyAIConnectionParams | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class AssemblyAISTTService(WebsocketSTTService):
"""AssemblyAI real-time speech-to-text service.
@@ -60,6 +77,8 @@ class AssemblyAISTTService(WebsocketSTTService):
for audio processing and connection management.
"""
_settings: AssemblyAISTTSettings
def __init__(
self,
*,
@@ -92,13 +111,18 @@ class AssemblyAISTTService(WebsocketSTTService):
connection_params = self._configure_manual_turn_mode(connection_params)
super().__init__(
sample_rate=connection_params.sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs
sample_rate=connection_params.sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AssemblyAISTTSettings(
model=None,
language=language,
connection_params=connection_params,
),
**kwargs,
)
self._api_key = api_key
self._language = language
self._api_endpoint_base_url = api_endpoint_base_url
self._connection_params = connection_params
self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._termination_event = asyncio.Event()
@@ -165,6 +189,37 @@ class AssemblyAISTTService(WebsocketSTTService):
"""
return True
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
Args:
delta: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# # Re-apply manual turn mode config if vad_force_turn_endpoint is active
# # and connection_params were updated.
# if self._vad_force_turn_endpoint and "connection_params" in changed:
# self._settings.connection_params = self._configure_manual_turn_mode(
# self._settings.connection_params
# )
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
@@ -239,7 +294,7 @@ class AssemblyAISTTService(WebsocketSTTService):
def _build_ws_url(self) -> str:
"""Build WebSocket URL with query parameters using urllib.parse.urlencode."""
params = {}
for k, v in self._connection_params.model_dump().items():
for k, v in self._settings.connection_params.model_dump().items():
if v is not None:
if k == "keyterms_prompt":
params[k] = json.dumps(v)
@@ -415,18 +470,18 @@ class AssemblyAISTTService(WebsocketSTTService):
if not message.transcript:
return
if message.end_of_turn and (
not self._connection_params.formatted_finals or message.turn_is_formatted
not self._settings.connection_params.formatted_finals or message.turn_is_formatted
):
await self.push_frame(
TranscriptionFrame(
message.transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
message,
)
)
await self._trace_transcription(message.transcript, True, self._language)
await self._trace_transcription(message.transcript, True, self._settings.language)
await self.stop_processing_metrics()
else:
await self.push_frame(
@@ -434,7 +489,7 @@ class AssemblyAISTTService(WebsocketSTTService):
message.transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
message,
)
)

223
src/pipecat/services/asyncai/tts.py
View File

@@ -9,8 +9,8 @@
import asyncio
import base64
import json
import uuid
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Mapping, Optional
import aiohttp
from loguru import logger
@@ -21,14 +21,14 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -73,12 +73,40 @@ def language_to_async_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class AsyncAITTSSettings(TTSSettings):
"""Settings for Async AI TTS services.
Parameters:
output_container: Audio container format (e.g. "raw").
output_encoding: Audio encoding format (e.g. "pcm_s16le").
output_sample_rate: Audio sample rate in Hz.
"""
output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "AsyncAITTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``output_format``."""
flat = dict(settings)
nested = flat.pop("output_format", None)
if isinstance(nested, dict):
flat.setdefault("output_container", nested.get("container"))
flat.setdefault("output_encoding", nested.get("encoding"))
flat.setdefault("output_sample_rate", nested.get("sample_rate"))
return super().from_mapping(flat)
class AsyncAITTSService(AudioContextTTSService):
"""Async TTS service with WebSocket streaming.
Provides text-to-speech using Async's streaming WebSocket API.
"""
_settings: AsyncAITTSSettings
class InputParams(BaseModel):
"""Input parameters for Async TTS configuration.
@@ -100,7 +128,8 @@ class AsyncAITTSService(AudioContextTTSService):
encoding: str = "pcm_s16le",
container: str = "raw",
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize the Async TTS service.
@@ -116,39 +145,55 @@ class AsyncAITTSService(AudioContextTTSService):
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to the parent service.
"""
params = params or AsyncAITTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs,
)
params = params or AsyncAITTSService.InputParams()
self._api_key = api_key
self._api_version = version
self._url = url
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._receive_task = None
self._keepalive_task = None
self._context_id = None
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
self._warn_unhandled_updated_settings(changed)
return changed
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -180,7 +225,7 @@ class AsyncAITTSService(AudioContextTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -234,10 +279,14 @@ class AsyncAITTSService(AudioContextTTSService):
f"{self._url}?api_key={self._api_key}&version={self._api_version}"
)
init_msg = {
"model_id": self._model_name,
"voice": {"mode": "id", "id": self._voice_id},
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"model_id": self._settings.model,
"voice": {"mode": "id", "id": self._settings.voice},
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"language": self._settings.language,
}
await self._get_websocket().send(json.dumps(init_msg))
@@ -255,7 +304,7 @@ class AsyncAITTSService(AudioContextTTSService):
if self._websocket:
logger.debug("Disconnecting from Async")
# Close all contexts and the socket
if self._context_id:
if self.has_active_audio_context():
await self._websocket.send(json.dumps({"terminate": True}))
await self._websocket.close()
logger.debug("Disconnected from Async")
@@ -263,7 +312,7 @@ class AsyncAITTSService(AudioContextTTSService):
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
finally:
self._websocket = None
self._context_id = None
await self.remove_active_audio_context()
await self._call_event_handler("on_disconnected")
def _get_websocket(self):
@@ -271,26 +320,13 @@ class AsyncAITTSService(AudioContextTTSService):
return self._websocket
raise Exception("Websocket not connected")
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request in case we don't have one already in progress.
Returns:
A unique string identifier for the TTS context.
"""
# If a context ID does not exist, create a new one.
# If an ID exists, continue using the current ID.
# When interruptions happen, user speech results in
# an interruption, which resets the context ID.
if not self._context_id:
return str(uuid.uuid4())
return self._context_id
async def flush_audio(self):
"""Flush any pending audio."""
if not self._context_id or not self._websocket:
context_id = self.get_active_audio_context_id()
if not context_id or not self._websocket:
return
logger.trace(f"{self}: flushing audio")
msg = self._build_msg(text=" ", context_id=self._context_id, force=True)
msg = self._build_msg(text=" ", context_id=context_id, force=True)
await self._websocket.send(msg)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
@@ -318,11 +354,11 @@ class AsyncAITTSService(AudioContextTTSService):
# Check if this message belongs to the current context.
if not self.audio_context_available(received_ctx_id):
if self._context_id == received_ctx_id:
if self.get_active_audio_context_id() == received_ctx_id:
logger.debug(
f"Received a delayed message, recreating the context: {self._context_id}"
f"Received a delayed message, recreating the context: {received_ctx_id}"
)
await self.create_audio_context(self._context_id)
await self.create_audio_context(received_ctx_id)
else:
# This can happen if a message is received _after_ we have closed a context
# due to user interruption but _before_ the `isFinal` message for the context
@@ -343,10 +379,11 @@ class AsyncAITTSService(AudioContextTTSService):
await asyncio.sleep(KEEPALIVE_SLEEP)
try:
if self._websocket and self._websocket.state is State.OPEN:
if self._context_id:
context_id = self.get_active_audio_context_id()
if context_id:
keepalive_message = {
"transcript": " ",
"context_id": self._context_id,
"context_id": context_id,
}
logger.trace("Sending keepalive message")
else:
@@ -360,21 +397,29 @@ class AsyncAITTSService(AudioContextTTSService):
logger.warning(f"{self} keepalive error: {e}")
break
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by closing the current context."""
await super()._handle_interruption(frame, direction)
# Close the current context when interrupted without closing the websocket
if self._context_id and self._websocket:
async def _close_context(self, context_id: str):
# Async AI requires explicit context closure to free server-side resources,
# both on interruption and on normal completion.
if context_id and self._websocket:
try:
await self._websocket.send(
json.dumps(
{"context_id": self._context_id, "close_context": True, "transcript": ""}
)
json.dumps({"context_id": context_id, "close_context": True, "transcript": ""})
)
except Exception as e:
logger.error(f"Error closing context on interruption: {e}")
self._context_id = None
logger.error(f"{self}: Error closing context {context_id}: {e}")
async def on_audio_context_interrupted(self, context_id: str):
"""Close the Async AI context when the bot is interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Close the Async AI context after all audio has been played.
Async AI does not send a server-side signal when a context is
exhausted, so Pipecat must explicitly close it with
``close_context: True`` to free server-side resources.
"""
await self._close_context(context_id)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -394,16 +439,13 @@ class AsyncAITTSService(AudioContextTTSService):
await self._connect()
try:
if not self._context_id:
if not self.has_active_audio_context():
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
if not self.audio_context_available(context_id):
await self.create_audio_context(context_id)
self._context_id = context_id
if not self.audio_context_available(self._context_id):
await self.create_audio_context(self._context_id)
msg = self._build_msg(text=text, force=True, context_id=self._context_id)
msg = self._build_msg(text=text, force=True, context_id=context_id)
await self._get_websocket().send(msg)
await self.start_tts_usage_metrics(text)
@@ -424,6 +466,8 @@ class AsyncAIHttpTTSService(TTSService):
connection is not required or desired.
"""
_settings: AsyncAITTSSettings
class InputParams(BaseModel):
"""Input parameters for Async API.
@@ -463,25 +507,26 @@ class AsyncAIHttpTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AsyncAIHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AsyncAITTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
),
**kwargs,
)
self._api_key = api_key
self._base_url = url
self._api_version = version
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
}
self.set_voice(voice_id)
self.set_model_name(model)
self._session = aiohttp_session
@@ -511,7 +556,7 @@ class AsyncAIHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -527,14 +572,18 @@ class AsyncAIHttpTTSService(TTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
voice_config = {"mode": "id", "id": self._voice_id}
voice_config = {"mode": "id", "id": self._settings.voice}
await self.start_ttfb_metrics()
payload = {
"model_id": self._model_name,
"model_id": self._settings.model,
"transcript": text,
"voice": voice_config,
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"language": self._settings.language,
}
yield TTSStartedFrame(context_id=context_id)
headers = {

81
src/pipecat/services/aws/llm.py
View File

@@ -18,8 +18,8 @@ import io
import json
import os
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Optional
from loguru import logger
from PIL import Image
@@ -40,7 +40,6 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
UserImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -57,6 +56,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -71,6 +71,21 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AWSBedrockLLMSettings(LLMSettings):
"""Settings for AWS Bedrock LLM services.
Parameters:
latency: Performance mode - "standard" or "optimized".
additional_model_request_fields: Additional model-specific parameters.
"""
latency: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
additional_model_request_fields: Dict[str, Any] | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
@dataclass
class AWSBedrockContextAggregatorPair:
"""Container for AWS Bedrock context aggregators.
@@ -730,6 +745,8 @@ class AWSBedrockLLMService(LLMService):
vision capabilities.
"""
_settings: AWSBedrockLLMSettings
# Overriding the default adapter to use the Anthropic one.
adapter_class = AWSBedrockLLMAdapter
@@ -780,10 +797,28 @@ class AWSBedrockLLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AWSBedrockLLMService.InputParams()
super().__init__(
settings=AWSBedrockLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
latency=params.latency,
additional_model_request_fields=params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
),
**kwargs,
)
# Initialize the AWS Bedrock client
if not client_config:
client_config = Config(
@@ -803,18 +838,8 @@ class AWSBedrockLLMService(LLMService):
"config": client_config,
}
self.set_model_name(model)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
"top_p": params.top_p,
"latency": params.latency,
"additional_model_request_fields": params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
}
logger.info(f"Using AWS Bedrock model: {model}")
@@ -836,12 +861,12 @@ class AWSBedrockLLMService(LLMService):
Dictionary containing only the inference parameters that are not None.
"""
inference_config = {}
if self._settings["max_tokens"] is not None:
inference_config["maxTokens"] = self._settings["max_tokens"]
if self._settings["temperature"] is not None:
inference_config["temperature"] = self._settings["temperature"]
if self._settings["top_p"] is not None:
inference_config["topP"] = self._settings["top_p"]
if self._settings.max_tokens is not None:
inference_config["maxTokens"] = self._settings.max_tokens
if self._settings.temperature is not None:
inference_config["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
inference_config["topP"] = self._settings.top_p
return inference_config
async def run_inference(
@@ -877,9 +902,9 @@ class AWSBedrockLLMService(LLMService):
inference_config["maxTokens"] = max_tokens
request_params = {
"modelId": self.model_name,
"modelId": self._settings.model,
"messages": messages,
"additionalModelRequestFields": self._settings["additional_model_request_fields"],
"additionalModelRequestFields": self._settings.additional_model_request_fields,
}
if inference_config:
@@ -1034,9 +1059,9 @@ class AWSBedrockLLMService(LLMService):
# Prepare request parameters
request_params = {
"modelId": self.model_name,
"modelId": self._settings.model,
"messages": messages,
"additionalModelRequestFields": self._settings["additional_model_request_fields"],
"additionalModelRequestFields": self._settings.additional_model_request_fields,
}
# Only add inference config if it has parameters
@@ -1081,8 +1106,8 @@ class AWSBedrockLLMService(LLMService):
request_params["toolConfig"] = tool_config
# Add performance config if latency is specified
if self._settings["latency"] in ["standard", "optimized"]:
request_params["performanceConfig"] = {"latency": self._settings["latency"]}
if self._settings.latency in ["standard", "optimized"]:
request_params["performanceConfig"] = {"latency": self._settings.latency}
# Log request params with messages redacted for logging
if isinstance(context, LLMContext):
@@ -1207,8 +1232,6 @@ class AWSBedrockLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = AWSBedrockLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

117
src/pipecat/services/aws/nova_sonic/llm.py
View File

@@ -16,7 +16,7 @@ import json
import time
import uuid
import wave
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from importlib.resources import files
from typing import Any, List, Optional
@@ -60,6 +60,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
try:
@@ -185,6 +186,20 @@ class Params(BaseModel):
endpointing_sensitivity: Optional[str] = Field(default=None)
@dataclass
class AWSNovaSonicLLMSettings(LLMSettings):
"""Settings for AWS Nova Sonic LLM service.
Parameters:
voice_id: Voice for speech synthesis.
endpointing_sensitivity: Controls how quickly Nova Sonic decides the
user has stopped speaking. Can be "LOW", "MEDIUM", or "HIGH".
"""
voice_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSNovaSonicLLMService(LLMService):
"""AWS Nova Sonic speech-to-speech LLM service.
@@ -192,6 +207,8 @@ class AWSNovaSonicLLMService(LLMService):
and function calling capabilities using AWS Nova Sonic model.
"""
_settings: AWSNovaSonicLLMSettings
# Override the default adapter to use the AWSNovaSonicLLMAdapter one
adapter_class = AWSNovaSonicLLMAdapter
@@ -237,28 +254,51 @@ class AWSNovaSonicLLMService(LLMService):
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs)
params = params or Params()
super().__init__(
settings=AWSNovaSonicLLMSettings(
model=model,
voice_id=voice_id,
temperature=params.temperature,
max_tokens=params.max_tokens,
top_p=params.top_p,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
endpointing_sensitivity=params.endpointing_sensitivity,
),
**kwargs,
)
self._secret_access_key = secret_access_key
self._access_key_id = access_key_id
self._session_token = session_token
self._region = region
self._model = model
self._client: Optional[BedrockRuntimeClient] = None
self._voice_id = voice_id
self._params = params or Params()
# Audio I/O config (hardware settings, not runtime-tunable)
self._input_sample_rate = params.input_sample_rate
self._input_sample_size = params.input_sample_size
self._input_channel_count = params.input_channel_count
self._output_sample_rate = params.output_sample_rate
self._output_sample_size = params.output_sample_size
self._output_channel_count = params.output_channel_count
self._system_instruction = system_instruction
self._tools = tools
# Validate endpointing_sensitivity parameter
if (
self._params.endpointing_sensitivity
self._settings.endpointing_sensitivity
and not self._is_endpointing_sensitivity_supported()
):
logger.warning(
f"endpointing_sensitivity is not supported for model '{model}' and will be ignored. "
"This parameter is only supported starting with Nova 2 Sonic (amazon.nova-2-sonic-v1:0)."
)
self._params.endpointing_sensitivity = None
self._settings.endpointing_sensitivity = None
if not send_transcription_frames:
import warnings
@@ -302,6 +342,29 @@ class AWSNovaSonicLLMService(LLMService):
with wave.open(file_path.open("rb"), "rb") as wav_file:
self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes())
#
# settings
#
async def _update_settings(self, delta: AWSNovaSonicLLMSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._start_connecting()
self._warn_unhandled_updated_settings(changed)
return changed
#
# standard AIService frame handling
#
@@ -472,7 +535,7 @@ class AWSNovaSonicLLMService(LLMService):
# Start the bidirectional stream
self._stream = await self._client.invoke_model_with_bidirectional_stream(
InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model)
InvokeModelWithBidirectionalStreamOperationInput(model_id=self._settings.model)
)
# Send session start event
@@ -639,7 +702,7 @@ class AWSNovaSonicLLMService(LLMService):
def _is_first_generation_sonic_model(self) -> bool:
# Nova Sonic (the older model) is identified by "amazon.nova-sonic-v1:0"
return self._model == "amazon.nova-sonic-v1:0"
return self._settings.model == "amazon.nova-sonic-v1:0"
def _is_endpointing_sensitivity_supported(self) -> bool:
# endpointing_sensitivity is only supported with Nova 2 Sonic (and,
@@ -658,9 +721,9 @@ class AWSNovaSonicLLMService(LLMService):
turn_detection_config = (
f""",
"turnDetectionConfiguration": {{
"endpointingSensitivity": "{self._params.endpointing_sensitivity}"
"endpointingSensitivity": "{self._settings.endpointing_sensitivity}"
}}"""
if self._params.endpointing_sensitivity
if self._settings.endpointing_sensitivity
else ""
)
@@ -669,9 +732,9 @@ class AWSNovaSonicLLMService(LLMService):
"event": {{
"sessionStart": {{
"inferenceConfiguration": {{
"maxTokens": {self._params.max_tokens},
"topP": {self._params.top_p},
"temperature": {self._params.temperature}
"maxTokens": {self._settings.max_tokens},
"topP": {self._settings.top_p},
"temperature": {self._settings.temperature}
}}{turn_detection_config}
}}
}}
@@ -706,10 +769,10 @@ class AWSNovaSonicLLMService(LLMService):
}},
"audioOutputConfiguration": {{
"mediaType": "audio/lpcm",
"sampleRateHertz": {self._params.output_sample_rate},
"sampleSizeBits": {self._params.output_sample_size},
"channelCount": {self._params.output_channel_count},
"voiceId": "{self._voice_id}",
"sampleRateHertz": {self._output_sample_rate},
"sampleSizeBits": {self._output_sample_size},
"channelCount": {self._output_channel_count},
"voiceId": "{self._settings.voice_id}",
"encoding": "base64",
"audioType": "SPEECH"
}}{tools_config}
@@ -734,9 +797,9 @@ class AWSNovaSonicLLMService(LLMService):
"role": "USER",
"audioInputConfiguration": {{
"mediaType": "audio/lpcm",
"sampleRateHertz": {self._params.input_sample_rate},
"sampleSizeBits": {self._params.input_sample_size},
"channelCount": {self._params.input_channel_count},
"sampleRateHertz": {self._input_sample_rate},
"sampleSizeBits": {self._input_sample_size},
"channelCount": {self._input_channel_count},
"audioType": "SPEECH",
"encoding": "base64"
}}
@@ -1019,8 +1082,8 @@ class AWSNovaSonicLLMService(LLMService):
audio = base64.b64decode(audio_content)
frame = TTSAudioRawFrame(
audio=audio,
sample_rate=self._params.output_sample_rate,
num_channels=self._params.output_channel_count,
sample_rate=self._output_sample_rate,
num_channels=self._output_channel_count,
)
await self.push_frame(frame)
@@ -1304,7 +1367,7 @@ class AWSNovaSonicLLMService(LLMService):
"""
if not self._is_assistant_response_trigger_needed():
logger.warning(
f"Assistant response trigger not needed for model '{self._model}'; skipping. "
f"Assistant response trigger not needed for model '{self._settings.model}'; skipping. "
"An LLMRunFrame() should be sufficient to prompt the assistant to respond, "
"assuming the context ends in a user message."
)
@@ -1332,9 +1395,9 @@ class AWSNovaSonicLLMService(LLMService):
chunk_duration = 0.02 # what we might get from InputAudioRawFrame
chunk_size = int(
chunk_duration
* self._params.input_sample_rate
* self._params.input_channel_count
* (self._params.input_sample_size / 8)
* self._input_sample_rate
* self._input_channel_count
* (self._input_sample_size / 8)
) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes
# Lead with a bit of blank audio, if needed.

97
src/pipecat/services/aws/stt.py
View File

@@ -14,7 +14,8 @@ import json
import os
import random
import string
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -28,6 +29,7 @@ from pipecat.frames.frames import (
TranscriptionFrame,
)
from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import AWS_TRANSCRIBE_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -43,6 +45,25 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AWSTranscribeSTTSettings(STTSettings):
"""Settings for the AWS Transcribe STT service.
Parameters:
sample_rate: Audio sample rate in Hz (8000 or 16000).
media_encoding: Audio encoding format (e.g. "linear16").
number_of_channels: Number of audio channels.
show_speaker_label: Whether to show speaker labels.
enable_channel_identification: Whether to enable channel identification.
"""
sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
media_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
number_of_channels: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
show_speaker_label: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_channel_identification: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSTranscribeSTTService(WebsocketSTTService):
"""AWS Transcribe Speech-to-Text service using WebSocket streaming.
@@ -51,6 +72,8 @@ class AWSTranscribeSTTService(WebsocketSTTService):
final transcription results.
"""
_settings: AWSTranscribeSTTSettings
def __init__(
self,
*,
@@ -76,23 +99,25 @@ class AWSTranscribeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._settings = {
"sample_rate": sample_rate,
"language": language,
"media_encoding": "linear16", # AWS expects raw PCM
"number_of_channels": 1,
"show_speaker_label": False,
"enable_channel_identification": False,
}
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=AWSTranscribeSTTSettings(
language=self.language_to_service_language(language) or "en-US",
sample_rate=sample_rate,
media_encoding="linear16",
number_of_channels=1,
show_speaker_label=False,
enable_channel_identification=False,
),
**kwargs,
)
# Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz
if sample_rate not in [8000, 16000]:
logger.warning(
f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz."
)
self._settings["sample_rate"] = 16000
self._settings.sample_rate = 16000
self._credentials = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -103,6 +128,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
self._receive_task = None
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as AWS Transcribe STT supports metrics generation.
"""
return True
def get_service_encoding(self, encoding: str) -> str:
"""Convert internal encoding format to AWS Transcribe format.
@@ -117,6 +150,26 @@ class AWSTranscribeSTTService(WebsocketSTTService):
}
return encoding_map.get(encoding, encoding)
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if changed and self._websocket:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Initialize the connection when the service starts.
@@ -208,9 +261,9 @@ class AWSTranscribeSTTService(WebsocketSTTService):
logger.debug("Connecting to AWS Transcribe WebSocket")
language_code = self.language_to_service_language(Language(self._settings["language"]))
language_code = self._settings.language
if not language_code:
raise ValueError(f"Unsupported language: {self._settings['language']}")
raise ValueError(f"Unsupported language: {language_code}")
# Generate random websocket key
websocket_key = "".join(
@@ -237,14 +290,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
},
language_code=language_code,
media_encoding=self.get_service_encoding(
self._settings["media_encoding"]
self._settings.media_encoding
), # Convert to AWS format
sample_rate=self._settings["sample_rate"],
number_of_channels=self._settings["number_of_channels"],
sample_rate=self._settings.sample_rate,
number_of_channels=self._settings.number_of_channels,
enable_partial_results_stabilization=True,
partial_results_stability="high",
show_speaker_label=self._settings["show_speaker_label"],
enable_channel_identification=self._settings["enable_channel_identification"],
show_speaker_label=self._settings.show_speaker_label,
enable_channel_identification=self._settings.enable_channel_identification,
)
logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...")
@@ -479,14 +532,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
result=result,
)
)
await self._handle_transcription(
transcript,
is_final,
self._settings["language"],
self._settings.language,
)
await self.stop_processing_metrics()
else:
@@ -495,7 +548,7 @@ class AWSTranscribeSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
result=result,
)
)

76
src/pipecat/services/aws/tts.py
View File

@@ -11,6 +11,7 @@ supporting multiple languages, voices, and SSML features.
"""
import os
from dataclasses import dataclass, field
from typing import AsyncGenerator, List, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -121,6 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class AWSPollyTTSSettings(TTSSettings):
"""Settings for AWS Polly TTS service.
Parameters:
engine: TTS engine to use ('standard', 'neural', etc.).
pitch: Voice pitch adjustment (for standard engine only).
rate: Speech rate adjustment.
volume: Voice volume adjustment.
lexicon_names: List of pronunciation lexicons to apply.
"""
engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
lexicon_names: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AWSPollyTTSService(TTSService):
"""AWS Polly text-to-speech service.
@@ -129,6 +150,8 @@ class AWSPollyTTSService(TTSService):
options including prosody controls.
"""
_settings: AWSPollyTTSSettings
class InputParams(BaseModel):
"""Input parameters for AWS Polly TTS configuration.
@@ -172,10 +195,25 @@ class AWSPollyTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AWSPollyTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AWSPollyTTSSettings(
model=None,
voice=voice_id,
engine=params.engine,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
volume=params.volume,
lexicon_names=params.lexicon_names,
),
**kwargs,
)
# Get credentials from environment variables if not provided
self._aws_params = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -185,21 +223,9 @@ class AWSPollyTTSService(TTSService):
}
self._aws_session = aioboto3.Session()
self._settings = {
"engine": params.engine,
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
"lexicon_names": params.lexicon_names,
}
self._resampler = create_stream_resampler()
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -222,19 +248,19 @@ class AWSPollyTTSService(TTSService):
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
language = self._settings["language"]
language = self._settings.language
ssml += f"<lang xml:lang='{language}'>"
prosody_attrs = []
# Prosody tags are only supported for standard and neural engines
if self._settings["engine"] == "standard":
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings.engine == "standard":
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
@@ -275,11 +301,11 @@ class AWSPollyTTSService(TTSService):
"Text": ssml,
"TextType": "ssml",
"OutputFormat": "pcm",
"VoiceId": self._voice_id,
"Engine": self._settings["engine"],
"VoiceId": self._settings.voice,
"Engine": self._settings.engine,
# AWS only supports 8000 and 16000 for PCM. We select 16000.
"SampleRate": "16000",
"LexiconNames": self._settings["lexicon_names"],
"LexiconNames": self._settings.lexicon_names,
}
# Filter out None values

14
src/pipecat/services/azure/image.py
View File

@@ -12,6 +12,7 @@ using REST endpoints for creating images from text prompts.
import asyncio
import io
from dataclasses import dataclass
from typing import AsyncGenerator
import aiohttp
@@ -19,6 +20,16 @@ from PIL import Image
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class AzureImageGenSettings(ImageGenSettings):
"""Settings for the Azure image generation service.
Parameters:
model: Azure image generation model identifier.
"""
class AzureImageGenServiceREST(ImageGenService):
@@ -49,12 +60,11 @@ class AzureImageGenServiceREST(ImageGenService):
aiohttp_session: Shared aiohttp session for HTTP requests.
api_version: Azure API version string. Defaults to "2023-06-01-preview".
"""
super().__init__()
super().__init__(settings=AzureImageGenSettings(model=model))
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self.set_model_name(model)
self._image_size = image_size
self._aiohttp_session = aiohttp_session

72
src/pipecat/services/azure/stt.py
View File

@@ -11,7 +11,8 @@ Speech SDK for real-time audio transcription.
"""
import asyncio
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -25,6 +26,7 @@ from pipecat.frames.frames import (
TranscriptionFrame,
)
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import AZURE_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -48,6 +50,19 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class AzureSTTSettings(STTSettings):
"""Settings for the Azure STT service.
Parameters:
region: Azure region for the Speech service.
sample_rate: Audio sample rate in Hz.
"""
region: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AzureSTTService(STTService):
"""Azure Speech-to-Text service for real-time audio transcription.
@@ -56,6 +71,8 @@ class AzureSTTService(STTService):
provides real-time transcription results with timing information.
"""
_settings: AzureSTTSettings
def __init__(
self,
*,
@@ -82,7 +99,17 @@ class AzureSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=AzureSTTSettings(
model=None,
region=region,
language=language_to_azure_language(language),
sample_rate=sample_rate,
),
**kwargs,
)
self._speech_config = SpeechConfig(
subscription=api_key,
@@ -96,11 +123,6 @@ class AzureSTTService(STTService):
self._audio_stream = None
self._speech_recognizer = None
self._settings = {
"region": region,
"language": language_to_azure_language(language),
"sample_rate": sample_rate,
}
def can_generate_metrics(self) -> bool:
"""Check if this service can generate performance metrics.
@@ -110,6 +132,38 @@ class AzureSTTService(STTService):
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Azure service-specific language code.
Args:
language: The language to convert.
Returns:
The Azure-specific language identifier, or None if not supported.
"""
return language_to_azure_language(language)
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active recognizer.
"""
changed = await super()._update_settings(delta)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if "language" in changed:
# self._speech_config.speech_recognition_language = self._settings.language
# if self._speech_recognizer:
# # Requires refactoring to set up and tear down recognizer, as
# # language is applied at recognizer initialization
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text conversion.
@@ -202,7 +256,7 @@ class AzureSTTService(STTService):
def _on_handle_recognized(self, event):
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
language = getattr(event.result, "language", None) or self._settings.get("language")
language = getattr(event.result, "language", None) or self._settings.language
frame = TranscriptionFrame(
event.result.text,
self._user_id,
@@ -217,7 +271,7 @@ class AzureSTTService(STTService):
def _on_handle_recognizing(self, event):
if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0:
language = getattr(event.result, "language", None) or self._settings.get("language")
language = getattr(event.result, "language", None) or self._settings.language
frame = InterimTranscriptionFrame(
event.result.text,
self._user_id,

147
src/pipecat/services/azure/tts.py
View File

@@ -7,6 +7,7 @@
"""Azure Cognitive Services Text-to-Speech service implementations."""
import asyncio
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -25,7 +26,8 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.tts_service import TTSService, WordTTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -65,6 +67,31 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma
return sample_rate_map.get(sample_rate, SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm)
@dataclass
class AzureTTSSettings(TTSSettings):
"""Settings for Azure TTS services.
Parameters:
emphasis: Emphasis level for speech ("strong", "moderate", "reduced").
language: Language for synthesis. Defaults to English (US).
pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high").
rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast").
role: Voice role for expression (e.g., "YoungAdultFemale").
style: Speaking style (e.g., "cheerful", "sad", "excited").
style_degree: Intensity of the speaking style (0.01 to 2.0).
volume: Volume level (e.g., "+20%", "loud", "x-soft").
"""
emphasis: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
role: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style_degree: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AzureBaseTTSService:
"""Base mixin class for Azure Cognitive Services text-to-speech implementations.
@@ -73,6 +100,8 @@ class AzureBaseTTSService:
This is a mixin class and should be used alongside TTSService or its subclasses.
"""
_settings: AzureTTSSettings
# Define SSML escape mappings based on SSML reserved characters
# See - https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-structure
SSML_ESCAPE_CHARS = {
@@ -112,7 +141,6 @@ class AzureBaseTTSService:
api_key: str,
region: str,
voice: str = "en-US-SaraNeural",
params: Optional[InputParams] = None,
):
"""Initialize Azure-specific configuration.
@@ -122,26 +150,9 @@ class AzureBaseTTSService:
api_key: Azure Cognitive Services subscription key.
region: Azure region identifier (e.g., "eastus", "westus2").
voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
params: Voice and synthesis parameters configuration.
"""
params = params or AzureBaseTTSService.InputParams()
self._settings = {
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"role": params.role,
"style": params.style,
"style_degree": params.style_degree,
"volume": params.volume,
}
self._api_key = api_key
self._region = region
self._voice_id = voice
self._speech_synthesizer = None
def language_to_service_language(self, language: Language) -> Optional[str]:
@@ -156,7 +167,7 @@ class AzureBaseTTSService:
return language_to_azure_language(language)
def _construct_ssml(self, text: str) -> str:
language = self._settings["language"]
language = self._settings.language
# Escape special characters
escaped_text = self._escape_text(text)
@@ -165,42 +176,42 @@ class AzureBaseTTSService:
f"<speak version='1.0' xml:lang='{language}' "
"xmlns='http://www.w3.org/2001/10/synthesis' "
"xmlns:mstts='http://www.w3.org/2001/mstts'>"
f"<voice name='{self._voice_id}'>"
f"<voice name='{self._settings.voice}'>"
"<mstts:silence type='Sentenceboundary' value='20ms' />"
)
if self._settings["style"]:
ssml += f"<mstts:express-as style='{self._settings['style']}'"
if self._settings["style_degree"]:
ssml += f" styledegree='{self._settings['style_degree']}'"
if self._settings["role"]:
ssml += f" role='{self._settings['role']}'"
if self._settings.style:
ssml += f"<mstts:express-as style='{self._settings.style}'"
if self._settings.style_degree:
ssml += f" styledegree='{self._settings.style_degree}'"
if self._settings.role:
ssml += f" role='{self._settings.role}'"
ssml += ">"
prosody_attrs = []
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
# Only wrap in prosody tag if there are prosody attributes
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
if self._settings.emphasis:
ssml += f"<emphasis level='{self._settings.emphasis}'>"
ssml += escaped_text
if self._settings["emphasis"]:
if self._settings.emphasis:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
if self._settings["style"]:
if self._settings.style:
ssml += "</mstts:express-as>"
ssml += "</voice></speak>"
@@ -229,7 +240,7 @@ class AzureBaseTTSService:
return escaped_text
class AzureTTSService(WordTTSService, AzureBaseTTSService):
class AzureTTSService(TTSService, AzureBaseTTSService):
"""Azure Cognitive Services streaming TTS service with word timestamps.
Provides real-time text-to-speech synthesis using Azure's WebSocket-based
@@ -245,7 +256,8 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
voice: str = "en-US-SaraNeural",
sample_rate: Optional[int] = None,
params: Optional[AzureBaseTTSService.InputParams] = None,
aggregate_sentences: bool = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize the Azure streaming TTS service.
@@ -256,21 +268,43 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
sample_rate: Audio sample rate in Hz. If None, uses service default.
params: Voice and synthesis parameters configuration.
aggregate_sentences: Whether to aggregate sentences before synthesis.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to parent WordTTSService.
"""
# Initialize WordTTSService first to set up word timestamp tracking
params = params or AzureBaseTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_text_frames=False, # We'll push text frames based on word timestamps
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs,
)
# Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None
self._speech_synthesizer = None
@@ -314,7 +348,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
subscription=self._api_key,
region=self._region,
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.speech_synthesis_language = self._settings.language
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)
@@ -364,7 +398,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
Returns:
True if the language is CJK, False otherwise.
"""
language = self._settings.get("language", "").lower()
language = (self._settings.language if self._settings.language else "").lower()
# Check if language starts with CJK language codes
return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu"))
@@ -704,10 +738,29 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
params: Voice and synthesis parameters configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AzureBaseTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=AzureTTSSettings(
model=None,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
pitch=params.pitch,
rate=params.rate,
role=params.role,
style=params.style,
style_degree=params.style_degree,
voice=voice,
volume=params.volume,
),
**kwargs,
)
# Initialize Azure-specific functionality from mixin
self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
self._init_azure_base(api_key=api_key, region=region, voice=voice)
self._speech_config = None
self._speech_synthesizer = None
@@ -735,7 +788,7 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
subscription=self._api_key,
region=self._region,
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.speech_synthesis_language = self._settings.language
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)

57
src/pipecat/services/camb/tts.py
View File

@@ -16,6 +16,7 @@ Features:
- Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz)
"""
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional
from camb import StreamTtsOutputConfiguration
@@ -31,6 +32,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -133,6 +135,18 @@ def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]:
return buffer[:aligned_size], buffer[aligned_size:]
@dataclass
class CambTTSSettings(TTSSettings):
"""Settings for Camb.ai TTS service.
Parameters:
user_instructions: Custom instructions for mars-instruct model only.
Ignored for other models. Max 1000 characters.
"""
user_instructions: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class CambTTSService(TTSService):
"""Camb.ai MARS text-to-speech service using the official SDK.
@@ -156,6 +170,8 @@ class CambTTSService(TTSService):
)
"""
_settings: CambTTSSettings
class InputParams(BaseModel):
"""Input parameters for Camb.ai TTS configuration.
@@ -197,11 +213,6 @@ class CambTTSService(TTSService):
params: Additional voice parameters. If None, uses defaults.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._timeout = timeout
params = params or CambTTSService.InputParams()
# Warn if sample rate doesn't match model's supported rate
@@ -211,17 +222,23 @@ class CambTTSService(TTSService):
f"sample rate. Current rate of {sample_rate}Hz may cause issues."
)
# Build settings
self._settings = {
"language": (
self.language_to_service_language(params.language) if params.language else "en-us"
super().__init__(
sample_rate=sample_rate,
settings=CambTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language)
if params.language
else "en-us"
),
user_instructions=params.user_instructions,
),
"user_instructions": params.user_instructions,
}
**kwargs,
)
self.set_model_name(model)
self.set_voice(str(voice_id))
self._voice_id = voice_id
self._api_key = api_key
self._timeout = timeout
self._client = None
@@ -256,7 +273,7 @@ class CambTTSService(TTSService):
# Use model-specific sample rate if not explicitly specified
if not self._init_sample_rate:
self._sample_rate = MODEL_SAMPLE_RATES.get(self.model_name, 22050)
self._sample_rate = MODEL_SAMPLE_RATES.get(self._settings.model, 22050)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -282,15 +299,15 @@ class CambTTSService(TTSService):
# Build SDK parameters
tts_kwargs: Dict[str, Any] = {
"text": text,
"voice_id": self._voice_id,
"language": self._settings["language"],
"speech_model": self.model_name,
"voice_id": self._settings.voice,
"language": self._settings.language,
"speech_model": self._settings.model,
"output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"),
}
# Add user instructions if using mars-instruct model
if self._model_name == "mars-instruct" and self._settings.get("user_instructions"):
tts_kwargs["user_instructions"] = self._settings["user_instructions"]
if self._settings.model == "mars-instruct" and self._settings.user_instructions:
tts_kwargs["user_instructions"] = self._settings.user_instructions
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame(context_id=context_id)

67
src/pipecat/services/cartesia/stt.py
View File

@@ -12,7 +12,8 @@ the Cartesia Live transcription API for real-time speech recognition.
import json
import urllib.parse
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import CARTESIA_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -42,6 +44,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class CartesiaSTTSettings(STTSettings):
"""Settings for the Cartesia STT service.
Parameters:
encoding: Audio encoding format (e.g. ``"pcm_s16le"``).
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class CartesiaLiveOptions:
"""Configuration options for Cartesia Live STT service.
@@ -136,6 +149,8 @@ class CartesiaSTTService(WebsocketSTTService):
See: https://docs.cartesia.ai/api-reference/stt/stt
"""
_settings: CartesiaSTTSettings
def __init__(
self,
*,
@@ -158,13 +173,6 @@ class CartesiaSTTService(WebsocketSTTService):
**kwargs: Additional arguments passed to parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=120,
keepalive_interval=30,
**kwargs,
)
default_options = CartesiaLiveOptions(
model="ink-whisper",
@@ -181,8 +189,19 @@ class CartesiaSTTService(WebsocketSTTService):
k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None"
}
self._settings = merged_options
self.set_model_name(merged_options["model"])
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=120,
keepalive_interval=30,
settings=CartesiaSTTSettings(
model=merged_options["model"],
language=merged_options.get("language"),
encoding=merged_options.get("encoding", "pcm_s16le"),
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url or "api.cartesia.ai"
self._receive_task = None
@@ -275,13 +294,39 @@ class CartesiaSTTService(WebsocketSTTService):
await self._disconnect_websocket()
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Args:
delta: A :class:`STTSettings` (or ``CartesiaSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if changed:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def _connect_websocket(self):
try:
if self._websocket and self._websocket.state is State.OPEN:
return
logger.debug("Connecting to Cartesia STT")
params = self._settings
params = {
"model": self._settings.model,
"language": self._settings.language,
"encoding": self._settings.encoding,
"sample_rate": str(self.sample_rate),
}
ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}"
headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key}

282
src/pipecat/services/cartesia/tts.py
View File

@@ -8,10 +8,10 @@
import base64
import json
import uuid
import warnings
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, List, Literal, Optional
from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional
from loguru import logger
from pydantic import BaseModel, Field
@@ -21,14 +21,13 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
@@ -192,7 +191,43 @@ class CartesiaEmotion(str, Enum):
DETERMINED = "determined"
class CartesiaTTSService(AudioContextWordTTSService):
@dataclass
class CartesiaTTSSettings(TTSSettings):
"""Settings for Cartesia TTS services.
Parameters:
output_container: Audio container format (e.g. "raw").
output_encoding: Audio encoding format (e.g. "pcm_s16le").
output_sample_rate: Audio sample rate in Hz.
speed: Voice speed control for non-Sonic-3 models (literal values).
emotion: List of emotion controls for non-Sonic-3 models.
generation_config: Generation configuration for Sonic-3 models. Includes volume,
speed (numeric), and emotion (string) parameters.
pronunciation_dict_id: The ID of the pronunciation dictionary to use for
custom pronunciations.
"""
output_container: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: Literal["slow", "normal", "fast"] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emotion: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
generation_config: GenerationConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pronunciation_dict_id: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "CartesiaTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``output_format``."""
flat = dict(settings)
nested = flat.pop("output_format", None)
if isinstance(nested, dict):
flat.setdefault("output_container", nested.get("container"))
flat.setdefault("output_encoding", nested.get("encoding"))
flat.setdefault("output_sample_rate", nested.get("sample_rate"))
return super().from_mapping(flat)
class CartesiaTTSService(AudioContextTTSService):
"""Cartesia TTS service with WebSocket streaming and word timestamps.
Provides text-to-speech using Cartesia's streaming WebSocket API.
@@ -200,6 +235,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
customization options including speed and emotion controls.
"""
_settings: CartesiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Cartesia TTS configuration.
@@ -235,7 +272,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
container: str = "raw",
params: Optional[InputParams] = None,
text_aggregator: Optional[BaseTextAggregator] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize the Cartesia TTS service.
@@ -255,25 +293,51 @@ class CartesiaTTSService(AudioContextWordTTSService):
.. deprecated:: 0.0.95
Use an LLMTextProcessor before the TTSService for custom text aggregation.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to the parent service.
"""
# Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
# 3 model, and it's worth it for the better audio quality.
# By default, we aggregate sentences before sending to TTS. This adds
# ~200-300ms of latency per sentence (waiting for the sentence-ending
# punctuation token from the LLM). Setting
# text_aggregation_mode=TextAggregationMode.TOKEN streams tokens
# directly, which reduces latency. Streaming quality is good but less
# tested than sentence aggregation.
# TODO: Consider making TOKEN the default for Cartesia in 1.0.
#
# We also don't want to automatically push LLM response text frames,
# because the context aggregators will add them to the LLM context even
# if we're interrupted. Cartesia gives us word-by-word timestamps. We
# can use those to generate text frames ourselves aligned with the
# playout timing of the audio!
params = params or CartesiaTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
text_aggregator=text_aggregator,
settings=CartesiaTTSSettings(
model=model,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
voice=voice_id,
),
**kwargs,
)
@@ -283,31 +347,14 @@ class CartesiaTTSService(AudioContextWordTTSService):
# The preferred way of taking advantage of Cartesia SSML Tags is
# to use an LLMTextProcessor and/or a text_transformer to identify
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("<spell>", "</spell>")])
params = params or CartesiaTTSService.InputParams()
self._text_aggregator = SkipTagsAggregator(
[("<spell>", "</spell>")], aggregation_type=self._text_aggregation_mode
)
self._api_key = api_key
self._cartesia_version = cartesia_version
self._url = url
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
"speed": params.speed,
"emotion": params.emotion,
"generation_config": params.generation_config,
"pronunciation_dict_id": params.pronunciation_dict_id,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._context_id = None
self._receive_task = None
def can_generate_metrics(self) -> bool:
@@ -318,16 +365,6 @@ class CartesiaTTSService(AudioContextWordTTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model.
Args:
model: The model name to use for synthesis.
"""
self._model_id = model
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Cartesia language format.
@@ -392,7 +429,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
Returns:
List of (word, start_time) tuples processed for the language.
"""
current_language = self._settings.get("language")
current_language = self._settings.language
# Check if this is a CJK language (if language is None, treat as non-CJK)
if current_language and self._is_cjk_language(current_language):
@@ -413,9 +450,9 @@ class CartesiaTTSService(AudioContextWordTTSService):
):
voice_config = {}
voice_config["mode"] = "id"
voice_config["id"] = self._voice_id
voice_config["id"] = self._settings.voice
if self._settings["emotion"]:
if self._settings.emotion:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -424,33 +461,36 @@ class CartesiaTTSService(AudioContextWordTTSService):
stacklevel=2,
)
voice_config["__experimental_controls"] = {}
if self._settings["emotion"]:
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
voice_config["__experimental_controls"]["emotion"] = self._settings.emotion
msg = {
"transcript": text,
"continue": continue_transcript,
"context_id": self._context_id,
"model_id": self.model_name,
"context_id": self.get_active_audio_context_id(),
"model_id": self._settings.model,
"voice": voice_config,
"output_format": self._settings["output_format"],
"output_format": {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
},
"add_timestamps": add_timestamps,
"use_original_timestamps": False if self.model_name == "sonic" else True,
"use_original_timestamps": False if self._settings.model == "sonic" else True,
}
if self._settings["language"]:
msg["language"] = self._settings["language"]
if self._settings.language:
msg["language"] = self._settings.language
if self._settings["speed"]:
msg["speed"] = self._settings["speed"]
if self._settings.speed:
msg["speed"] = self._settings.speed
if self._settings["generation_config"]:
msg["generation_config"] = self._settings["generation_config"].model_dump(
if self._settings.generation_config:
msg["generation_config"] = self._settings.generation_config.model_dump(
exclude_none=True
)
if self._settings["pronunciation_dict_id"]:
msg["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
if self._settings.pronunciation_dict_id:
msg["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
return json.dumps(msg)
@@ -461,7 +501,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -523,7 +563,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
finally:
self._context_id = None
await self.remove_active_audio_context()
self._websocket = None
await self._call_event_handler("on_disconnected")
@@ -532,36 +572,31 @@ class CartesiaTTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
async def on_audio_context_interrupted(self, context_id: str):
"""Cancel the active Cartesia context when the bot is interrupted."""
await self.stop_all_metrics()
if self._context_id:
cancel_msg = json.dumps({"context_id": self._context_id, "cancel": True})
if context_id:
cancel_msg = json.dumps({"context_id": context_id, "cancel": True})
await self._get_websocket().send(cancel_msg)
self._context_id = None
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request in case we don't have one already in progress.
async def on_audio_context_completed(self, context_id: str):
"""Close the Cartesia context after all audio has been played.
Returns:
A unique string identifier for the TTS context.
No close message is needed: the server already considers the context
done once it has sent its ``done`` message, which is handled in
``_process_messages``.
"""
# If a context ID does not exist, create a new one.
# If an ID exists, continue using the current ID.
# When interruptions happen, user speech results in
# an interruption, which resets the context ID.
if not self._context_id:
return str(uuid.uuid4())
return self._context_id
pass
async def flush_audio(self):
"""Flush any pending audio and finalize the current context."""
if not self._context_id or not self._websocket:
context_id = self.get_active_audio_context_id()
if not context_id or not self._websocket:
return
logger.trace(f"{self}: flushing audio")
msg = self._build_msg(text="", continue_transcript=False)
await self._websocket.send(msg)
self._context_id = None
self.reset_active_audio_context()
async def _process_messages(self):
async for message in self._get_websocket():
@@ -593,7 +628,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
await self.push_frame(TTSStoppedFrame(context_id=ctx_id))
await self.stop_all_metrics()
await self.push_error(error_msg=f"Error: {msg}")
self._context_id = None
self.reset_active_audio_context()
else:
await self.push_error(error_msg=f"Error, unknown message type: {msg}")
@@ -616,17 +651,19 @@ class CartesiaTTSService(AudioContextWordTTSService):
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]")
if not self._is_streaming_tokens:
logger.debug(f"{self}: Generating TTS [{text}]")
else:
logger.trace(f"{self}: Generating TTS [{text}]")
try:
if not self._websocket or self._websocket.state is State.CLOSED:
await self._connect()
if not self._context_id:
if not self.has_active_audio_context():
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
self._context_id = context_id
await self.create_audio_context(self._context_id)
await self.create_audio_context(context_id)
msg = self._build_msg(text=text)
@@ -652,6 +689,8 @@ class CartesiaHttpTTSService(TTSService):
integration is preferred.
"""
_settings: CartesiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Cartesia HTTP TTS configuration.
@@ -702,29 +741,30 @@ class CartesiaHttpTTSService(TTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or CartesiaHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=CartesiaTTSSettings(
model=model,
voice=voice_id,
output_container=container,
output_encoding=encoding,
output_sample_rate=0,
language=self.language_to_service_language(params.language)
if params.language
else None,
speed=params.speed,
emotion=params.emotion,
generation_config=params.generation_config,
pronunciation_dict_id=params.pronunciation_dict_id,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._cartesia_version = cartesia_version
self._settings = {
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
else None,
"speed": params.speed,
"emotion": params.emotion,
"generation_config": params.generation_config,
"pronunciation_dict_id": params.pronunciation_dict_id,
}
self.set_voice(voice_id)
self.set_model_name(model)
self._client = AsyncCartesia(
api_key=api_key,
@@ -757,7 +797,7 @@ class CartesiaHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
self._settings.output_sample_rate = self.sample_rate
async def stop(self, frame: EndFrame):
"""Stop the Cartesia HTTP TTS service.
@@ -791,9 +831,9 @@ class CartesiaHttpTTSService(TTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
voice_config = {"mode": "id", "id": self._voice_id}
voice_config = {"mode": "id", "id": self._settings.voice}
if self._settings["emotion"]:
if self._settings.emotion:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -801,30 +841,36 @@ class CartesiaHttpTTSService(TTSService):
DeprecationWarning,
stacklevel=2,
)
voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]}
voice_config["__experimental_controls"] = {"emotion": self._settings.emotion}
await self.start_ttfb_metrics()
payload = {
"model_id": self._model_name,
"transcript": text,
"voice": voice_config,
"output_format": self._settings["output_format"],
output_format = {
"container": self._settings.output_container,
"encoding": self._settings.output_encoding,
"sample_rate": self._settings.output_sample_rate,
}
if self._settings["language"]:
payload["language"] = self._settings["language"]
payload = {
"model_id": self._settings.model,
"transcript": text,
"voice": voice_config,
"output_format": output_format,
}
if self._settings["speed"]:
payload["speed"] = self._settings["speed"]
if self._settings.language:
payload["language"] = self._settings.language
if self._settings["generation_config"]:
payload["generation_config"] = self._settings["generation_config"].model_dump(
if self._settings.speed:
payload["speed"] = self._settings.speed
if self._settings.generation_config:
payload["generation_config"] = self._settings.generation_config.model_dump(
exclude_none=True
)
if self._settings["pronunciation_dict_id"]:
payload["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
if self._settings.pronunciation_dict_id:
payload["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
yield TTSStartedFrame(context_id=context_id)

12
src/pipecat/services/cerebras/llm.py
View File

@@ -66,16 +66,16 @@ class CerebrasLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"seed": self._settings.seed,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_completion_tokens": self._settings.max_completion_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

102
src/pipecat/services/deepgram/flux/stt.py
View File

@@ -9,6 +9,7 @@
import asyncio
import json
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator, Dict, Optional
from urllib.parse import urlencode
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
@@ -67,6 +69,34 @@ class FluxEventType(str, Enum):
UPDATE = "Update"
@dataclass
class DeepgramFluxSTTSettings(STTSettings):
"""Settings for the Deepgram Flux STT service.
Parameters:
eager_eot_threshold: EagerEndOfTurn/TurnResumed threshold. Off by default.
Lower values = more aggressive (faster response, more LLM calls).
Higher values = more conservative (slower response, fewer LLM calls).
eot_threshold: End-of-turn confidence required to finish a turn (default 0.7).
eot_timeout_ms: Time in ms after speech to finish a turn regardless of EOT
confidence (default 5000).
keyterm: Keyterms to boost recognition accuracy for specialized terminology.
mip_opt_out: Opt out of the Deepgram Model Improvement Program (default False).
tag: Tags to label requests for identification during usage reporting.
min_confidence: Minimum confidence required to create a TranscriptionFrame.
encoding: Audio encoding format (e.g. ``"linear16"``).
"""
eager_eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
eot_timeout_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
keyterm: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
mip_opt_out: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
tag: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_confidence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramFluxSTTService(WebsocketSTTService):
"""Deepgram Flux speech-to-text service.
@@ -89,6 +119,8 @@ class DeepgramFluxSTTService(WebsocketSTTService):
...
"""
_settings: DeepgramFluxSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Deepgram Flux API.
@@ -175,20 +207,27 @@ class DeepgramFluxSTTService(WebsocketSTTService):
# was never destroyed.
# So we can keep it here as false, because inside the method send_with_retry, it will
# already try to reconnect if needed.
params = params or DeepgramFluxSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
reconnect_on_error=False,
settings=DeepgramFluxSTTSettings(
model=model,
language=Language.EN,
encoding=flux_encoding,
eager_eot_threshold=params.eager_eot_threshold,
eot_threshold=params.eot_threshold,
eot_timeout_ms=params.eot_timeout_ms,
keyterm=params.keyterm or [],
mip_opt_out=params.mip_opt_out,
tag=params.tag or [],
min_confidence=params.min_confidence,
),
**kwargs,
)
self._api_key = api_key
self._url = url
self._model = model
self._params = params or DeepgramFluxSTTService.InputParams()
self._should_interrupt = should_interrupt
self._flux_encoding = flux_encoding
# This is the currently only supported language
self._language = Language.EN
self._websocket_url = None
self._receive_task = None
# Flux event handlers
@@ -343,6 +382,25 @@ class DeepgramFluxSTTService(WebsocketSTTService):
"""
return True
async def _update_settings(self, delta: DeepgramFluxSTTSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram Flux STT service.
@@ -355,29 +413,29 @@ class DeepgramFluxSTTService(WebsocketSTTService):
await super().start(frame)
url_params = [
f"model={self._model}",
f"model={self._settings.model}",
f"sample_rate={self.sample_rate}",
f"encoding={self._flux_encoding}",
f"encoding={self._settings.encoding}",
]
if self._params.eager_eot_threshold is not None:
url_params.append(f"eager_eot_threshold={self._params.eager_eot_threshold}")
if self._settings.eager_eot_threshold is not None:
url_params.append(f"eager_eot_threshold={self._settings.eager_eot_threshold}")
if self._params.eot_threshold is not None:
url_params.append(f"eot_threshold={self._params.eot_threshold}")
if self._settings.eot_threshold is not None:
url_params.append(f"eot_threshold={self._settings.eot_threshold}")
if self._params.eot_timeout_ms is not None:
url_params.append(f"eot_timeout_ms={self._params.eot_timeout_ms}")
if self._settings.eot_timeout_ms is not None:
url_params.append(f"eot_timeout_ms={self._settings.eot_timeout_ms}")
if self._params.mip_opt_out is not None:
url_params.append(f"mip_opt_out={str(self._params.mip_opt_out).lower()}")
if self._settings.mip_opt_out is not None:
url_params.append(f"mip_opt_out={str(self._settings.mip_opt_out).lower()}")
# Add keyterm parameters (can have multiple)
for keyterm in self._params.keyterm:
for keyterm in self._settings.keyterm:
url_params.append(urlencode({"keyterm": keyterm}))
# Add tag parameters (can have multiple)
for tag_value in self._params.tag:
for tag_value in self._settings.tag:
url_params.append(urlencode({"tag": tag_value}))
self._websocket_url = f"{self._url}?{'&'.join(url_params)}"
@@ -676,7 +734,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
# Compute the average confidence
average_confidence = self._calculate_average_confidence(data)
if not self._params.min_confidence or average_confidence > self._params.min_confidence:
if not self._settings.min_confidence or average_confidence > self._settings.min_confidence:
# EndOfTurn means Flux has determined the turn is complete,
# so this TranscriptionFrame is always finalized
await self.push_frame(
@@ -684,7 +742,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
result=data,
finalized=True,
)
@@ -694,7 +752,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
f"Transcription confidence below min_confidence threshold: {average_confidence}"
)
await self._handle_transcription(transcript, True, self._language)
await self._handle_transcription(transcript, True, self._settings.language)
await self.stop_processing_metrics()
await self.broadcast_frame(UserStoppedSpeakingFrame)
await self._call_event_handler("on_end_of_turn", transcript)
@@ -738,7 +796,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language,
self._settings.language,
result=data,
)
)

233
src/pipecat/services/deepgram/stt.py
View File

@@ -6,7 +6,9 @@
"""Deepgram speech-to-text service implementation."""
from typing import AsyncGenerator, Dict, Optional
import inspect
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Mapping, Optional, Type
from loguru import logger
@@ -23,6 +25,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import _S, NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import DEEPGRAM_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -45,6 +48,168 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class _DeepgramSTTSettingsBase(STTSettings):
"""Base settings for Deepgram STT services that use ``LiveOptions``.
Shared by ``DeepgramSTTSettings`` and ``DeepgramSageMakerSTTSettings``.
Not intended for other Deepgram services that don't use ``LiveOptions``.
Wraps the Deepgram SDK's ``LiveOptions`` in a single ``live_options``
field and provides delta-merge semantics: when used as a delta (e.g.
via ``STTUpdateSettingsFrame``), only the non-None fields of
``live_options`` are merged into the stored options rather than
replacing them wholesale.
``model`` and ``language`` are kept in sync bidirectionally between
the top-level settings fields and the nested ``live_options``.
Parameters:
live_options: Deepgram ``LiveOptions`` for STT configuration.
In delta mode only its non-None fields are merged into the
stored options.
"""
live_options: LiveOptions | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
# Valid LiveOptions __init__ parameter names (cached at class level).
_live_options_params: set[str] | None = field(default=None, init=False, repr=False)
@classmethod
def _get_live_options_params(cls) -> set[str]:
"""Return the set of valid ``LiveOptions.__init__`` parameter names."""
if cls._live_options_params is None:
cls._live_options_params = set(inspect.signature(LiveOptions.__init__).parameters) - {
"self"
}
return cls._live_options_params
def _merge_live_options_delta(self, delta: LiveOptions) -> Dict[str, Any]:
"""Merge a ``LiveOptions`` delta into the stored ``live_options``.
Non-None fields from *delta* overwrite corresponding fields in the
stored ``LiveOptions``. ``model`` and ``language`` are synced to
the top-level settings fields when they change.
Args:
delta: A ``LiveOptions`` whose non-None fields are the desired
overrides.
Returns:
Dict mapping each changed key to its **previous** value (same
contract as ``apply_update``).
"""
old_dict = self.live_options.to_dict() # type: ignore[union-attr]
delta_dict = delta.to_dict()
# Deepgram SDK bug: model initialised to the *string* "None".
if delta_dict.get("model") == "None":
del delta_dict["model"]
if not delta_dict:
return {}
merged = {**old_dict, **delta_dict}
self.live_options = LiveOptions(**merged)
# Track what changed.
changed: Dict[str, Any] = {}
for key in delta_dict:
old_val = old_dict.get(key, NOT_GIVEN)
if old_val != delta_dict[key]:
changed[key] = old_val
# Sync model/language from live_options delta to top-level fields.
if "model" in delta_dict and delta_dict["model"] != self.model:
changed.setdefault("model", self.model)
self.model = delta_dict["model"]
if "language" in delta_dict and delta_dict["language"] != self.language:
changed.setdefault("language", self.language)
self.language = delta_dict["language"]
return changed
def apply_update(self: _S, delta: _S) -> Dict[str, Any]:
"""Merge a delta into this store, with delta-merge for ``live_options``.
``live_options`` is merged field-by-field via
``_merge_live_options_delta`` rather than being replaced wholesale.
``model`` and ``language`` are kept in sync bidirectionally between
the top-level settings fields and ``live_options``.
"""
# Pull live_options out of the delta so super() doesn't replace it.
delta_lo = getattr(delta, "live_options", NOT_GIVEN)
if is_given(delta_lo):
delta.live_options = NOT_GIVEN # type: ignore[assignment]
# Let the base class handle model, language, extra.
changed = super().apply_update(delta)
# Sync top-level model/language changes into stored live_options.
if "model" in changed:
self.live_options.model = self.model # type: ignore[union-attr]
if "language" in changed:
self.live_options.language = self.language # type: ignore[union-attr]
# Merge live_options delta. Top-level model/language take precedence
# over conflicting values in live_options, so write them into the
# delta before merging.
if is_given(delta_lo):
if "model" in changed:
delta_lo.model = self.model
if "language" in changed:
delta_lo.language = self.language
for key, old_val in self._merge_live_options_delta(delta_lo).items():
changed.setdefault(key, old_val)
return changed
@classmethod
def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S:
"""Build a delta from a plain dict, routing LiveOptions keys correctly.
Keys that are valid ``LiveOptions.__init__`` parameters (and not
top-level ``STTSettings`` fields like ``model`` / ``language``) are
collected into a ``LiveOptions`` object. ``model`` and ``language``
are routed to the top-level settings fields. Truly unknown keys go
to ``extra``.
"""
lo_params = cls._get_live_options_params()
stt_field_names = {"model", "language"}
kwargs: Dict[str, Any] = {}
lo_kwargs: Dict[str, Any] = {}
extra: Dict[str, Any] = {}
for key, value in settings.items():
canonical = cls._aliases.get(key, key)
if canonical in stt_field_names:
kwargs[canonical] = value
elif canonical in lo_params:
lo_kwargs[canonical] = value
else:
extra[key] = value
if lo_kwargs:
kwargs["live_options"] = LiveOptions(**lo_kwargs)
instance = cls(**kwargs)
instance.extra = extra
return instance
@dataclass
class DeepgramSTTSettings(_DeepgramSTTSettingsBase):
"""Settings for the Deepgram STT service.
See ``_DeepgramSTTSettingsBase`` for full documentation.
"""
pass
class DeepgramSTTService(STTService):
"""Deepgram speech-to-text service.
@@ -63,6 +228,8 @@ class DeepgramSTTService(STTService):
...
"""
_settings: DeepgramSTTSettings
def __init__(
self,
*,
@@ -87,7 +254,9 @@ class DeepgramSTTService(STTService):
base_url: Custom Deepgram API base URL.
sample_rate: Audio sample rate. If None, uses default or live_options value.
live_options: Deepgram LiveOptions for detailed configuration.
live_options: Deepgram LiveOptions configuration. Treated as a
delta from a set of sensible defaults — only the fields you
set are overridden; all others keep their default values.
addons: Additional Deepgram features to enable.
should_interrupt: Determine whether the bot should be interrupted when Deepgram VAD events are enabled and the system detects that the user is speaking.
@@ -102,7 +271,6 @@ class DeepgramSTTService(STTService):
The `vad_events` option in LiveOptions is deprecated as of version 0.0.99 and will be removed in a future version. Please use the Silero VAD instead.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if url:
import warnings
@@ -127,24 +295,25 @@ class DeepgramSTTService(STTService):
vad_events=False,
)
merged_options = default_options.to_dict()
settings = DeepgramSTTSettings(
model=default_options.model,
language=default_options.language,
live_options=default_options,
)
if live_options:
default_model = default_options.model
merged_options.update(live_options.to_dict())
# NOTE(aleix): Fixes an in deepgram-sdk where `model` is initialized
# to the string "None" instead of the value `None`.
if "model" in merged_options and merged_options["model"] == "None":
merged_options["model"] = default_model
settings._merge_live_options_delta(live_options)
if "language" in merged_options and isinstance(merged_options["language"], Language):
merged_options["language"] = merged_options["language"].value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
self.set_model_name(merged_options["model"])
self._settings = merged_options
self._addons = addons
self._should_interrupt = should_interrupt
if merged_options.get("vad_events"):
if self._settings.live_options.vad_events:
import warnings
with warnings.catch_warnings():
@@ -175,7 +344,7 @@ class DeepgramSTTService(STTService):
Returns:
True if VAD events are enabled in the current settings.
"""
return self._settings["vad_events"]
return self._settings.live_options.vad_events
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -185,28 +354,17 @@ class DeepgramSTTService(STTService):
"""
return True
async def set_model(self, model: str):
"""Set the Deepgram model and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if anything changed."""
changed = await super()._update_settings(delta)
if not changed:
return changed
Args:
model: The Deepgram model name to use.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._settings["model"] = model
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
Args:
language: The language to use for speech recognition.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram STT service.
@@ -215,7 +373,6 @@ class DeepgramSTTService(STTService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -268,7 +425,11 @@ class DeepgramSTTService(STTService):
self._on_utterance_end,
)
if not await self._connection.start(options=self._settings, addons=self._addons):
live_options = LiveOptions(
**{**self._settings.live_options.to_dict(), "sample_rate": self.sample_rate}
)
if not await self._connection.start(options=live_options, addons=self._addons):
await self.push_error(error_msg=f"Unable to connect to Deepgram")
else:
headers = {

114
src/pipecat/services/deepgram/stt_sagemaker.py
View File

@@ -14,7 +14,8 @@ languages, and various Deepgram features.
import asyncio
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Dict, Optional
from loguru import logger
@@ -31,6 +32,8 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
from pipecat.services.deepgram.stt import _DeepgramSTTSettingsBase
from pipecat.services.settings import STTSettings
from pipecat.services.stt_latency import DEEPGRAM_SAGEMAKER_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
@@ -47,6 +50,16 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class DeepgramSageMakerSTTSettings(_DeepgramSTTSettingsBase):
"""Settings for the Deepgram SageMaker STT service.
See ``_DeepgramSTTSettingsBase`` for full documentation.
"""
pass
class DeepgramSageMakerSTTService(STTService):
"""Deepgram speech-to-text service for AWS SageMaker.
@@ -75,6 +88,8 @@ class DeepgramSageMakerSTTService(STTService):
)
"""
_settings: DeepgramSageMakerSTTSettings
def __init__(
self,
*,
@@ -93,19 +108,15 @@ class DeepgramSageMakerSTTService(STTService):
region: AWS region where the endpoint is deployed (e.g., "us-east-2").
sample_rate: Audio sample rate in Hz. If None, uses value from
live_options or defaults to the value from StartFrame.
live_options: Deepgram LiveOptions for detailed configuration. If None,
uses sensible defaults (nova-3 model, English, interim results enabled).
live_options: Deepgram LiveOptions configuration. Treated as a
delta from a set of sensible defaults — only the fields you
set are overridden; all others keep their default values.
ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self._endpoint_name = endpoint_name
self._region = region
# Create default options similar to DeepgramSTTService
default_options = LiveOptions(
encoding="linear16",
language=Language.EN,
@@ -115,21 +126,23 @@ class DeepgramSageMakerSTTService(STTService):
punctuate=True,
)
# Merge with provided options
merged_options = default_options.to_dict()
settings = DeepgramSageMakerSTTSettings(
model=default_options.model,
language=default_options.language,
live_options=default_options,
)
if live_options:
default_model = default_options.model
merged_options.update(live_options.to_dict())
# Handle the "None" string bug from deepgram-sdk
if "model" in merged_options and merged_options["model"] == "None":
merged_options["model"] = default_model
settings._merge_live_options_delta(live_options)
# Convert Language enum to string if needed
if "language" in merged_options and isinstance(merged_options["language"], Language):
merged_options["language"] = merged_options["language"].value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
self.set_model_name(merged_options["model"])
self._settings = merged_options
self._endpoint_name = endpoint_name
self._region = region
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None
@@ -143,35 +156,21 @@ class DeepgramSageMakerSTTService(STTService):
"""
return True
async def set_model(self, model: str):
"""Set the Deepgram model and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and warn about unhandled changes."""
changed = await super()._update_settings(delta)
Disconnects from the current session, updates the model setting, and
establishes a new connection with the updated model.
if not changed:
return changed
Args:
model: The Deepgram model name to use (e.g., "nova-3").
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._settings["model"] = model
await self._disconnect()
await self._connect()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
self._warn_unhandled_updated_settings(changed)
Disconnects from the current session, updates the language setting, and
establishes a new connection with the updated language.
Args:
language: The language to use for speech recognition (e.g., Language.EN,
Language.ES).
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Deepgram SageMaker STT service.
@@ -180,7 +179,6 @@ class DeepgramSageMakerSTTService(STTService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -226,12 +224,13 @@ class DeepgramSageMakerSTTService(STTService):
"""
logger.debug("Connecting to Deepgram on SageMaker...")
# Update sample rate in settings
self._settings["sample_rate"] = self.sample_rate
live_options = LiveOptions(
**{**self._settings.live_options.to_dict(), "sample_rate": self.sample_rate}
)
# Build query string from settings, converting booleans to strings
# Build query string from live_options, converting booleans to strings
query_params = {}
for key, value in self._settings.items():
for key, value in live_options.to_dict().items():
if value is not None:
# Convert boolean values to lowercase strings for Deepgram API
if isinstance(value, bool):
@@ -368,7 +367,6 @@ class DeepgramSageMakerSTTService(STTService):
return
is_final = parsed.get("is_final", False)
speech_final = parsed.get("speech_final", False)
# Extract language if available
language = None
@@ -376,8 +374,12 @@ class DeepgramSageMakerSTTService(STTService):
language = alternatives[0]["languages"][0]
language = Language(language)
if is_final and speech_final:
# Final transcription
if is_final:
# Check if this response is from a finalize() call.
# Only mark as finalized when both we requested it AND Deepgram confirms it.
from_finalize = parsed.get("from_finalize", False)
if from_finalize:
self.confirm_finalize()
await self.push_frame(
TranscriptionFrame(
transcript,
@@ -435,10 +437,12 @@ class DeepgramSageMakerSTTService(STTService):
if isinstance(frame, VADUserStartedSpeakingFrame):
await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
# Send finalize message to Deepgram when user stops speaking
# This tells Deepgram to flush any remaining audio and return final results
# https://developers.deepgram.com/docs/finalize
# Mark that we're awaiting a from_finalize response
self.request_finalize()
if self._client and self._client.is_active:
try:
await self._client.send_json({"type": "Finalize"})
except Exception as e:
logger.warning(f"Error sending Finalize message: {e}")
logger.trace(f"Triggered finalize event on: {frame.name=}, {direction=}")

74
src/pipecat/services/deepgram/tts.py
View File

@@ -11,7 +11,8 @@ for generating speech from text using various voice models.
"""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -29,6 +30,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService, WebsocketTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -43,6 +45,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class DeepgramTTSSettings(TTSSettings):
"""Settings for Deepgram TTS service.
Parameters:
encoding: Audio encoding format (linear16, mulaw, alaw).
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramTTSService(WebsocketTTSService):
"""Deepgram WebSocket-based text-to-speech service.
@@ -51,6 +64,8 @@ class DeepgramTTSService(WebsocketTTSService):
message for conversational AI use cases.
"""
_settings: DeepgramTTSSettings
SUPPORTED_ENCODINGS = ("linear16", "mulaw", "alaw")
def __init__(
@@ -86,15 +101,17 @@ class DeepgramTTSService(WebsocketTTSService):
pause_frame_processing=True,
push_stop_frames=True,
append_trailing_space=True,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._settings = {
"encoding": encoding,
}
self.set_voice(voice)
self._receive_task = None
self._context_id: Optional[str] = None
@@ -166,6 +183,28 @@ class DeepgramTTSService(WebsocketTTSService):
await self._disconnect_websocket()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Args:
delta: A :class:`TTSSettings` (or ``DeepgramTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
# Deepgram uses voice as the model, so keep them in sync for metrics
if "voice" in changed:
self._settings.model = self._settings.voice
self._sync_model_name_to_metrics()
if changed:
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to Deepgram WebSocket API with configured settings."""
try:
@@ -176,8 +215,8 @@ class DeepgramTTSService(WebsocketTTSService):
# Build WebSocket URL with query parameters
params = []
params.append(f"model={self._voice_id}")
params.append(f"encoding={self._settings['encoding']}")
params.append(f"model={self._settings.voice}")
params.append(f"encoding={self._settings.encoding}")
params.append(f"sample_rate={self.sample_rate}")
url = f"{self._base_url}/v1/speak?{'&'.join(params)}"
@@ -330,6 +369,8 @@ class DeepgramHttpTTSService(TTSService):
configurable sample rates and quality settings.
"""
_settings: DeepgramTTSSettings
def __init__(
self,
*,
@@ -352,15 +393,20 @@ class DeepgramHttpTTSService(TTSService):
encoding: Audio encoding format. Defaults to "linear16".
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
super().__init__(
sample_rate=sample_rate,
settings=DeepgramTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = base_url
self._settings = {
"encoding": encoding,
}
self.set_voice(voice)
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
@@ -389,8 +435,8 @@ class DeepgramHttpTTSService(TTSService):
headers = {"Authorization": f"Token {self._api_key}", "Content-Type": "application/json"}
params = {
"model": self._voice_id,
"encoding": self._settings["encoding"],
"model": self._settings.voice,
"encoding": self._settings.encoding,
"sample_rate": self.sample_rate,
"container": "none",
}

360
src/pipecat/services/deepgram/tts_sagemaker.py Normal file
View File

@@ -0,0 +1,360 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Deepgram text-to-speech service for AWS SageMaker.
This module provides a Pipecat TTS service that connects to Deepgram models
deployed on AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for
low-latency real-time speech synthesis with support for interruptions and
streaming audio output.
"""
import asyncio
import json
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pipecat.frames.frames import (
BotStoppedSpeakingFrame,
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
LLMFullResponseEndFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@dataclass
class DeepgramSageMakerTTSSettings(TTSSettings):
"""Settings for Deepgram SageMaker TTS service.
Parameters:
encoding: Audio encoding format (e.g. "linear16").
"""
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class DeepgramSageMakerTTSService(TTSService):
"""Deepgram text-to-speech service for AWS SageMaker.
Provides real-time speech synthesis using Deepgram models deployed on
AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for low-latency
audio generation with support for interruptions via the Clear message.
Requirements:
- AWS credentials configured (via environment variables, AWS CLI, or instance metadata)
- A deployed SageMaker endpoint with Deepgram TTS model: https://developers.deepgram.com/docs/deploy-amazon-sagemaker
- ``pipecat-ai[sagemaker]`` installed
Example::
tts = DeepgramSageMakerTTSService(
endpoint_name="my-deepgram-tts-endpoint",
region="us-east-2",
voice="aura-2-helena-en",
)
"""
_settings: DeepgramSageMakerTTSSettings
def __init__(
self,
*,
endpoint_name: str,
region: str,
voice: str = "aura-2-helena-en",
sample_rate: Optional[int] = None,
encoding: str = "linear16",
**kwargs,
):
"""Initialize the Deepgram SageMaker TTS service.
Args:
endpoint_name: Name of the SageMaker endpoint with Deepgram TTS model
deployed (e.g., "my-deepgram-tts-endpoint").
region: AWS region where the endpoint is deployed (e.g., "us-east-2").
voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en".
sample_rate: Audio sample rate in Hz. If None, uses the value from StartFrame.
encoding: Audio encoding format. Defaults to "linear16".
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(
sample_rate=sample_rate,
push_stop_frames=True,
pause_frame_processing=True,
append_trailing_space=True,
settings=DeepgramSageMakerTTSSettings(
model=voice,
voice=voice,
language=None,
encoding=encoding,
),
**kwargs,
)
self._endpoint_name = endpoint_name
self._region = region
self._client: Optional[SageMakerBidiClient] = None
self._response_task: Optional[asyncio.Task] = None
self._context_id: Optional[str] = None
self._ttfb_started: bool = False
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Deepgram SageMaker TTS service supports metrics generation.
"""
return True
async def start(self, frame: StartFrame):
"""Start the Deepgram SageMaker TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
await self._connect()
async def stop(self, frame: EndFrame):
"""Stop the Deepgram SageMaker TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
"""Cancel the Deepgram SageMaker TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame)
await self._disconnect()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with special handling for LLM response end.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction)
if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
await self.flush_audio()
elif isinstance(frame, BotStoppedSpeakingFrame):
self._ttfb_started = False
async def _connect(self):
"""Connect to the SageMaker endpoint and start the BiDi session.
Builds the Deepgram TTS query string, creates the BiDi client,
starts the streaming session, and launches a background task for processing
responses.
"""
logger.debug("Connecting to Deepgram TTS on SageMaker...")
query_string = (
f"model={self._settings.voice}&encoding={self._settings.encoding}"
f"&sample_rate={self.sample_rate}"
)
self._client = SageMakerBidiClient(
endpoint_name=self._endpoint_name,
region=self._region,
model_invocation_path="v1/speak",
model_query_string=query_string,
)
try:
await self._client.start_session()
self._response_task = self.create_task(self._process_responses())
logger.debug("Connected to Deepgram TTS on SageMaker")
await self._call_event_handler("on_connected")
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
await self._call_event_handler("on_connection_error", str(e))
async def _disconnect(self):
"""Disconnect from the SageMaker endpoint.
Sends a Close message to Deepgram, cancels the response processing task,
and closes the BiDi session. Safe to call multiple times.
"""
if self._client and self._client.is_active:
logger.debug("Disconnecting from Deepgram TTS on SageMaker...")
try:
await self._client.send_json({"type": "Close"})
except Exception as e:
logger.warning(f"Failed to send Close message: {e}")
if self._response_task and not self._response_task.done():
await self.cancel_task(self._response_task)
await self._client.close_session()
logger.debug("Disconnected from Deepgram TTS on SageMaker")
await self._call_event_handler("on_disconnected")
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if necessary.
Since all settings are part of the SageMaker session query string,
any setting change requires reconnecting to apply the new values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# Deepgram uses voice as the model, so keep them in sync for metrics
if "voice" in changed:
self._settings.model = self._settings.voice
self._sync_model_name_to_metrics()
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def _process_responses(self):
"""Process streaming responses from Deepgram TTS on SageMaker.
Continuously receives responses from the BiDi stream. Attempts to decode
each payload as UTF-8 JSON for control messages (Flushed, Cleared, Metadata,
Warning). If decoding fails, treats the payload as raw audio bytes and pushes
a TTSAudioRawFrame downstream.
"""
try:
while self._client and self._client.is_active:
result = await self._client.receive_response()
if result is None:
break
if hasattr(result, "value") and hasattr(result.value, "bytes_"):
if result.value.bytes_:
payload = result.value.bytes_
# Try to decode as JSON control message first
try:
response_data = payload.decode("utf-8")
parsed = json.loads(response_data)
msg_type = parsed.get("type")
if msg_type == "Metadata":
logger.trace(f"Received metadata: {parsed}")
elif msg_type == "Flushed":
logger.trace(f"Received Flushed: {parsed}")
elif msg_type == "Cleared":
logger.trace(f"Received Cleared: {parsed}")
elif msg_type == "Warning":
logger.warning(
f"{self} warning: "
f"{parsed.get('description', 'Unknown warning')}"
)
else:
logger.debug(f"Received unknown message type: {parsed}")
except (UnicodeDecodeError, json.JSONDecodeError):
# Not JSON — treat as raw audio bytes
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
payload,
self.sample_rate,
1,
context_id=self._context_id,
)
await self.push_frame(frame)
except asyncio.CancelledError:
logger.debug("TTS response processor cancelled")
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
finally:
logger.debug("TTS response processor stopped")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by sending Clear message to Deepgram.
The Clear message will clear Deepgram's internal text buffer and stop
sending audio, allowing for a new response to be generated.
"""
await super()._handle_interruption(frame, direction)
self._ttfb_started = False
if self._client and self._client.is_active:
try:
await self._client.send_json({"type": "Clear"})
except Exception as e:
logger.error(f"{self} error sending Clear message: {e}")
async def flush_audio(self):
"""Flush any pending audio synthesis by sending Flush command.
This should be called when the LLM finishes a complete response to force
generation of audio from Deepgram's internal text buffer.
"""
if self._client and self._client.is_active:
try:
await self._client.send_json({"type": "Flush"})
except Exception as e:
logger.error(f"{self} error sending Flush message: {e}")
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Deepgram TTS on SageMaker.
Args:
text: The text to synthesize into speech.
context_id: The context ID for tracking audio frames.
Yields:
Frame: TTSStartedFrame, then None (audio comes asynchronously via
the response processor).
"""
logger.debug(f"{self}: Generating TTS [{text}]")
try:
if not self._ttfb_started:
await self.start_ttfb_metrics()
self._ttfb_started = True
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame(context_id=context_id)
self._context_id = context_id
await self._client.send_json({"type": "Speak", "text": text})
yield None
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")

14
src/pipecat/services/deepseek/llm.py
View File

@@ -65,18 +65,18 @@ class DeepSeekLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

242
src/pipecat/services/elevenlabs/stt.py
View File

@@ -11,11 +11,13 @@ using segmented audio processing. The service uploads audio files and receives
transcription results directly.
"""
import asyncio
import base64
import io
import json
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, Optional
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -33,6 +35,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import ELEVENLABS_REALTIME_TTFS_P99, ELEVENLABS_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -167,6 +170,51 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
class CommitStrategy(str, Enum):
"""Commit strategies for transcript segmentation."""
MANUAL = "manual"
VAD = "vad"
@dataclass
class ElevenLabsSTTSettings(STTSettings):
"""Settings for the ElevenLabs file-based STT service.
Parameters:
tag_audio_events: Whether to include audio event tags in transcription.
"""
tag_audio_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class ElevenLabsRealtimeSTTSettings(STTSettings):
"""Settings for the ElevenLabs Realtime STT service.
See ``ElevenLabsRealtimeSTTService.InputParams`` for detailed descriptions.
Parameters:
commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD).
vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0).
vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive).
min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms).
min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms).
include_timestamps: Whether to include word-level timestamps in transcripts.
enable_logging: Whether to enable logging on ElevenLabs' side.
include_language_detection: Whether to include language detection in transcripts.
"""
commit_strategy: CommitStrategy | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_silence_threshold_secs: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_speech_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_silence_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_timestamps: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_logging: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_language_detection: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class ElevenLabsSTTService(SegmentedSTTService):
"""Speech-to-text service using ElevenLabs' file-based API.
@@ -175,6 +223,8 @@ class ElevenLabsSTTService(SegmentedSTTService):
The service uploads audio files to ElevenLabs and receives transcription results directly.
"""
_settings: ElevenLabsSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for ElevenLabs STT API.
@@ -211,25 +261,24 @@ class ElevenLabsSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
params = params or ElevenLabsSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=ElevenLabsSTTSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
tag_audio_events=params.tag_audio_events,
),
**kwargs,
)
params = params or ElevenLabsSTTService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
self._model_id = model
self._tag_audio_events = params.tag_audio_events
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "eng",
}
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -250,28 +299,6 @@ class ElevenLabsSTTService(SegmentedSTTService):
"""
return language_to_elevenlabs_language(language)
async def set_language(self, language: Language):
"""Set the transcription language.
Args:
language: The language to use for speech-to-text transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = self.language_to_service_language(language)
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
Note:
ElevenLabs STT API does not currently support model selection.
This method is provided for interface compatibility.
"""
await super().set_model(model)
logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model")
async def _transcribe_audio(self, audio_data: bytes) -> dict:
"""Upload audio data to ElevenLabs and get transcription result.
@@ -297,9 +324,9 @@ class ElevenLabsSTTService(SegmentedSTTService):
)
# Add required model_id, language_code, and tag_audio_events
data.add_field("model_id", self._model_id)
data.add_field("language_code", self._settings["language"])
data.add_field("tag_audio_events", str(self._tag_audio_events).lower())
data.add_field("model_id", self._settings.model)
data.add_field("language_code", self._settings.language)
data.add_field("tag_audio_events", str(self._settings.tag_audio_events).lower())
async with self._session.post(url, data=data, headers=headers) as response:
if response.status != 200:
@@ -385,13 +412,6 @@ def audio_format_from_sample_rate(sample_rate: int) -> str:
return "pcm_16000"
class CommitStrategy(str, Enum):
"""Commit strategies for transcript segmentation."""
MANUAL = "manual"
VAD = "vad"
class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""Speech-to-text service using ElevenLabs' Realtime WebSocket API.
@@ -404,6 +424,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
commit transcript segments, providing consistency with other STT services.
"""
_settings: ElevenLabsRealtimeSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for ElevenLabs Realtime STT API.
@@ -456,24 +478,35 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to WebsocketSTTService.
"""
params = params or ElevenLabsRealtimeSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=10,
keepalive_interval=5,
settings=ElevenLabsRealtimeSTTSettings(
model=model,
language=params.language_code,
commit_strategy=params.commit_strategy,
vad_silence_threshold_secs=params.vad_silence_threshold_secs,
vad_threshold=params.vad_threshold,
min_speech_duration_ms=params.min_speech_duration_ms,
min_silence_duration_ms=params.min_silence_duration_ms,
include_timestamps=params.include_timestamps,
enable_logging=params.enable_logging,
include_language_detection=params.include_language_detection,
),
**kwargs,
)
params = params or ElevenLabsRealtimeSTTService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._model_id = model
self._params = params
self._audio_format = "" # initialized in start()
self._receive_task = None
self._settings = {"language": params.language_code}
self._connected_event = asyncio.Event()
self._connected_event.set()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -483,42 +516,24 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the transcription language.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if anything changed.
Args:
language: The language to use for speech-to-text transcription.
delta: A :class:`STTSettings` (or ``ElevenLabsRealtimeSTTSettings``) delta.
Note:
Changing language requires reconnecting to the WebSocket.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.info(f"Switching STT language to: [{language}]")
new_language = (
language_to_elevenlabs_language(language)
if isinstance(language, Language)
else language
)
self._params.language_code = new_language
self._settings["language"] = new_language
# Reconnect with new settings
changed = await super()._update_settings(delta)
if not changed:
return changed
await self._disconnect()
await self._connect()
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
Note:
Changing model requires reconnecting to the WebSocket.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
self._model_id = model
# Reconnect with new settings
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the STT service and establish WebSocket connection.
@@ -566,7 +581,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
# Send commit when user stops speaking (manual commit mode)
if self._params.commit_strategy == CommitStrategy.MANUAL:
if self._settings.commit_strategy == CommitStrategy.MANUAL:
if self._websocket and self._websocket.state is State.OPEN:
try:
commit_message = {
@@ -589,6 +604,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
Yields:
None - transcription results are handled via WebSocket responses.
"""
# Wait for any in-flight _connect() to finish before checking state
await self._connected_event.wait()
# Reconnect if connection is closed
if not self._websocket or self._websocket.state is State.CLOSED:
await self._connect()
@@ -613,12 +631,18 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
async def _connect(self):
"""Establish WebSocket connection to ElevenLabs Realtime STT."""
await self._connect_websocket()
self._connected_event.clear()
try:
await self._connect_websocket()
await super()._connect()
await super()._connect()
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(
self._receive_task_handler(self._report_error)
)
finally:
self._connected_event.set()
async def _disconnect(self):
"""Close WebSocket connection and cleanup tasks."""
@@ -654,38 +678,42 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
logger.debug("Connecting to ElevenLabs Realtime STT")
# Build query parameters
params = [f"model_id={self._model_id}"]
params = [f"model_id={self._settings.model}"]
if self._params.language_code:
params.append(f"language_code={self._params.language_code}")
if self._settings.language:
params.append(f"language_code={self._settings.language}")
params.append(f"audio_format={self._audio_format}")
params.append(f"commit_strategy={self._params.commit_strategy.value}")
params.append(f"commit_strategy={self._settings.commit_strategy.value}")
# Add optional parameters
if self._params.include_timestamps:
params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}")
if self._params.enable_logging:
params.append(f"enable_logging={str(self._params.enable_logging).lower()}")
if self._params.include_language_detection:
if self._settings.include_timestamps:
params.append(
f"include_language_detection={str(self._params.include_language_detection).lower()}"
f"include_timestamps={str(self._settings.include_timestamps).lower()}"
)
if self._settings.enable_logging:
params.append(f"enable_logging={str(self._settings.enable_logging).lower()}")
if self._settings.include_language_detection:
params.append(
f"include_language_detection={str(self._settings.include_language_detection).lower()}"
)
# Add VAD parameters if using VAD commit strategy and values are specified
if self._params.commit_strategy == CommitStrategy.VAD:
if self._params.vad_silence_threshold_secs is not None:
if self._settings.commit_strategy == CommitStrategy.VAD:
if self._settings.vad_silence_threshold_secs is not None:
params.append(
f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}"
f"vad_silence_threshold_secs={self._settings.vad_silence_threshold_secs}"
)
if self._settings.vad_threshold is not None:
params.append(f"vad_threshold={self._settings.vad_threshold}")
if self._settings.min_speech_duration_ms is not None:
params.append(f"min_speech_duration_ms={self._settings.min_speech_duration_ms}")
if self._settings.min_silence_duration_ms is not None:
params.append(
f"min_silence_duration_ms={self._settings.min_silence_duration_ms}"
)
if self._params.vad_threshold is not None:
params.append(f"vad_threshold={self._params.vad_threshold}")
if self._params.min_speech_duration_ms is not None:
params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}")
if self._params.min_silence_duration_ms is not None:
params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}")
ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}"
@@ -817,7 +845,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
"""
# If timestamps are enabled, skip this message and wait for the
# committed_transcript_with_timestamps message which contains all the data
if self._params.include_timestamps:
if self._settings.include_timestamps:
return
text = data.get("text", "").strip()
@@ -833,6 +861,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await self._handle_transcription(text, True, language)
finalized = self._settings.commit_strategy == CommitStrategy.MANUAL
await self.push_frame(
TranscriptionFrame(
text,
@@ -840,6 +870,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
time_now_iso8601(),
language,
result=data,
finalized=finalized,
)
)
@@ -874,6 +905,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await self._handle_transcription(text, True, language)
finalized = self._settings.commit_strategy == CommitStrategy.MANUAL
# This message is sent after committed_transcript when include_timestamps=true.
# It contains the full transcript data including text and word-level timestamps.
await self.push_frame(
@@ -883,5 +916,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
time_now_iso8601(),
language,
result=data,
finalized=finalized,
)
)

437
src/pipecat/services/elevenlabs/tts.py
View File

@@ -13,8 +13,19 @@ with support for streaming audio, word timestamps, and voice customization.
import asyncio
import base64
import json
import uuid
from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union
from dataclasses import dataclass, field
from typing import (
Any,
AsyncGenerator,
ClassVar,
Dict,
List,
Literal,
Mapping,
Optional,
Tuple,
Union,
)
import aiohttp
from loguru import logger
@@ -33,9 +44,11 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import (
AudioContextWordTTSService,
WordTTSService,
AudioContextTTSService,
TextAggregationMode,
TTSService,
)
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -137,12 +150,12 @@ def output_format_from_sample_rate(sample_rate: int) -> str:
def build_elevenlabs_voice_settings(
settings: Dict[str, Any],
settings: Union[Dict[str, Any], "TTSSettings"],
) -> Optional[Dict[str, Union[float, bool]]]:
"""Build voice settings dictionary for ElevenLabs based on provided settings.
Args:
settings: Dictionary containing voice settings parameters.
settings: Dictionary or settings containing voice settings parameters.
Returns:
Dictionary of voice settings or None if no valid settings are provided.
@@ -151,8 +164,11 @@ def build_elevenlabs_voice_settings(
voice_settings = {}
for key in voice_setting_keys:
if key in settings and settings[key] is not None:
voice_settings[key] = settings[key]
val = (
getattr(settings, key, None) if isinstance(settings, TTSSettings) else settings.get(key)
)
if val is not None:
voice_settings[key] = val
return voice_settings or None
@@ -169,6 +185,79 @@ class PronunciationDictionaryLocator(BaseModel):
version_id: str
@dataclass
class ElevenLabsTTSSettings(TTSSettings):
"""Settings for the ElevenLabs WebSocket TTS service.
Fields that appear in the WebSocket URL (``voice``, ``model``,
``language``) require a full reconnect when changed. Fields that
affect the voice character (``stability``, ``similarity_boost``,
``style``, ``use_speaker_boost``, ``speed``) can be applied by closing
the current audio context so a new one is opened with updated settings.
Parameters:
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.7 to 1.2).
auto_mode: Whether to enable automatic mode optimization.
enable_ssml_parsing: Whether to parse SSML tags in text.
enable_logging: Whether to enable ElevenLabs logging.
apply_text_normalization: Text normalization mode ("auto", "on", "off").
"""
stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
auto_mode: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_ssml_parsing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_logging: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
#: Fields in the WS URL — changing any of these requires a reconnect.
URL_FIELDS: ClassVar[frozenset[str]] = frozenset({"voice", "model", "language"})
#: Fields affecting voice character — changing these requires closing the
#: current audio context so the next one picks up new settings.
VOICE_SETTINGS_FIELDS: ClassVar[frozenset[str]] = frozenset(
{"stability", "similarity_boost", "style", "use_speaker_boost", "speed"}
)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@dataclass
class ElevenLabsHttpTTSSettings(TTSSettings):
"""Settings for the ElevenLabs HTTP TTS service.
Parameters:
optimize_streaming_latency: Latency optimization level (0-4).
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.25 to 4.0).
apply_text_normalization: Text normalization mode ("auto", "on", "off").
"""
optimize_streaming_latency: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
stability: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
similarity_boost: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
style: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_speaker_boost: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: Literal["auto", "on", "off"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
def calculate_word_times(
alignment_info: Mapping[str, Any],
cumulative_time: float,
@@ -229,7 +318,7 @@ def calculate_word_times(
return (word_times, new_partial_word, new_partial_word_start_time)
class ElevenLabsTTSService(AudioContextWordTTSService):
class ElevenLabsTTSService(AudioContextTTSService):
"""ElevenLabs WebSocket-based TTS service with word timestamps.
Provides real-time text-to-speech using ElevenLabs' WebSocket streaming API.
@@ -237,6 +326,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
customization options including stability, similarity boost, and speed controls.
"""
_settings: ElevenLabsTTSSettings
class InputParams(BaseModel):
"""Input parameters for ElevenLabs TTS configuration.
@@ -275,7 +366,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
url: str = "wss://api.elevenlabs.io",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize the ElevenLabs TTS service.
@@ -287,13 +379,20 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
url: WebSocket URL for ElevenLabs TTS API.
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to the parent service.
"""
# Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
# 3 model, and it's worth it for the better audio quality.
# By default, we aggregate sentences before sending to TTS. This adds
# ~200-300ms of latency per sentence (waiting for the sentence-ending
# punctuation token from the LLM). Setting
# text_aggregation_mode=TextAggregationMode.TOKEN streams tokens
# directly. To use this mode, you must set auto_mode=False. This
# eliminates aggregation time, but slows down ElevenLabs.
#
# We also don't want to automatically push LLM response text frames,
# because the context aggregators will add them to the LLM context even
@@ -304,35 +403,38 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
# Finally, ElevenLabs doesn't provide information on when the bot stops
# speaking for a while, so we want the parent class to send TTSStopFrame
# after a short period not receiving any audio.
params = params or ElevenLabsTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=ElevenLabsTTSSettings(
model=model,
voice=voice_id,
language=(
self.language_to_service_language(params.language) if params.language else None
),
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
auto_mode=str(params.auto_mode).lower(),
enable_ssml_parsing=params.enable_ssml_parsing,
enable_logging=params.enable_logging,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs,
)
params = params or ElevenLabsTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else None,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
"speed": params.speed,
"auto_mode": str(params.auto_mode).lower(),
"enable_ssml_parsing": params.enable_ssml_parsing,
"enable_logging": params.enable_logging,
"apply_text_normalization": params.apply_text_normalization,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
@@ -343,7 +445,6 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
self._partial_word_start_time = 0.0
# Context management for v1 multi API
self._context_id = None
self._receive_task = None
self._keepalive_task = None
@@ -367,62 +468,74 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
return language_to_elevenlabs_language(language)
def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings)
ts = self._settings
voice_setting_keys = [
"stability",
"similarity_boost",
"style",
"use_speaker_boost",
"speed",
]
voice_settings = {}
for key in voice_setting_keys:
val = getattr(ts, key, None)
if val is not None:
voice_settings[key] = val
return voice_settings or None
async def set_model(self, model: str):
"""Set the TTS model and reconnect.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta, reconnecting as needed.
Uses the declarative ``URL_FIELDS`` and ``VOICE_SETTINGS_FIELDS``
sets on :class:`ElevenLabsTTSSettings` to decide whether to
reconnect the WebSocket or close the current audio context.
Args:
model: The model name to use for synthesis.
delta: A :class:`TTSSettings` (or ``ElevenLabsTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
changed = await super()._update_settings(delta)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice, model, or language changed."""
# Track previous values for settings that require reconnection
prev_voice = self._voice_id
prev_model = self.model_name
prev_language = self._settings.get("language")
# Create snapshot of current voice settings to detect changes after update
prev_voice_settings = self._voice_settings.copy() if self._voice_settings else None
if not changed:
return changed
await super()._update_settings(settings)
# Update voice settings for the next context creation
# Rebuild voice settings for next context
self._voice_settings = self._set_voice_settings()
# Check if URL-level settings changed (these require reconnection)
url_changed = (
prev_voice != self._voice_id
or prev_model != self.model_name
or prev_language != self._settings.get("language")
)
# Check if only voice settings changed (speed, stability, etc.)
voice_settings_changed = prev_voice_settings != self._voice_settings
url_changed = bool(changed.keys() & ElevenLabsTTSSettings.URL_FIELDS)
voice_settings_changed = bool(changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS)
if url_changed:
# These settings are in the WebSocket URL, so we need to reconnect
logger.debug(
f"URL-level setting changed (voice/model/language), reconnecting WebSocket"
f"URL-level setting changed ({changed.keys() & ElevenLabsTTSSettings.URL_FIELDS}), "
f"reconnecting WebSocket"
)
await self._disconnect()
await self._connect()
elif voice_settings_changed and self._context_id:
# Voice settings can be updated by closing current context
# so new one gets created with updated voice settings
logger.debug(f"Voice settings changed, closing current context to apply changes")
elif voice_settings_changed and self.has_active_audio_context():
logger.debug(
f"Voice settings changed ({changed.keys() & ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS}), "
f"closing current context to apply changes"
)
context_id = self.get_active_audio_context_id()
try:
if self._websocket:
await self._websocket.send(
json.dumps({"context_id": self._context_id, "close_context": True})
json.dumps({"context_id": context_id, "close_context": True})
)
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
self._context_id = None
self.reset_active_audio_context()
if not url_changed:
# Reconnect applies all settings; only warn about fields not handled
# by voice settings or URL changes.
handled = ElevenLabsTTSSettings.URL_FIELDS | ElevenLabsTTSSettings.VOICE_SETTINGS_FIELDS
self._warn_unhandled_updated_settings(changed.keys() - handled)
return changed
async def start(self, frame: StartFrame):
"""Start the ElevenLabs TTS service.
@@ -454,10 +567,11 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
async def flush_audio(self):
"""Flush any pending audio and finalize the current context."""
if not self._context_id or not self._websocket:
context_id = self.get_active_audio_context_id()
if not context_id or not self._websocket:
return
logger.trace(f"{self}: flushing audio")
msg = {"context_id": self._context_id, "flush": True}
msg = {"context_id": context_id, "flush": True}
await self._websocket.send(json.dumps(msg))
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
@@ -470,7 +584,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await super().push_frame(frame, direction)
if isinstance(frame, (TTSStoppedFrame, InterruptionFrame)):
if isinstance(frame, TTSStoppedFrame):
await self.add_word_timestamps([("Reset", 0)], self._context_id)
await self.add_word_timestamps([("Reset", 0)], self.get_active_audio_context_id())
async def _connect(self):
await super()._connect()
@@ -503,22 +617,22 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
logger.debug("Connecting to ElevenLabs")
voice_id = self._voice_id
model = self.model_name
voice_id = self._settings.voice
model = self._settings.model
output_format = self._output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings.auto_mode}"
if self._settings["enable_ssml_parsing"]:
url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}"
if self._settings.enable_ssml_parsing:
url += f"&enable_ssml_parsing={self._settings.enable_ssml_parsing}"
if self._settings["enable_logging"]:
url += f"&enable_logging={self._settings['enable_logging']}"
if self._settings.enable_logging:
url += f"&enable_logging={self._settings.enable_logging}"
if self._settings["apply_text_normalization"] is not None:
url += f"&apply_text_normalization={self._settings['apply_text_normalization']}"
if self._settings.apply_text_normalization is not None:
url += f"&apply_text_normalization={self._settings.apply_text_normalization}"
# Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS
language = self._settings["language"]
language = self._settings.language
if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None:
url += f"&language_code={language}"
logger.debug(f"Using language code: {language}")
@@ -545,14 +659,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
if self._websocket:
logger.debug("Disconnecting from ElevenLabs")
# Close all contexts and the socket
if self._context_id:
if self.has_active_audio_context():
await self._websocket.send(json.dumps({"close_socket": True}))
await self._websocket.close()
logger.debug("Disconnected from ElevenLabs")
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
finally:
self._context_id = None
await self.remove_active_audio_context()
self._websocket = None
await self._call_event_handler("on_disconnected")
@@ -561,13 +675,11 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by closing the current context."""
await super()._handle_interruption(frame, direction)
# Close the current context when interrupted without closing the websocket
if self._context_id and self._websocket:
logger.trace(f"Closing context {self._context_id} due to interruption")
async def _close_context(self, context_id: str):
# ElevenLabs requires that Pipecat explicitly closes contexts to free
# server-side resources, both on interruption and on normal completion.
if context_id and self._websocket:
logger.trace(f"{self}: Closing context {context_id}")
try:
# ElevenLabs requires that Pipecat manages the contexts and closes them
# when they're not longer in use. Since an InterruptionFrame is pushed
@@ -576,13 +688,25 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
# Note: We do not need to call remove_audio_context here, as the context is
# automatically reset when super ()._handle_interruption is called.
await self._websocket.send(
json.dumps({"context_id": self._context_id, "close_context": True})
json.dumps({"context_id": context_id, "close_context": True})
)
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
self._context_id = None
self._partial_word = ""
self._partial_word_start_time = 0.0
self._partial_word = ""
self._partial_word_start_time = 0.0
async def on_audio_context_interrupted(self, context_id: str):
"""Close the ElevenLabs context when the bot is interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Close the ElevenLabs context after all audio has been played.
ElevenLabs does not send a server-side signal when a context is
exhausted, so Pipecat must explicitly close it with
``close_context: True`` to free server-side resources.
"""
await self._close_context(context_id)
async def _receive_messages(self):
"""Handle incoming WebSocket messages from ElevenLabs."""
@@ -600,11 +724,11 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
# Check if this message belongs to the current context.
if not self.audio_context_available(received_ctx_id):
if self._context_id == received_ctx_id:
if self.get_active_audio_context_id() == received_ctx_id:
logger.debug(
f"Received a delayed message, recreating the context: {self._context_id}"
f"Received a delayed message, recreating the context: {received_ctx_id}"
)
await self.create_audio_context(self._context_id)
await self.create_audio_context(received_ctx_id)
else:
# This can happen if a message is received _after_ we have closed a context
# due to user interruption but _before_ the `isFinal` message for the context
@@ -657,13 +781,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await asyncio.sleep(KEEPALIVE_SLEEP)
try:
if self._websocket and self._websocket.state is State.OPEN:
if self._context_id:
context_id = self.get_active_audio_context_id()
if context_id:
# Send keepalive with context ID to keep the connection alive
keepalive_message = {
"text": "",
"context_id": self._context_id,
"context_id": context_id,
}
logger.trace(f"Sending keepalive for context {self._context_id}")
logger.trace(f"Sending keepalive for context {context_id}")
else:
# It's possible to have a user interruption which clears the context
# without generating a new TTS response. In this case, we'll just send
@@ -677,24 +802,11 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
async def _send_text(self, text: str):
"""Send text to the WebSocket for synthesis."""
if self._websocket and self._context_id:
msg = {"text": text, "context_id": self._context_id}
context_id = self.get_active_audio_context_id()
if self._websocket and context_id:
msg = {"text": text, "context_id": context_id}
await self._websocket.send(json.dumps(msg))
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request in case we don't have one already in progress.
Returns:
A unique string identifier for the TTS context.
"""
# If a context ID does not exist, create a new one.
# If an ID exists, continue using the current ID.
# When interruptions happens, user speech results in
# an interruption, which resets the context ID.
if not self._context_id:
return str(uuid.uuid4())
return self._context_id
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using ElevenLabs' streaming WebSocket API.
@@ -713,19 +825,18 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await self._connect()
try:
if not self._context_id:
if not self.has_active_audio_context():
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
self._context_id = context_id
self._cumulative_time = 0
self._partial_word = ""
self._partial_word_start_time = 0.0
if not self.audio_context_available(self._context_id):
await self.create_audio_context(self._context_id)
if not self.audio_context_available(context_id):
await self.create_audio_context(context_id)
# Initialize context with voice settings and pronunciation dictionaries
msg = {"text": " ", "context_id": self._context_id}
msg = {"text": " ", "context_id": context_id}
if self._voice_settings:
msg["voice_settings"] = self._voice_settings
if self._pronunciation_dictionary_locators:
@@ -734,7 +845,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
for locator in self._pronunciation_dictionary_locators
]
await self._websocket.send(json.dumps(msg))
logger.trace(f"Created new context {self._context_id}")
logger.trace(f"Created new context {context_id}")
await self._send_text(text)
await self.start_tts_usage_metrics(text)
@@ -747,7 +858,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
yield ErrorFrame(error=f"Unknown error occurred: {e}")
class ElevenLabsHttpTTSService(WordTTSService):
class ElevenLabsHttpTTSService(TTSService):
"""ElevenLabs HTTP-based TTS service with word timestamps.
Provides text-to-speech using ElevenLabs' HTTP streaming API for simpler,
@@ -755,6 +866,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
connection is not required or desired.
"""
_settings: ElevenLabsHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for ElevenLabs HTTP TTS configuration.
@@ -790,7 +903,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
base_url: str = "https://api.elevenlabs.io",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize the ElevenLabs HTTP TTS service.
@@ -803,38 +917,44 @@ class ElevenLabsHttpTTSService(WordTTSService):
base_url: Base URL for ElevenLabs HTTP API.
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to the parent service.
"""
params = params or ElevenLabsHttpTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=ElevenLabsHttpTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(params.language)
if params.language
else None,
optimize_streaming_latency=params.optimize_streaming_latency,
stability=params.stability,
similarity_boost=params.similarity_boost,
style=params.style,
use_speaker_boost=params.use_speaker_boost,
speed=params.speed,
apply_text_normalization=params.apply_text_normalization,
),
**kwargs,
)
params = params or ElevenLabsHttpTTSService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._params = params
self._session = aiohttp_session
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
"speed": params.speed,
"apply_text_normalization": params.apply_text_normalization,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
self._pronunciation_dictionary_locators = params.pronunciation_dictionary_locators
@@ -871,10 +991,19 @@ class ElevenLabsHttpTTSService(WordTTSService):
def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings)
async def _update_settings(self, settings: Mapping[str, Any]):
await super()._update_settings(settings)
# Update voice settings for the next context creation
self._voice_settings = self._set_voice_settings()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and rebuild voice settings.
Args:
delta: A :class:`TTSSettings` (or ``ElevenLabsHttpTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if changed:
self._voice_settings = self._set_voice_settings()
return changed
def _reset_state(self):
"""Reset internal state variables."""
@@ -992,11 +1121,11 @@ class ElevenLabsHttpTTSService(WordTTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
# Use the with-timestamps endpoint
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream/with-timestamps"
url = f"{self._base_url}/v1/text-to-speech/{self._settings.voice}/stream/with-timestamps"
payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = {
"text": text,
"model_id": self._model_name,
"model_id": self._settings.model,
}
# Include previous text as context if available
@@ -1011,11 +1140,11 @@ class ElevenLabsHttpTTSService(WordTTSService):
locator.model_dump() for locator in self._pronunciation_dictionary_locators
]
if self._settings["apply_text_normalization"] is not None:
payload["apply_text_normalization"] = self._settings["apply_text_normalization"]
if self._settings.apply_text_normalization is not None:
payload["apply_text_normalization"] = self._settings.apply_text_normalization
language = self._settings["language"]
if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language:
language = self._settings.language
if self._settings.model in ELEVENLABS_MULTILINGUAL_MODELS and language:
payload["language_code"] = language
logger.debug(f"Using language code: {language}")
elif language:
@@ -1032,8 +1161,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
params = {
"output_format": self._output_format,
}
if self._settings["optimize_streaming_latency"] is not None:
params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"]
if self._settings.optimize_streaming_latency is not None:
params["optimize_streaming_latency"] = self._settings.optimize_streaming_latency
try:
await self.start_ttfb_metrics()

16
src/pipecat/services/fal/image.py
View File

@@ -13,6 +13,7 @@ for creating images from text prompts using various AI models.
import asyncio
import io
import os
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Optional, Union
import aiohttp
@@ -22,6 +23,7 @@ from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try:
import fal_client
@@ -31,6 +33,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class FalImageGenSettings(ImageGenSettings):
"""Settings for the Fal image generation service.
Parameters:
model: Fal.ai model identifier.
"""
class FalImageGenService(ImageGenService):
"""Fal's image generation service.
@@ -77,8 +88,7 @@ class FalImageGenService(ImageGenService):
key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable.
**kwargs: Additional arguments passed to parent ImageGenService.
"""
super().__init__(**kwargs)
self.set_model_name(model)
super().__init__(settings=FalImageGenSettings(model=model), **kwargs)
self._params = params
self._aiohttp_session = aiohttp_session
if key:
@@ -103,7 +113,7 @@ class FalImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
response = await fal_client.run_async(
self.model_name,
self._settings.model,
arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)},
)

67
src/pipecat/services/fal/stt.py
View File

@@ -11,12 +11,14 @@ transcription using segmented audio processing.
"""
import os
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import FAL_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -146,6 +148,22 @@ def language_to_fal_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class FalSTTSettings(STTSettings):
"""Settings for the Fal Wizper STT service.
Parameters:
task: Task to perform ('transcribe' or 'translate'). Defaults to
'transcribe'.
chunk_level: Level of chunking ('segment'). Defaults to 'segment'.
version: Version of Wizper model to use. Defaults to '3'.
"""
task: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
chunk_level: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
version: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class FalSTTService(SegmentedSTTService):
"""Speech-to-text service using Fal's Wizper API.
@@ -153,6 +171,8 @@ class FalSTTService(SegmentedSTTService):
segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
"""
_settings: FalSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Fal's Wizper API.
@@ -187,14 +207,23 @@ class FalSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
params = params or FalSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=FalSTTSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en",
task=params.task,
chunk_level=params.chunk_level,
version=params.version,
),
**kwargs,
)
params = params or FalSTTService.InputParams()
if api_key:
os.environ["FAL_KEY"] = api_key
elif "FAL_KEY" not in os.environ:
@@ -203,14 +232,6 @@ class FalSTTService(SegmentedSTTService):
)
self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY"))
self._settings = {
"task": params.task,
"language": self.language_to_service_language(params.language)
if params.language
else "en",
"chunk_level": params.chunk_level,
"version": params.version,
}
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -231,24 +252,6 @@ class FalSTTService(SegmentedSTTService):
"""
return language_to_fal_language(language)
async def set_language(self, language: Language):
"""Set the transcription language.
Args:
language: The language to use for speech-to-text transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = self.language_to_service_language(language)
async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
"""
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[str] = None
@@ -276,19 +279,19 @@ class FalSTTService(SegmentedSTTService):
data_uri = fal_client.encode(audio, "audio/x-wav")
response = await self._fal_client.run(
"fal-ai/wizper",
arguments={"audio_url": data_uri, **self._settings},
arguments={"audio_url": data_uri, **self._settings.given_fields()},
)
if response and "text" in response:
text = response["text"].strip()
if text: # Only yield non-empty text
await self._handle_transcription(text, True, self._settings["language"])
await self._handle_transcription(text, True, self._settings.language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text,
self._user_id,
time_now_iso8601(),
Language(self._settings["language"]),
Language(self._settings.language),
result=response,
)

14
src/pipecat/services/fireworks/llm.py
View File

@@ -66,17 +66,17 @@ class FireworksLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

119
src/pipecat/services/fish/tts.py
View File

@@ -11,7 +11,8 @@ for streaming text-to-speech synthesis with customizable voice parameters.
"""
import uuid
from typing import AsyncGenerator, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Literal, Mapping, Optional
from loguru import logger
from pydantic import BaseModel
@@ -28,6 +29,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -45,6 +47,41 @@ except ModuleNotFoundError as e:
FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"]
@dataclass
class FishAudioTTSSettings(TTSSettings):
"""Settings for Fish Audio TTS service.
Parameters:
fish_sample_rate: Audio sample rate sent to the API.
latency: Latency mode ("normal" or "balanced"). Defaults to "normal".
format: Audio output format.
normalize: Whether to normalize audio output. Defaults to True.
prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0.
prosody_volume: Volume adjustment in dB. Defaults to 0.
reference_id: Reference ID of the voice model.
"""
fish_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
latency: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
normalize: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prosody_volume: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
reference_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "fish_sample_rate"}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "FishAudioTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``prosody``."""
flat = dict(settings)
nested = flat.pop("prosody", None)
if isinstance(nested, dict):
flat.setdefault("prosody_speed", nested.get("speed"))
flat.setdefault("prosody_volume", nested.get("volume"))
return super().from_mapping(flat)
class FishAudioTTSService(InterruptibleTTSService):
"""Fish Audio text-to-speech service with WebSocket streaming.
@@ -53,6 +90,8 @@ class FishAudioTTSService(InterruptibleTTSService):
audio generation with interruption handling.
"""
_settings: FishAudioTTSSettings
class InputParams(BaseModel):
"""Input parameters for Fish Audio TTS configuration.
@@ -99,13 +138,6 @@ class FishAudioTTSService(InterruptibleTTSService):
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent service.
"""
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or FishAudioTTSService.InputParams()
# Validation for model and reference_id parameters
@@ -130,26 +162,30 @@ class FishAudioTTSService(InterruptibleTTSService):
)
reference_id = model
super().__init__(
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
settings=FishAudioTTSSettings(
model=model_id,
voice=reference_id,
fish_sample_rate=0,
latency=params.latency,
format=output_format,
normalize=params.normalize,
prosody_speed=params.prosody_speed,
prosody_volume=params.prosody_volume,
reference_id=reference_id,
),
**kwargs,
)
self._api_key = api_key
self._base_url = "wss://api.fish.audio/v1/tts/live"
self._websocket = None
self._receive_task = None
self._request_id = None
self._settings = {
"sample_rate": 0,
"latency": params.latency,
"format": output_format,
"normalize": params.normalize,
"prosody": {
"speed": params.prosody_speed,
"volume": params.prosody_volume,
},
"reference_id": reference_id,
}
self.set_model_name(model_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -158,16 +194,24 @@ class FishAudioTTSService(InterruptibleTTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model and reconnect.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if needed.
Any change to voice or model triggers a WebSocket reconnect.
Args:
model: The model name to use for synthesis.
delta: A :class:`TTSSettings` (or ``FishAudioTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
changed = await super()._update_settings(delta)
if changed:
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the Fish Audio TTS service.
@@ -176,7 +220,7 @@ class FishAudioTTSService(InterruptibleTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.fish_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -221,11 +265,22 @@ class FishAudioTTSService(InterruptibleTTSService):
logger.debug("Connecting to Fish Audio")
headers = {"Authorization": f"Bearer {self._api_key}"}
headers["model"] = self.model_name
headers["model"] = self._settings.model
self._websocket = await websocket_connect(self._base_url, additional_headers=headers)
# Send initial start message with ormsgpack
start_message = {"event": "start", "request": {"text": "", **self._settings}}
request_settings = {
"sample_rate": self._settings.fish_sample_rate,
"latency": self._settings.latency,
"format": self._settings.format,
"normalize": self._settings.normalize,
"prosody": {
"speed": self._settings.prosody_speed,
"volume": self._settings.prosody_volume,
},
"reference_id": self._settings.reference_id,
}
start_message = {"event": "start", "request": {"text": "", **request_settings}}
await self._websocket.send(ormsgpack.packb(start_message))
logger.debug("Sent start message to Fish Audio")

171
src/pipecat/services/gladia/stt.py
View File

@@ -14,6 +14,7 @@ import asyncio
import base64
import json
import warnings
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Literal, Optional
import aiohttp
@@ -31,7 +32,14 @@ from pipecat.frames.frames import (
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.services.gladia.config import GladiaInputParams
from pipecat.services.gladia.config import (
GladiaInputParams,
LanguageConfig,
MessagesConfig,
PreProcessingConfig,
RealtimeProcessingConfig,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GLADIA_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -178,6 +186,43 @@ class _InputParamsDescriptor:
return GladiaInputParams
@dataclass
class GladiaSTTSettings(STTSettings):
"""Settings for Gladia STT service.
Parameters:
encoding: Audio encoding format.
bit_depth: Audio bit depth.
channels: Number of audio channels.
custom_metadata: Additional metadata to include with requests.
endpointing: Silence duration in seconds to mark end of speech.
maximum_duration_without_endpointing: Maximum utterance duration without silence.
language_config: Detailed language configuration.
pre_processing: Audio pre-processing options.
realtime_processing: Real-time processing features.
messages_config: WebSocket message filtering options.
enable_vad: Enable VAD to trigger end of utterance detection.
"""
encoding: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
bit_depth: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
channels: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
custom_metadata: Dict[str, Any] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
endpointing: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
maximum_duration_without_endpointing: int | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
language_config: LanguageConfig | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pre_processing: PreProcessingConfig | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
realtime_processing: RealtimeProcessingConfig | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
messages_config: MessagesConfig | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_vad: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GladiaSTTService(WebsocketSTTService):
"""Speech-to-Text service using Gladia's API.
@@ -191,6 +236,8 @@ class GladiaSTTService(WebsocketSTTService):
Use :class:`~pipecat.services.gladia.config.GladiaInputParams` directly instead.
"""
_settings: GladiaSTTSettings
# Maintain backward compatibility
InputParams = _InputParamsDescriptor()
@@ -231,14 +278,6 @@ class GladiaSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService parent class.
"""
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=20,
keepalive_interval=5,
**kwargs,
)
params = params or GladiaInputParams()
if params.language is not None:
@@ -261,13 +300,40 @@ class GladiaSTTService(WebsocketSTTService):
stacklevel=2,
)
# Resolve deprecated language → language_config at init time
language_config = params.language_config
if not language_config and params.language:
language_code = self.language_to_service_language(params.language)
if language_code:
language_config = LanguageConfig(languages=[language_code], code_switching=False)
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=20,
keepalive_interval=5,
settings=GladiaSTTSettings(
model=model,
language=None,
encoding=params.encoding,
bit_depth=params.bit_depth,
channels=params.channels,
custom_metadata=params.custom_metadata,
endpointing=params.endpointing,
maximum_duration_without_endpointing=params.maximum_duration_without_endpointing,
language_config=language_config,
pre_processing=params.pre_processing,
realtime_processing=params.realtime_processing,
messages_config=params.messages_config,
enable_vad=params.enable_vad,
),
**kwargs,
)
self._api_key = api_key
self._region = region
self._url = url
self.set_model_name(model)
self._params = params
self._receive_task = None
self._settings = {}
# Session management
self._session_url = None
@@ -307,53 +373,43 @@ class GladiaSTTService(WebsocketSTTService):
return language_to_gladia_language(language)
def _prepare_settings(self) -> Dict[str, Any]:
s = self._settings
settings = {
"encoding": self._params.encoding or "wav/pcm",
"bit_depth": self._params.bit_depth or 16,
"encoding": s.encoding or "wav/pcm",
"bit_depth": s.bit_depth or 16,
"sample_rate": self.sample_rate,
"channels": self._params.channels or 1,
"model": self._model_name,
"channels": s.channels or 1,
"model": s.model,
}
# Add custom_metadata if provided
settings["custom_metadata"] = dict(self._params.custom_metadata or {})
settings["custom_metadata"] = dict(s.custom_metadata or {})
settings["custom_metadata"]["pipecat"] = pipecat_version()
# Add endpointing parameters if provided
if self._params.endpointing is not None:
settings["endpointing"] = self._params.endpointing
if self._params.maximum_duration_without_endpointing is not None:
if s.endpointing is not None:
settings["endpointing"] = s.endpointing
if s.maximum_duration_without_endpointing is not None:
settings["maximum_duration_without_endpointing"] = (
self._params.maximum_duration_without_endpointing
s.maximum_duration_without_endpointing
)
# Add language configuration (prioritize language_config over deprecated language)
if self._params.language_config:
settings["language_config"] = self._params.language_config.model_dump(exclude_none=True)
elif self._params.language: # Backward compatibility for deprecated parameter
language_code = self.language_to_service_language(self._params.language)
if language_code:
settings["language_config"] = {
"languages": [language_code],
"code_switching": False,
}
# Add language configuration
if s.language_config:
settings["language_config"] = s.language_config.model_dump(exclude_none=True)
# Add pre_processing configuration if provided
if self._params.pre_processing:
settings["pre_processing"] = self._params.pre_processing.model_dump(exclude_none=True)
if s.pre_processing:
settings["pre_processing"] = s.pre_processing.model_dump(exclude_none=True)
# Add realtime_processing configuration if provided
if self._params.realtime_processing:
settings["realtime_processing"] = self._params.realtime_processing.model_dump(
exclude_none=True
)
if s.realtime_processing:
settings["realtime_processing"] = s.realtime_processing.model_dump(exclude_none=True)
# Add messages_config if provided
if self._params.messages_config:
settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True)
# Store settings for tracing
self._settings = settings
if s.messages_config:
settings["messages_config"] = s.messages_config.model_dump(exclude_none=True)
return settings
@@ -366,6 +422,33 @@ class GladiaSTTService(WebsocketSTTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, delta: GladiaSTTSettings) -> dict[str, Any]:
"""Apply settings delta.
Settings are stored but not applied to the active session.
Args:
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# self._session_url = None
# self._session_id = None
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def stop(self, frame: EndFrame):
"""Stop the Gladia STT websocket connection.
@@ -522,7 +605,7 @@ class GladiaSTTService(WebsocketSTTService):
Broadcasts UserStartedSpeakingFrame and optionally triggers interruption
when VAD is enabled.
"""
if not self._params.enable_vad or self._is_speaking:
if not self._settings.enable_vad or self._is_speaking:
return
logger.debug(f"{self} User started speaking")
@@ -537,7 +620,7 @@ class GladiaSTTService(WebsocketSTTService):
Broadcasts UserStoppedSpeakingFrame when VAD is enabled.
"""
if not self._params.enable_vad or not self._is_speaking:
if not self._settings.enable_vad or not self._is_speaking:
return
self._is_speaking = False
await self.broadcast_frame(UserStoppedSpeakingFrame)

165
src/pipecat/services/google/gemini_live/llm.py
View File

@@ -17,9 +17,9 @@ import io
import time
import uuid
import warnings
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from typing import Any, ClassVar, Dict, List, Optional, Union
from loguru import logger
from PIL import Image
@@ -47,7 +47,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
TranscriptionFrame,
TTSAudioRawFrame,
@@ -77,6 +76,7 @@ from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.string import match_endofsentence
from pipecat.utils.time import time_now_iso8601
@@ -602,6 +602,33 @@ class InputParams(BaseModel):
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
@dataclass
class GeminiLiveLLMSettings(LLMSettings):
"""Settings for Gemini Live LLM services.
Parameters:
modalities: Response modalities.
language: Language for generation.
media_resolution: Media resolution setting.
vad: Voice activity detection parameters.
context_window_compression: Context window compression configuration.
thinking: Thinking configuration.
enable_affective_dialog: Whether to enable affective dialog.
proactivity: Proactivity configuration.
"""
modalities: GeminiModalities | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: Language | str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
media_resolution: GeminiMediaResolution | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad: GeminiVADParams | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
context_window_compression: ContextWindowCompressionParams | dict | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
thinking: ThinkingConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_affective_dialog: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
proactivity: ProactivityConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GeminiLiveLLMService(LLMService):
"""Provides access to Google's Gemini Live API.
@@ -610,6 +637,8 @@ class GeminiLiveLLMService(LLMService):
responses, and tool usage.
"""
_settings: GeminiLiveLLMSettings
# Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter
@@ -666,13 +695,40 @@ class GeminiLiveLLMService(LLMService):
stacklevel=2,
)
super().__init__(base_url=base_url, **kwargs)
params = params or InputParams()
super().__init__(
base_url=base_url,
settings=GeminiLiveLLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
max_tokens=params.max_tokens,
presence_penalty=params.presence_penalty,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
modalities=params.modalities,
language=language_to_gemini_language(params.language)
if params.language
else "en-US",
media_resolution=params.media_resolution,
vad=params.vad,
context_window_compression=params.context_window_compression.model_dump()
if params.context_window_compression
else {},
thinking=params.thinking or {},
enable_affective_dialog=params.enable_affective_dialog or False,
proactivity=params.proactivity or {},
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._last_sent_time = 0
self._base_url = base_url
self.set_model_name(model)
self._voice_id = voice_id
self._language_code = params.language
@@ -714,26 +770,6 @@ class GeminiLiveLLMService(LLMService):
self._consecutive_failures = 0
self._connection_start_time = None
self._settings = {
"frequency_penalty": params.frequency_penalty,
"max_tokens": params.max_tokens,
"presence_penalty": params.presence_penalty,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"modalities": params.modalities,
"language": self._language_code,
"media_resolution": params.media_resolution,
"vad": params.vad,
"context_window_compression": params.context_window_compression.model_dump()
if params.context_window_compression
else {},
"thinking": params.thinking or {},
"enable_affective_dialog": params.enable_affective_dialog or False,
"proactivity": params.proactivity or {},
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self._file_api_base_url = file_api_base_url
self._file_api: Optional[GeminiFileAPI] = None
@@ -776,6 +812,25 @@ class GeminiLiveLLMService(LLMService):
"""
return True
async def _update_settings(self, delta: LLMSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
def set_audio_input_paused(self, paused: bool):
"""Set the audio input pause state.
@@ -798,7 +853,7 @@ class GeminiLiveLLMService(LLMService):
Args:
modalities: The modalities to use for responses.
"""
self._settings["modalities"] = modalities
self._settings.modalities = modalities
def set_language(self, language: Language):
"""Set the language for generation.
@@ -808,7 +863,7 @@ class GeminiLiveLLMService(LLMService):
"""
self._language = language
self._language_code = language_to_gemini_language(language) or "en-US"
self._settings["language"] = self._language_code
self._settings.language = self._language_code
logger.info(f"Set Gemini language to: {self._language_code}")
async def set_context(self, context: OpenAILLMContext):
@@ -866,7 +921,7 @@ class GeminiLiveLLMService(LLMService):
async def _handle_interruption(self):
if self._bot_is_responding:
await self._set_bot_is_responding(False)
if self._settings.get("modalities") == GeminiModalities.AUDIO:
if self._settings.modalities == GeminiModalities.AUDIO:
await self.push_frame(TTSStoppedFrame())
# Do not send LLMFullResponseEndFrame here - an interruption
# already tells the assistant context aggregator that the response
@@ -947,10 +1002,9 @@ class GeminiLiveLLMService(LLMService):
# uses this frame *without* a user context aggregator still works
# (we have an example that does just that, actually).
await self._create_single_response(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
# TODO: implement runtime tool updates for Gemini Live.
pass
else:
await self.push_frame(frame, direction)
@@ -1074,20 +1128,20 @@ class GeminiLiveLLMService(LLMService):
# Assemble basic configuration
config = LiveConnectConfig(
generation_config=GenerationConfig(
frequency_penalty=self._settings["frequency_penalty"],
max_output_tokens=self._settings["max_tokens"],
presence_penalty=self._settings["presence_penalty"],
temperature=self._settings["temperature"],
top_k=self._settings["top_k"],
top_p=self._settings["top_p"],
response_modalities=[Modality(self._settings["modalities"].value)],
frequency_penalty=self._settings.frequency_penalty,
max_output_tokens=self._settings.max_tokens,
presence_penalty=self._settings.presence_penalty,
temperature=self._settings.temperature,
top_k=self._settings.top_k,
top_p=self._settings.top_p,
response_modalities=[Modality(self._settings.modalities.value)],
speech_config=SpeechConfig(
voice_config=VoiceConfig(
prebuilt_voice_config={"voice_name": self._voice_id}
),
language_code=self._settings["language"],
language_code=self._settings.language,
),
media_resolution=MediaResolution(self._settings["media_resolution"].value),
media_resolution=MediaResolution(self._settings.media_resolution.value),
),
input_audio_transcription=AudioTranscriptionConfig(),
output_audio_transcription=AudioTranscriptionConfig(),
@@ -1095,37 +1149,36 @@ class GeminiLiveLLMService(LLMService):
)
# Add context window compression to configuration, if enabled
if self._settings.get("context_window_compression", {}).get("enabled", False):
cwc = self._settings.context_window_compression or {}
if cwc.get("enabled", False):
compression_config = ContextWindowCompressionConfig()
# Add sliding window (always true if compression is enabled)
compression_config.sliding_window = SlidingWindow()
# Add trigger_tokens if specified
trigger_tokens = self._settings.get("context_window_compression", {}).get(
"trigger_tokens"
)
trigger_tokens = cwc.get("trigger_tokens")
if trigger_tokens is not None:
compression_config.trigger_tokens = trigger_tokens
config.context_window_compression = compression_config
# Add thinking configuration to configuration, if provided
if self._settings.get("thinking"):
config.thinking_config = self._settings["thinking"]
if self._settings.thinking:
config.thinking_config = self._settings.thinking
# Add affective dialog setting, if provided
if self._settings.get("enable_affective_dialog", False):
config.enable_affective_dialog = self._settings["enable_affective_dialog"]
if self._settings.enable_affective_dialog:
config.enable_affective_dialog = self._settings.enable_affective_dialog
# Add proactivity configuration to configuration, if provided
if self._settings.get("proactivity"):
config.proactivity = self._settings["proactivity"]
if self._settings.proactivity:
config.proactivity = self._settings.proactivity
# Add VAD configuration to configuration, if provided
if self._settings.get("vad"):
if self._settings.vad:
vad_config = AutomaticActivityDetection()
vad_params = self._settings["vad"]
vad_params = self._settings.vad
has_vad_settings = False
# Only add parameters that are explicitly set
@@ -1183,7 +1236,9 @@ class GeminiLiveLLMService(LLMService):
await self.push_error(error_msg=f"Initialization error: {e}", exception=e)
async def _connection_task_handler(self, config: LiveConnectConfig):
async with self._client.aio.live.connect(model=self._model_name, config=config) as session:
async with self._client.aio.live.connect(
model=self._settings.model, config=config
) as session:
logger.info("Connected to Gemini service")
# Mark connection start time
@@ -1604,7 +1659,7 @@ class GeminiLiveLLMService(LLMService):
text: The transcription text to push
result: Optional LiveServerMessage that triggered this transcription
"""
await self._handle_user_transcription(text, True, self._settings["language"])
await self._handle_user_transcription(text, True, self._settings.language)
await self.push_frame(
TranscriptionFrame(
text=text,

17
src/pipecat/services/google/image.py
View File

@@ -16,6 +16,7 @@ import os
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -25,6 +26,7 @@ from pydantic import BaseModel, Field
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.google.utils import update_google_client_http_options
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
try:
from google import genai
@@ -35,6 +37,15 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class GoogleImageGenSettings(ImageGenSettings):
"""Settings for the Google image generation service.
Parameters:
model: Google Imagen model identifier.
"""
class GoogleImageGenService(ImageGenService):
"""Google AI image generation service using Imagen models.
@@ -72,14 +83,14 @@ class GoogleImageGenService(ImageGenService):
http_options: HTTP options for the client.
**kwargs: Additional arguments passed to the parent ImageGenService.
"""
super().__init__(**kwargs)
self._params = params or GoogleImageGenService.InputParams()
params = params or GoogleImageGenService.InputParams()
super().__init__(settings=GoogleImageGenSettings(model=params.model), **kwargs)
self._params = params
# Add client header
http_options = update_google_client_http_options(http_options)
self._client = genai.Client(api_key=api_key, http_options=http_options)
self.set_model_name(self._params.model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.

163
src/pipecat/services/google/llm.py
View File

@@ -15,8 +15,8 @@ import io
import json
import os
import uuid
from dataclasses import dataclass
from typing import Any, AsyncIterator, Dict, List, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncIterator, ClassVar, Dict, List, Literal, Optional
from loguru import logger
from PIL import Image
@@ -39,7 +39,6 @@ from pipecat.frames.frames import (
LLMThoughtEndFrame,
LLMThoughtStartFrame,
LLMThoughtTextFrame,
LLMUpdateSettingsFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -59,6 +58,7 @@ from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven, is_given
from pipecat.utils.tracing.service_decorators import traced_llm
# Suppress gRPC fork warnings
@@ -673,6 +673,62 @@ class GoogleLLMContext(OpenAILLMContext):
self._messages = [m for m in self._messages if m.parts]
class GoogleThinkingConfig(BaseModel):
"""Configuration for controlling the model's internal "thinking" process used before generating a response.
Gemini 2.5 and 3 series models have this thinking process.
Parameters:
thinking_level: Thinking level for Gemini 3 models.
For Gemini 3 Pro, this can be "low" or "high".
For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high".
If not provided, Gemini 3 models default to "high".
Note: Gemini 2.5 series must use thinking_budget instead.
thinking_budget: Token budget for thinking, for Gemini 2.5 series.
-1 for dynamic thinking (model decides), 0 to disable thinking,
or a specific token count (e.g., 128-32768 for 2.5 Pro).
If not provided, most models today default to dynamic thinking.
See https://ai.google.dev/gemini-api/docs/thinking#set-budget
for default values and allowed ranges.
Note: Gemini 3 models must use thinking_level instead.
include_thoughts: Whether to include thought summaries in the response.
Today's models default to not including thoughts (False).
"""
thinking_budget: Optional[int] = Field(default=None)
# Why `| str` here? To not break compatibility in case Google adds more
# levels in the future.
thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field(
default=None
)
include_thoughts: Optional[bool] = Field(default=None)
@dataclass
class GoogleLLMSettings(LLMSettings):
"""Settings for Google LLM services.
Parameters:
thinking: Thinking configuration.
"""
thinking: GoogleThinkingConfig | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@classmethod
def from_mapping(cls, settings):
"""Convert a plain dict to settings, coercing thinking dicts.
For backward compatibility, a ``thinking`` value that is a plain dict
is converted to a :class:`GoogleThinkingConfig`.
"""
instance = super().from_mapping(settings)
if is_given(instance.thinking) and isinstance(instance.thinking, dict):
instance.thinking = GoogleThinkingConfig(**instance.thinking)
return instance
class GoogleLLMService(LLMService):
"""Google AI (Gemini) LLM service implementation.
@@ -681,40 +737,13 @@ class GoogleLLMService(LLMService):
expected by the Google AI model.
"""
_settings: GoogleLLMSettings
# Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter
class ThinkingConfig(BaseModel):
"""Configuration for controlling the model's internal "thinking" process used before generating a response.
Gemini 2.5 and 3 series models have this thinking process.
Parameters:
thinking_level: Thinking level for Gemini 3 models.
For Gemini 3 Pro, this can be "low" or "high".
For Gemini 3 Flash, this can be "minimal", "low", "medium", or "high".
If not provided, Gemini 3 models default to "high".
Note: Gemini 2.5 series must use thinking_budget instead.
thinking_budget: Token budget for thinking, for Gemini 2.5 series.
-1 for dynamic thinking (model decides), 0 to disable thinking,
or a specific token count (e.g., 128-32768 for 2.5 Pro).
If not provided, most models today default to dynamic thinking.
See https://ai.google.dev/gemini-api/docs/thinking#set-budget
for default values and allowed ranges.
Note: Gemini 3 models must use thinking_level instead.
include_thoughts: Whether to include thought summaries in the response.
Today's models default to not including thoughts (False).
"""
thinking_budget: Optional[int] = Field(default=None)
# Why `| str` here? To not break compatibility in case Google adds more
# levels in the future.
thinking_level: Optional[Literal["low", "high", "medium", "minimal"] | str] = Field(
default=None
)
include_thoughts: Optional[bool] = Field(default=None)
# Backward compatibility: ThinkingConfig used to be defined inline here.
ThinkingConfig = GoogleThinkingConfig
class InputParams(BaseModel):
"""Input parameters for Google AI models.
@@ -737,7 +766,7 @@ class GoogleLLMService(LLMService):
temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
top_k: Optional[int] = Field(default=None, ge=0)
top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0)
thinking: Optional["GoogleLLMService.ThinkingConfig"] = Field(default=None)
thinking: Optional[GoogleThinkingConfig] = Field(default=None)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def __init__(
@@ -764,23 +793,29 @@ class GoogleLLMService(LLMService):
http_options: HTTP options for the client.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
params = params or GoogleLLMService.InputParams()
self.set_model_name(model)
super().__init__(
settings=GoogleLLMSettings(
model=model,
max_tokens=params.max_tokens,
temperature=params.temperature,
top_k=params.top_k,
top_p=params.top_p,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
thinking=params.thinking,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._api_key = api_key
self._system_instruction = system_instruction
self._http_options = update_google_client_http_options(http_options)
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"thinking": params.thinking,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self._tools = tools
self._tool_config = tool_config
@@ -840,7 +875,7 @@ class GoogleLLMService(LLMService):
# Use the new google-genai client's async method
response = await self._client.aio.models.generate_content(
model=self._model_name,
model=self._settings.model,
contents=messages,
config=generation_config,
)
@@ -874,10 +909,10 @@ class GoogleLLMService(LLMService):
k: v
for k, v in {
"system_instruction": system_instruction,
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"top_k": self._settings["top_k"],
"max_output_tokens": self._settings["max_tokens"],
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"top_k": self._settings.top_k,
"max_output_tokens": self._settings.max_tokens,
"tools": tools,
"tool_config": tool_config,
}.items()
@@ -885,13 +920,13 @@ class GoogleLLMService(LLMService):
}
# Add thinking parameters if configured
if self._settings["thinking"]:
generation_params["thinking_config"] = self._settings["thinking"].model_dump(
if self._settings.thinking:
generation_params["thinking_config"] = self._settings.thinking.model_dump(
exclude_unset=True
)
if self._settings["extra"]:
generation_params.update(self._settings["extra"])
if self._settings.extra:
generation_params.update(self._settings.extra)
return generation_params
@@ -900,10 +935,10 @@ class GoogleLLMService(LLMService):
# There's no way to introspect on model capabilities, so
# to check for models that we know default to thinkin on
# and can be configured to turn it off.
if not self._model_name.startswith("gemini-2.5-flash"):
if not self._settings.model.startswith("gemini-2.5-flash"):
return
# If we have an image model, we don't use a budget either.
if "image" in self._model_name:
if "image" in self._settings.model:
return
# If thinking_config is already set, don't override it.
if "thinking_config" in generation_params:
@@ -944,7 +979,7 @@ class GoogleLLMService(LLMService):
await self.start_ttfb_metrics()
return await self._client.aio.models.generate_content_stream(
model=self._model_name,
model=self._settings.model,
contents=messages,
config=generation_config,
)
@@ -1190,8 +1225,6 @@ class GoogleLLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = GoogleLLMContext(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -1215,14 +1248,6 @@ class GoogleLLMService(LLMService):
# Do nothing - we're shutting down anyway
pass
async def _update_settings(self, settings):
"""Override to handle ThinkingConfig validation."""
# Convert thinking dict to ThinkingConfig if needed
if "thinking" in settings and isinstance(settings["thinking"], dict):
settings = dict(settings) # Make a copy to avoid modifying the original
settings["thinking"] = self.ThinkingConfig(**settings["thinking"])
await super()._update_settings(settings)
def create_context_aggregator(
self,
context: OpenAILLMContext,

254
src/pipecat/services/google/stt.py
View File

@@ -15,13 +15,15 @@ import asyncio
import json
import os
import time
import warnings
from dataclasses import dataclass, field
from pipecat.utils.tracing.service_decorators import traced_stt
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from typing import AsyncGenerator, List, Optional, Union
from typing import Any, AsyncGenerator, List, Optional, Union
from loguru import logger
from pydantic import BaseModel, Field, field_validator
@@ -34,6 +36,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GOOGLE_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -355,6 +358,46 @@ def language_to_google_stt_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class GoogleSTTSettings(STTSettings):
"""Settings for Google Cloud Speech-to-Text V2.
Parameters:
languages: List of ``Language`` enums for recognition
(e.g. ``[Language.EN_US]``). Preferred over ``language_codes``.
language_codes: List of Google STT language code strings
(e.g. ``["en-US"]``).
.. deprecated:: 0.0.104
Use ``languages`` instead. If both are provided, ``languages``
takes precedence. This field is here just for backward
compatibility with dict-based settings updates.
use_separate_recognition_per_channel: Process each audio channel separately.
enable_automatic_punctuation: Add punctuation to transcripts.
enable_spoken_punctuation: Include spoken punctuation in transcript.
enable_spoken_emojis: Include spoken emojis in transcript.
profanity_filter: Filter profanity from transcript.
enable_word_time_offsets: Include timing information for each word.
enable_word_confidence: Include confidence scores for each word.
enable_interim_results: Stream partial recognition results.
enable_voice_activity_events: Detect voice activity in audio.
"""
languages: List[Language] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_codes: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
use_separate_recognition_per_channel: bool | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
enable_automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_spoken_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_spoken_emojis: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_word_time_offsets: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_word_confidence: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_interim_results: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_voice_activity_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GoogleSTTService(STTService):
"""Google Cloud Speech-to-Text V2 service implementation.
@@ -371,6 +414,8 @@ class GoogleSTTService(STTService):
ValueError: If project ID is not found in credentials.
"""
_settings: GoogleSTTSettings
# Google Cloud's STT service has a connection time limit of 5 minutes per stream.
# They've shared an "endless streaming" example that guided this implementation:
# https://cloud.google.com/speech-to-text/docs/transcribe-streaming-audio#endless-streaming
@@ -454,10 +499,29 @@ class GoogleSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or GoogleSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=GoogleSTTSettings(
language=None,
languages=list(params.language_list),
language_codes=None,
model=params.model,
use_separate_recognition_per_channel=params.use_separate_recognition_per_channel,
enable_automatic_punctuation=params.enable_automatic_punctuation,
enable_spoken_punctuation=params.enable_spoken_punctuation,
enable_spoken_emojis=params.enable_spoken_emojis,
profanity_filter=params.profanity_filter,
enable_word_time_offsets=params.enable_word_time_offsets,
enable_word_confidence=params.enable_word_confidence,
enable_interim_results=params.enable_interim_results,
enable_voice_activity_events=params.enable_voice_activity_events,
),
**kwargs,
)
self._location = location
self._stream = None
self._config = None
@@ -508,22 +572,6 @@ class GoogleSTTService(STTService):
self._client = speech_v2.SpeechAsyncClient(credentials=creds, client_options=client_options)
self._settings = {
"language_codes": [
self.language_to_service_language(lang) for lang in params.language_list
],
"model": params.model,
"use_separate_recognition_per_channel": params.use_separate_recognition_per_channel,
"enable_automatic_punctuation": params.enable_automatic_punctuation,
"enable_spoken_punctuation": params.enable_spoken_punctuation,
"enable_spoken_emojis": params.enable_spoken_emojis,
"profanity_filter": params.profanity_filter,
"enable_word_time_offsets": params.enable_word_time_offsets,
"enable_word_confidence": params.enable_word_confidence,
"enable_interim_results": params.enable_interim_results,
"enable_voice_activity_events": params.enable_voice_activity_events,
}
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
@@ -545,6 +593,21 @@ class GoogleSTTService(STTService):
return [language_to_google_stt_language(lang) or "en-US" for lang in language]
return language_to_google_stt_language(language) or "en-US"
def _get_language_codes(self) -> List[str]:
"""Resolve the current language settings to Google STT language code strings.
Prefers ``languages`` (``Language`` enums) over the deprecated
``language_codes`` (raw strings). Falls back to ``["en-US"]``.
Returns:
List[str]: Google STT language code strings.
"""
if self._settings.languages:
return [self.language_to_service_language(lang) for lang in self._settings.languages]
if self._settings.language_codes:
return list(self._settings.language_codes)
return ["en-US"]
async def _reconnect_if_needed(self):
"""Reconnect the stream if it's currently active."""
if self._streaming_task:
@@ -552,41 +615,65 @@ class GoogleSTTService(STTService):
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
"""Update the service's recognition language.
A convenience method for setting a single language.
Args:
language: New language for recognition.
"""
logger.debug(f"Switching STT language to: {language}")
await self.set_languages([language])
async def set_languages(self, languages: List[Language]):
"""Update the service's recognition languages.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(languages=...)``
instead.
Args:
languages: List of languages for recognition. First language is primary.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"set_languages() is deprecated. Use STTUpdateSettingsFrame with "
"GoogleSTTSettings(languages=...) instead.",
DeprecationWarning,
)
logger.debug(f"Switching STT languages to: {languages}")
self._settings["language_codes"] = [
self.language_to_service_language(lang) for lang in languages
]
# Recreate stream with new languages
await self._reconnect_if_needed()
await self._update_settings(GoogleSTTSettings(languages=list(languages)))
async def set_model(self, model: str):
"""Update the service's recognition model.
async def _update_settings(self, delta: GoogleSTTSettings) -> dict[str, Any]:
"""Apply settings delta and reconnect if anything changed.
Handles ``language`` from base ``set_language`` by converting it to
``languages``. Emits a deprecation warning if ``language_codes`` is
used. All other fields (model, boolean flags) are applied directly.
Reconnects the stream on any change.
Args:
model: The new recognition model to use.
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.debug(f"Switching STT model to: {model}")
await super().set_model(model)
self._settings["model"] = model
# Recreate stream with new model
await self._reconnect_if_needed()
from pipecat.services.settings import is_given
# If base set_language sent a Language value, convert to languages list
if is_given(delta.language):
delta.languages = [delta.language]
# Clear language so the base class doesn't try to store it
delta.language = NOT_GIVEN
# Warn on deprecated language_codes usage
if is_given(delta.language_codes):
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"GoogleSTTSettings.language_codes is deprecated. "
"Use GoogleSTTSettings.languages (List[Language]) instead.",
DeprecationWarning,
stacklevel=2,
)
changed = await super()._update_settings(delta)
if changed:
await self._reconnect_if_needed()
return changed
async def start(self, frame: StartFrame):
"""Start the STT service and establish connection.
@@ -632,6 +719,10 @@ class GoogleSTTService(STTService):
) -> None:
"""Update service options dynamically.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with ``GoogleSTTSettings(...)``
instead.
Args:
languages: New list of recognition languages.
model: New recognition model.
@@ -649,55 +740,42 @@ class GoogleSTTService(STTService):
Changes that affect the streaming configuration will cause
the stream to be reconnected.
"""
# Update settings with new values
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"update_options() is deprecated. Use STTUpdateSettingsFrame with "
"GoogleSTTSettings(...) instead.",
DeprecationWarning,
)
# Build a settings delta from the provided options
delta = GoogleSTTSettings()
if languages is not None:
logger.debug(f"Updating language to: {languages}")
self._settings["language_codes"] = [
self.language_to_service_language(lang) for lang in languages
]
delta.languages = list(languages)
if model is not None:
logger.debug(f"Updating model to: {model}")
self._settings["model"] = model
delta.model = model
if enable_automatic_punctuation is not None:
logger.debug(f"Updating automatic punctuation to: {enable_automatic_punctuation}")
self._settings["enable_automatic_punctuation"] = enable_automatic_punctuation
delta.enable_automatic_punctuation = enable_automatic_punctuation
if enable_spoken_punctuation is not None:
logger.debug(f"Updating spoken punctuation to: {enable_spoken_punctuation}")
self._settings["enable_spoken_punctuation"] = enable_spoken_punctuation
delta.enable_spoken_punctuation = enable_spoken_punctuation
if enable_spoken_emojis is not None:
logger.debug(f"Updating spoken emojis to: {enable_spoken_emojis}")
self._settings["enable_spoken_emojis"] = enable_spoken_emojis
delta.enable_spoken_emojis = enable_spoken_emojis
if profanity_filter is not None:
logger.debug(f"Updating profanity filter to: {profanity_filter}")
self._settings["profanity_filter"] = profanity_filter
delta.profanity_filter = profanity_filter
if enable_word_time_offsets is not None:
logger.debug(f"Updating word time offsets to: {enable_word_time_offsets}")
self._settings["enable_word_time_offsets"] = enable_word_time_offsets
delta.enable_word_time_offsets = enable_word_time_offsets
if enable_word_confidence is not None:
logger.debug(f"Updating word confidence to: {enable_word_confidence}")
self._settings["enable_word_confidence"] = enable_word_confidence
delta.enable_word_confidence = enable_word_confidence
if enable_interim_results is not None:
logger.debug(f"Updating interim results to: {enable_interim_results}")
self._settings["enable_interim_results"] = enable_interim_results
delta.enable_interim_results = enable_interim_results
if enable_voice_activity_events is not None:
logger.debug(f"Updating voice activity events to: {enable_voice_activity_events}")
self._settings["enable_voice_activity_events"] = enable_voice_activity_events
delta.enable_voice_activity_events = enable_voice_activity_events
if location is not None:
logger.debug(f"Updating location to: {location}")
self._location = location
# Reconnect the stream for updates
await self._reconnect_if_needed()
await self._update_settings(delta)
async def _connect(self):
"""Initialize streaming recognition config and stream."""
@@ -714,20 +792,20 @@ class GoogleSTTService(STTService):
sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
language_codes=self._settings["language_codes"],
model=self._settings["model"],
language_codes=self._get_language_codes(),
model=self._settings.model,
features=cloud_speech.RecognitionFeatures(
enable_automatic_punctuation=self._settings["enable_automatic_punctuation"],
enable_spoken_punctuation=self._settings["enable_spoken_punctuation"],
enable_spoken_emojis=self._settings["enable_spoken_emojis"],
profanity_filter=self._settings["profanity_filter"],
enable_word_time_offsets=self._settings["enable_word_time_offsets"],
enable_word_confidence=self._settings["enable_word_confidence"],
enable_automatic_punctuation=self._settings.enable_automatic_punctuation,
enable_spoken_punctuation=self._settings.enable_spoken_punctuation,
enable_spoken_emojis=self._settings.enable_spoken_emojis,
profanity_filter=self._settings.profanity_filter,
enable_word_time_offsets=self._settings.enable_word_time_offsets,
enable_word_confidence=self._settings.enable_word_confidence,
),
),
streaming_features=cloud_speech.StreamingRecognitionFeatures(
enable_voice_activity_events=self._settings["enable_voice_activity_events"],
interim_results=self._settings["enable_interim_results"],
enable_voice_activity_events=self._settings.enable_voice_activity_events,
interim_results=self._settings.enable_interim_results,
),
)
@@ -857,7 +935,7 @@ class GoogleSTTService(STTService):
if not transcript:
continue
primary_language = self._settings["language_codes"][0]
primary_language = self._get_language_codes()[0]
if result.is_final:
self._last_transcript_was_final = True

285
src/pipecat/services/google/tts.py
View File

@@ -23,7 +23,8 @@ from pipecat.utils.tracing.service_decorators import traced_tts
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from typing import Any, AsyncGenerator, List, Literal, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional
from loguru import logger
from pydantic import BaseModel
@@ -36,6 +37,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, is_given
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
@@ -474,6 +476,71 @@ def language_to_gemini_tts_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class GoogleHttpTTSSettings(TTSSettings):
"""Settings for Google HTTP TTS service.
Parameters:
pitch: Voice pitch adjustment (e.g., "+2st", "-50%").
rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). Used for
SSML prosody tags (non-Chirp voices).
speaking_rate: Speaking rate for AudioConfig (Chirp/Journey voices).
Range [0.25, 2.0].
volume: Volume adjustment (e.g., "loud", "soft", "+6dB").
emphasis: Emphasis level for the text.
language: Language for synthesis. Defaults to English.
gender: Voice gender preference.
google_style: Google-specific voice style.
"""
pitch: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
rate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emphasis: Literal["strong", "moderate", "reduced", "none"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
gender: Literal["male", "female", "neutral"] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
google_style: (
Literal["apologetic", "calm", "empathetic", "firm", "lively"] | None | _NotGiven
) = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class GoogleStreamTTSSettings(TTSSettings):
"""Settings for Google streaming TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
speaking_rate: The speaking rate, in the range [0.25, 2.0].
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class GeminiTTSSettings(TTSSettings):
"""Settings for Gemini TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
prompt: Optional style instructions for how to synthesize the content.
multi_speaker: Whether to enable multi-speaker support.
speaker_configs: List of speaker configurations for multi-speaker mode.
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
multi_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_configs: list[dict[str, Any]] | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class GoogleHttpTTSService(TTSService):
"""Google Cloud Text-to-Speech HTTP service with SSML support.
@@ -488,6 +555,8 @@ class GoogleHttpTTSService(TTSService):
Chirp and Journey voices don't support SSML and will use plain text input.
"""
_settings: GoogleHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for Google HTTP TTS voice customization.
@@ -533,24 +602,28 @@ class GoogleHttpTTSService(TTSService):
params: Voice customization parameters including pitch, rate, volume, etc.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=GoogleHttpTTSSettings(
model=None,
pitch=params.pitch,
rate=params.rate,
speaking_rate=params.speaking_rate,
volume=params.volume,
emphasis=params.emphasis,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
gender=params.gender,
google_style=params.google_style,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._settings = {
"pitch": params.pitch,
"rate": params.rate,
"speaking_rate": params.speaking_rate,
"volume": params.volume,
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"gender": params.gender,
"google_style": params.google_style,
}
self.set_voice(voice_id)
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
@@ -619,61 +692,60 @@ class GoogleHttpTTSService(TTSService):
"""
return language_to_google_tts_language(language)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle speaking_rate updates for Chirp/Journey voices.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Override to handle speaking_rate validation.
Args:
settings: Dictionary of settings to update. Can include 'speaking_rate' (float)
delta: Settings delta. Can include 'speaking_rate' (float).
"""
if "speaking_rate" in settings:
rate_value = float(settings["speaking_rate"])
if 0.25 <= rate_value <= 2.0:
self._settings["speaking_rate"] = rate_value
else:
if isinstance(delta, GoogleHttpTTSSettings) and is_given(delta.speaking_rate):
rate_value = float(delta.speaking_rate)
if not (0.25 <= rate_value <= 2.0):
logger.warning(
f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0"
)
await super()._update_settings(settings)
delta.speaking_rate = NOT_GIVEN
return await super()._update_settings(delta)
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
# Voice tag
voice_attrs = [f"name='{self._voice_id}'"]
voice_attrs = [f"name='{self._settings.voice}'"]
language = self._settings["language"]
language = self._settings.language
voice_attrs.append(f"language='{language}'")
if self._settings["gender"]:
voice_attrs.append(f"gender='{self._settings['gender']}'")
if self._settings.gender:
voice_attrs.append(f"gender='{self._settings.gender}'")
ssml += f"<voice {' '.join(voice_attrs)}>"
# Prosody tag
prosody_attrs = []
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if self._settings.pitch:
prosody_attrs.append(f"pitch='{self._settings.pitch}'")
if self._settings.rate:
prosody_attrs.append(f"rate='{self._settings.rate}'")
if self._settings.volume:
prosody_attrs.append(f"volume='{self._settings.volume}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
# Emphasis tag
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
if self._settings.emphasis:
ssml += f"<emphasis level='{self._settings.emphasis}'>"
# Google style tag
if self._settings["google_style"]:
ssml += f"<google:style name='{self._settings['google_style']}'>"
if self._settings.google_style:
ssml += f"<google:style name='{self._settings.google_style}'>"
ssml += text
# Close tags
if self._settings["google_style"]:
if self._settings.google_style:
ssml += "</google:style>"
if self._settings["emphasis"]:
if self._settings.emphasis:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
@@ -698,8 +770,8 @@ class GoogleHttpTTSService(TTSService):
await self.start_ttfb_metrics()
# Check if the voice is a Chirp voice (including Chirp 3) or Journey voice
is_chirp_voice = "chirp" in self._voice_id.lower()
is_journey_voice = "journey" in self._voice_id.lower()
is_chirp_voice = "chirp" in self._settings.voice.lower()
is_journey_voice = "journey" in self._settings.voice.lower()
# Create synthesis input based on voice_id
if is_chirp_voice or is_journey_voice:
@@ -710,7 +782,7 @@ class GoogleHttpTTSService(TTSService):
synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], name=self._voice_id
language_code=self._settings.language, name=self._settings.voice
)
# Build audio config with conditional speaking_rate
audio_config_params = {
@@ -719,8 +791,8 @@ class GoogleHttpTTSService(TTSService):
}
# For Chirp and Journey voices, include speaking_rate in AudioConfig
if (is_chirp_voice or is_journey_voice) and self._settings["speaking_rate"] is not None:
audio_config_params["speaking_rate"] = self._settings["speaking_rate"]
if (is_chirp_voice or is_journey_voice) and self._settings.speaking_rate is not None:
audio_config_params["speaking_rate"] = self._settings.speaking_rate
audio_config = texttospeech_v1.AudioConfig(**audio_config_params)
@@ -910,6 +982,8 @@ class GoogleTTSService(GoogleBaseTTSService):
)
"""
_settings: GoogleStreamTTSSettings
class InputParams(BaseModel):
"""Input parameters for Google streaming TTS configuration.
@@ -945,38 +1019,41 @@ class GoogleTTSService(GoogleBaseTTSService):
params: Language configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=GoogleStreamTTSSettings(
model=None,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
speaking_rate=params.speaking_rate,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"speaking_rate": params.speaking_rate,
}
self.set_voice(voice_id)
self._voice_cloning_key = voice_cloning_key
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle speaking_rate updates for streaming API.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Override to handle speaking_rate validation.
Args:
settings: Dictionary of settings to update. Can include 'speaking_rate' (float)
delta: Settings delta. Can include 'speaking_rate' (float).
"""
if "speaking_rate" in settings:
rate_value = float(settings["speaking_rate"])
if 0.25 <= rate_value <= 2.0:
self._settings["speaking_rate"] = rate_value
else:
if isinstance(delta, GoogleStreamTTSSettings) and is_given(delta.speaking_rate):
rate_value = float(delta.speaking_rate)
if not (0.25 <= rate_value <= 2.0):
logger.warning(
f"Invalid speaking_rate value: {rate_value}. Must be between 0.25 and 2.0"
)
await super()._update_settings(settings)
delta.speaking_rate = NOT_GIVEN
return await super()._update_settings(delta)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -1000,11 +1077,11 @@ class GoogleTTSService(GoogleBaseTTSService):
voice_cloning_key=self._voice_cloning_key
)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], voice_clone=voice_clone_params
language_code=self._settings.language, voice_clone=voice_clone_params
)
else:
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], name=self._voice_id
language_code=self._settings.language, name=self._settings.voice
)
# Create streaming config
@@ -1013,7 +1090,7 @@ class GoogleTTSService(GoogleBaseTTSService):
streaming_audio_config=texttospeech_v1.StreamingAudioConfig(
audio_encoding=texttospeech_v1.AudioEncoding.PCM,
sample_rate_hertz=self.sample_rate,
speaking_rate=self._settings["speaking_rate"],
speaking_rate=self._settings.speaking_rate,
),
)
@@ -1052,6 +1129,8 @@ class GeminiTTSService(GoogleBaseTTSService):
)
"""
_settings: GeminiTTSSettings
GOOGLE_SAMPLE_RATE = 24000 # Google TTS always outputs at 24kHz
# List of available Gemini TTS voices
@@ -1149,25 +1228,27 @@ class GeminiTTSService(GoogleBaseTTSService):
f"Google TTS only supports {self.GOOGLE_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GeminiTTSService.InputParams()
if voice_id not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
self._location = location
self._model = model
self._voice_id = voice_id
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "en-US",
"prompt": params.prompt,
"multi_speaker": params.multi_speaker,
"speaker_configs": params.speaker_configs,
}
super().__init__(
sample_rate=sample_rate,
settings=GeminiTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "en-US",
prompt=params.prompt,
multi_speaker=params.multi_speaker,
speaker_configs=params.speaker_configs,
voice=voice_id,
),
**kwargs,
)
self._location = location
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
@@ -1183,16 +1264,6 @@ class GeminiTTSService(GoogleBaseTTSService):
"""
return language_to_gemini_tts_language(language)
def set_voice(self, voice_id: str):
"""Set the voice for TTS generation.
Args:
voice_id: Name of the voice to use from AVAILABLE_VOICES.
"""
if voice_id not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{voice_id}' not in known voices list. Using anyway.")
self._voice_id = voice_id
async def start(self, frame: StartFrame):
"""Start the Gemini TTS service.
@@ -1206,15 +1277,19 @@ class GeminiTTSService(GoogleBaseTTSService):
f"Current rate of {self.sample_rate}Hz may cause issues."
)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Override to handle prompt updates.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta with voice validation.
Args:
settings: Dictionary of settings to update. Can include 'prompt' (str)
delta: Settings delta. Can include 'voice', 'prompt', etc.
Returns:
Dict mapping changed field names to their previous values.
"""
if "prompt" in settings:
self._settings["prompt"] = settings["prompt"]
await super()._update_settings(settings)
if is_given(delta.voice) and delta.voice not in self.AVAILABLE_VOICES:
logger.warning(f"Voice '{delta.voice}' not in known voices list. Using anyway.")
return await super()._update_settings(delta)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -1234,14 +1309,14 @@ class GeminiTTSService(GoogleBaseTTSService):
await self.start_ttfb_metrics()
# Build voice selection params
if self._settings["multi_speaker"] and self._settings["speaker_configs"]:
if self._settings.multi_speaker and self._settings.speaker_configs:
# Multi-speaker mode
speaker_voice_configs = []
for speaker_config in self._settings["speaker_configs"]:
for speaker_config in self._settings.speaker_configs:
speaker_voice_configs.append(
texttospeech_v1.MultispeakerPrebuiltVoice(
speaker_alias=speaker_config["speaker_alias"],
speaker_id=speaker_config.get("speaker_id", self._voice_id),
speaker_id=speaker_config.get("speaker_id", self._settings.voice),
)
)
@@ -1250,16 +1325,16 @@ class GeminiTTSService(GoogleBaseTTSService):
)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"],
model_name=self._model,
language_code=self._settings.language,
model_name=self._settings.model,
multi_speaker_voice_config=multi_speaker_voice_config,
)
else:
# Single speaker mode
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"],
name=self._voice_id,
model_name=self._model,
language_code=self._settings.language,
name=self._settings.voice,
model_name=self._settings.model,
)
# Create streaming config
@@ -1273,7 +1348,7 @@ class GeminiTTSService(GoogleBaseTTSService):
# Use base class streaming logic with prompt support
async for frame in self._stream_tts(
streaming_config, text, context_id, self._settings["prompt"]
streaming_config, text, context_id, self._settings.prompt
):
yield frame

58
src/pipecat/services/gradium/stt.py
View File

@@ -12,7 +12,8 @@ WebSocket API for streaming audio transcription.
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
@@ -27,6 +28,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import GRADIUM_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -64,6 +66,18 @@ def language_to_gradium_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class GradiumSTTSettings(STTSettings):
"""Settings for the Gradium STT service.
Parameters:
delay_in_frames: Delay in audio frames (80ms each) before text is
generated. Higher delays allow more context but increase latency.
"""
delay_in_frames: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GradiumSTTService(WebsocketSTTService):
"""Gradium real-time speech-to-text service.
@@ -72,6 +86,8 @@ class GradiumSTTService(WebsocketSTTService):
for audio processing and connection management.
"""
_settings: GradiumSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Gradium STT API.
@@ -113,8 +129,6 @@ class GradiumSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(sample_rate=SAMPLE_RATE, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
if json_config is not None:
import warnings
@@ -124,10 +138,22 @@ class GradiumSTTService(WebsocketSTTService):
stacklevel=2,
)
params = params or GradiumSTTService.InputParams()
super().__init__(
sample_rate=SAMPLE_RATE,
ttfs_p99_latency=ttfs_p99_latency,
settings=GradiumSTTSettings(
model=None,
language=params.language,
delay_in_frames=params.delay_in_frames or None,
),
**kwargs,
)
self._api_key = api_key
self._api_endpoint_base_url = api_endpoint_base_url
self._websocket = None
self._params = params or GradiumSTTService.InputParams()
self._json_config = json_config
self._receive_task = None
@@ -149,16 +175,22 @@ class GradiumSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta, sync params, and reconnect.
Args:
language: The language to use for speech recognition.
delta: A :class:`STTSettings` (or ``GradiumSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
logger.info(f"Switching STT language to: [{language}]")
self._params.language = language
changed = await super()._update_settings(delta)
if not changed:
return changed
await self._disconnect()
await self._connect()
return changed
async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
@@ -298,12 +330,12 @@ class GradiumSTTService(WebsocketSTTService):
json_config = {}
if self._json_config:
json_config = json.loads(self._json_config)
if self._params.language:
gradium_language = language_to_gradium_language(self._params.language)
if self._settings.language:
gradium_language = language_to_gradium_language(self._settings.language)
if gradium_language:
json_config["language"] = gradium_language
if self._params.delay_in_frames:
json_config["delay_in_frames"] = self._params.delay_in_frames
if self._settings.delay_in_frames:
json_config["delay_in_frames"] = self._settings.delay_in_frames
if json_config:
setup_msg["json_config"] = json_config
await self._websocket.send(json.dumps(setup_msg))

144
src/pipecat/services/gradium/tts.py
View File

@@ -6,7 +6,8 @@
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
@@ -21,8 +22,8 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleWordTTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
try:
@@ -37,9 +38,22 @@ except ModuleNotFoundError as e:
SAMPLE_RATE = 48000
class GradiumTTSService(InterruptibleWordTTSService):
@dataclass
class GradiumTTSSettings(TTSSettings):
"""Settings for the Gradium TTS service.
Parameters:
output_format: Audio output format.
"""
output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GradiumTTSService(AudioContextTTSService):
"""Text-to-Speech service using Gradium's websocket API."""
_settings: GradiumTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Gradium TTS service.
@@ -71,31 +85,30 @@ class GradiumTTSService(InterruptibleWordTTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent class.
"""
# Initialize with parent class settings for proper frame handling
params = params or GradiumTTSService.InputParams()
super().__init__(
push_stop_frames=True,
push_text_frames=False,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=SAMPLE_RATE,
settings=GradiumTTSSettings(
model=model,
voice=voice_id,
language=None,
output_format="pcm",
),
**kwargs,
)
params = params or GradiumTTSService.InputParams()
# Store service configuration
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._json_config = json_config
self._model = model
self._settings = {
"voice_id": voice_id,
"model_name": model,
"output_format": "pcm",
}
# State tracking
self._receive_task = None
self._current_context_id: Optional[str] = None
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -105,28 +118,30 @@ class GradiumTTSService(InterruptibleWordTTSService):
"""
return True
async def set_model(self, model: str):
"""Update the TTS model.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if voice changed.
Args:
model: The model name to use for synthesis.
"""
self._model = model
await super().set_model(model)
delta: A :class:`TTSSettings` (or ``GradiumTTSSettings``) delta.
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
self._settings["voice_id"] = self._voice_id
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if "voice" in changed:
await self._disconnect()
await self._connect()
else:
self._warn_unhandled_updated_settings(changed)
return changed
def _build_msg(self, text: str = "") -> dict:
"""Build JSON message for Gradium API."""
return {"text": text, "type": "text"}
msg = {"text": text, "type": "text"}
context_id = self.get_active_audio_context_id()
if context_id:
msg["client_req_id"] = context_id
return msg
async def start(self, frame: StartFrame):
"""Start the service and establish websocket connection.
@@ -196,7 +211,8 @@ class GradiumTTSService(InterruptibleWordTTSService):
setup_msg = {
"type": "setup",
"output_format": "pcm",
"voice_id": self._voice_id,
"voice_id": self._settings.voice,
"close_ws_on_eos": False,
}
if self._json_config is not None:
setup_msg["json_config"] = self._json_config
@@ -223,6 +239,7 @@ class GradiumTTSService(InterruptibleWordTTSService):
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
finally:
await self.remove_active_audio_context()
self._websocket = None
await self._call_event_handler("on_disconnected")
@@ -234,18 +251,38 @@ class GradiumTTSService(InterruptibleWordTTSService):
async def flush_audio(self):
"""Flush any pending audio synthesis."""
if not self._websocket:
context_id = self.get_active_audio_context_id()
if not context_id or not self._websocket:
return
try:
msg = {"type": "end_of_stream"}
msg = {"type": "end_of_stream", "client_req_id": context_id}
await self._websocket.send(json.dumps(msg))
self.reset_active_audio_context()
except ConnectionClosedOK:
logger.debug(f"{self}: connection closed normally during flush")
except Exception as e:
logger.error(f"{self} exception: {e}")
async def on_audio_context_interrupted(self, context_id: str):
"""Called when an audio context is cancelled due to an interruption.
No WebSocket message is needed — audio from the interrupted
``client_req_id`` will be silently dropped by the base class once the
audio context no longer exists.
"""
await self.stop_all_metrics()
async def on_audio_context_completed(self, context_id: str):
"""Called after an audio context has finished playing all of its audio.
No close message is needed: Gradium signals completion with an
``end_of_stream`` message (handled in ``_receive_messages``), after
which the server-side context is already closed.
"""
pass
async def _receive_messages(self):
"""Process incoming websocket messages."""
"""Process incoming websocket messages, demultiplexing by client_req_id."""
# TODO(laurent): This should not be necessary as it should happen when
# receiving the messages but this does not seem to always be the case
# and that may lead to a busy polling loop.
@@ -253,41 +290,35 @@ class GradiumTTSService(InterruptibleWordTTSService):
raise ConnectionClosedOK(None, None)
async for message in self._get_websocket():
msg = json.loads(message)
ctx_id = msg.get("client_req_id")
if msg["type"] == "audio":
# Process audio chunk
if not ctx_id or not self.audio_context_available(ctx_id):
continue
await self.stop_ttfb_metrics()
await self.start_word_timestamps()
frame = TTSAudioRawFrame(
audio=base64.b64decode(msg["audio"]),
sample_rate=self.sample_rate,
num_channels=1,
context_id=self._current_context_id,
context_id=ctx_id,
)
await self.push_frame(frame)
await self.append_to_audio_context(ctx_id, frame)
elif msg["type"] == "text":
if self._current_context_id:
await self.add_word_timestamps(
[(msg["text"], msg["start_s"])], self._current_context_id
)
if ctx_id and self.audio_context_available(ctx_id):
await self.add_word_timestamps([(msg["text"], msg["start_s"])], ctx_id)
elif msg["type"] == "end_of_stream":
await self.push_frame(TTSStoppedFrame())
if ctx_id and self.audio_context_available(ctx_id):
await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)], ctx_id)
await self.remove_audio_context(ctx_id)
await self.stop_all_metrics()
elif msg["type"] == "error":
await self.push_frame(TTSStoppedFrame())
await self.push_frame(TTSStoppedFrame(context_id=ctx_id))
await self.stop_all_metrics()
await self.push_error(error_msg=f"Error: {msg['message']}")
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push frame and handle end-of-turn conditions.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
await super().push_frame(frame, direction)
await self.push_error(error_msg=f"Error: {msg.get('message', msg)}")
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -300,16 +331,17 @@ class GradiumTTSService(InterruptibleWordTTSService):
Yields:
Frame: Audio frames containing the synthesized speech.
"""
_state = self._websocket.state if self._websocket is not None else None
logger.debug(f"{self}: Generating TTS [{text}] {_state}")
logger.debug(f"{self}: Generating TTS [{text}]")
try:
if not self._websocket or self._websocket.state is State.CLOSED:
self._websocket = None
await self._connect()
try:
self._current_context_id = context_id
yield TTSStartedFrame(context_id=context_id)
if not self.has_active_audio_context():
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
await self.create_audio_context(context_id)
msg = self._build_msg(text=text)
await self._get_websocket().send(json.dumps(msg))

143
src/pipecat/services/grok/realtime/llm.py
View File

@@ -13,8 +13,8 @@ https://docs.x.ai/docs/guides/voice/agent
import base64
import json
import time
from dataclasses import dataclass
from typing import Optional
from dataclasses import dataclass, field
from typing import Any, Optional
from loguru import logger
@@ -56,6 +56,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
from . import events
@@ -85,6 +86,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class GrokRealtimeLLMSettings(LLMSettings):
"""Settings for Grok Realtime LLM services.
Parameters:
session_properties: Grok Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class GrokRealtimeLLMService(LLMService):
"""Grok Realtime Voice Agent LLM service providing real-time audio and text communication.
@@ -101,6 +115,8 @@ class GrokRealtimeLLMService(LLMService):
- Server-side VAD (Voice Activity Detection)
"""
_settings: GrokRealtimeLLMSettings
# Use the Grok-specific adapter
adapter_class = GrokRealtimeLLMAdapter
@@ -129,16 +145,27 @@ class GrokRealtimeLLMService(LLMService):
start_audio_paused: Whether to start with audio input paused. Defaults to False.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(base_url=base_url, **kwargs)
super().__init__(
base_url=base_url,
settings=GrokRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = base_url
# Initialize session_properties
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._websocket = None
self._receive_task = None
@@ -186,13 +213,13 @@ class GrokRealtimeLLMService(LLMService):
Configured sample rate or None if not manually configured.
For PCMU/PCMA formats, returns 8000 Hz (G.711 standard).
"""
if not self._session_properties.audio:
if not self._settings.session_properties.audio:
return None
audio_config = (
self._session_properties.audio.input
self._settings.session_properties.audio.input
if direction == "input"
else self._session_properties.audio.output
else self._settings.session_properties.audio.output
)
if audio_config and audio_config.format:
@@ -222,8 +249,8 @@ class GrokRealtimeLLMService(LLMService):
def _is_turn_detection_enabled(self) -> bool:
"""Check if server-side VAD is enabled."""
if self._session_properties.turn_detection:
return self._session_properties.turn_detection.type == "server_vad"
if self._settings.session_properties.turn_detection:
return self._settings.session_properties.turn_detection.type == "server_vad"
return False
async def _handle_interruption(self):
@@ -281,6 +308,27 @@ class GrokRealtimeLLMService(LLMService):
# Standard AIService frame handling
#
def _ensure_audio_config(self, input_sample_rate: int, output_sample_rate: int):
"""Ensure session_properties.audio has input and output configs.
Fills in any missing audio configuration using the given sample rates.
Args:
input_sample_rate: Sample rate for audio input (Hz).
output_sample_rate: Sample rate for audio output (Hz).
"""
props = self._settings.session_properties
if not props.audio:
props.audio = events.AudioConfiguration()
if not props.audio.input:
props.audio.input = events.AudioInput(
format=events.PCMAudioFormat(rate=input_sample_rate)
)
if not props.audio.output:
props.audio.output = events.AudioOutput(
format=events.PCMAudioFormat(rate=output_sample_rate)
)
async def start(self, frame: StartFrame):
"""Start the service and establish WebSocket connection.
@@ -288,23 +336,7 @@ class GrokRealtimeLLMService(LLMService):
frame: The start frame triggering service initialization.
"""
await super().start(frame)
# Ensure audio configuration exists with both input and output
if not self._session_properties.audio:
self._session_properties.audio = events.AudioConfiguration()
# Fill in missing input configuration
if not self._session_properties.audio.input:
self._session_properties.audio.input = events.AudioInput(
format=events.PCMAudioFormat(rate=frame.audio_in_sample_rate)
)
# Fill in missing output configuration
if not self._session_properties.audio.output:
self._session_properties.audio.output = events.AudioOutput(
format=events.PCMAudioFormat(rate=frame.audio_out_sample_rate)
)
self._ensure_audio_config(frame.audio_in_sample_rate, frame.audio_out_sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -336,6 +368,16 @@ class GrokRealtimeLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.delta path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.delta is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -355,11 +397,8 @@ class GrokRealtimeLLMService(LLMService):
await self._handle_bot_stopped_speaking()
elif isinstance(frame, LLMMessagesAppendFrame):
await self._handle_messages_append(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
await self.push_frame(frame, direction)
@@ -436,9 +475,30 @@ class GrokRealtimeLLMService(LLMService):
return
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update if needed."""
# Capture current sample rates before the update replaces them.
input_rate = self._get_configured_sample_rate("input")
output_rate = self._get_configured_sample_rate("output")
changed = await super()._update_settings(delta)
if "session_properties" in changed:
if input_rate and output_rate:
self._ensure_audio_config(input_rate, output_rate)
else:
logger.warning(
"Attempting to apply session properties update without configured sample rates. "
"Audio configuration may be incomplete."
)
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"})
return changed
async def _send_session_update(self):
"""Update session settings on the server."""
settings = self._session_properties
settings = self._settings.session_properties
adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
@@ -511,12 +571,15 @@ class GrokRealtimeLLMService(LLMService):
elif evt.type == "response.function_call_arguments.done":
await self._handle_evt_function_call_arguments_done(evt)
elif evt.type == "error":
await self._handle_evt_error(evt)
return
if evt.error.code == "response_cancel_not_active":
logger.debug(f"{self} {evt.error.message}")
else:
await self._handle_evt_error(evt)
return
async def _handle_evt_conversation_created(self, evt):
"""Handle conversation.created event - first event after connecting."""
await self._update_settings()
await self._send_session_update()
async def _handle_evt_response_created(self, evt):
"""Handle response.created event - response generation started."""
@@ -719,7 +782,7 @@ class GrokRealtimeLLMService(LLMService):
self._messages_added_manually[evt.item.id] = True
await self.send_client_event(evt)
await self._update_settings()
await self._send_session_update()
self._llm_needs_conversation_setup = False
logger.debug("Creating Grok response")

2
src/pipecat/services/groq/stt.py
View File

@@ -62,7 +62,7 @@ class GroqSTTService(BaseWhisperSTTService):
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"model": self._settings.model,
# Use verbose_json to get probability metrics
"response_format": "verbose_json" if self._include_prob_metrics else "json",
"language": self._language,

54
src/pipecat/services/groq/tts.py
View File

@@ -8,7 +8,8 @@
import io
import wave
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, ClassVar, Dict, Optional
from loguru import logger
from pydantic import BaseModel
@@ -20,6 +21,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -32,6 +34,23 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class GroqTTSSettings(TTSSettings):
"""Settings for the Groq TTS service.
Parameters:
output_format: Audio output format.
speed: Speech speed multiplier. Defaults to 1.0.
groq_sample_rate: Audio sample rate.
"""
output_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
groq_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice", "sample_rate": "groq_sample_rate"}
class GroqTTSService(TTSService):
"""Groq text-to-speech service implementation.
@@ -40,6 +59,8 @@ class GroqTTSService(TTSService):
and output formats.
"""
_settings: GroqTTSSettings
class InputParams(BaseModel):
"""Input parameters for Groq TTS configuration.
@@ -78,28 +99,24 @@ class GroqTTSService(TTSService):
if sample_rate != self.GROQ_SAMPLE_RATE:
logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ")
params = params or GroqTTSService.InputParams()
super().__init__(
pause_frame_processing=True,
sample_rate=sample_rate,
settings=GroqTTSSettings(
model=model_name,
voice=voice_id,
language=str(params.language) if params.language else "en",
output_format=output_format,
speed=params.speed,
groq_sample_rate=sample_rate,
),
**kwargs,
)
params = params or GroqTTSService.InputParams()
self._api_key = api_key
self._model_name = model_name
self._output_format = output_format
self._voice_id = voice_id
self._params = params
self._settings = {
"model": model_name,
"voice_id": voice_id,
"output_format": output_format,
"language": str(params.language) if params.language else "en",
"speed": params.speed,
"sample_rate": sample_rate,
}
self._client = AsyncGroq(api_key=self._api_key)
@@ -129,9 +146,12 @@ class GroqTTSService(TTSService):
try:
response = await self._client.audio.speech.create(
model=self._model_name,
voice=self._voice_id,
model=self._settings.model,
voice=self._settings.voice,
response_format=self._output_format,
# Note: as of 2026-02-25, only a speed of 1.0 is supported, but
# here we pass it for completeness and future-proofing
speed=self._settings.speed,
input=text,
)

47
src/pipecat/services/hathora/stt.py
View File

@@ -8,6 +8,7 @@
import base64
import os
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional
import aiohttp
@@ -18,6 +19,7 @@ from pipecat.frames.frames import (
Frame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import HATHORA_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
@@ -27,12 +29,27 @@ from pipecat.utils.tracing.service_decorators import traced_stt
from .utils import ConfigOption
@dataclass
class HathoraSTTSettings(STTSettings):
"""Settings for the Hathora STT service.
Parameters:
config: Some models support additional config, refer to
`docs <https://models.hathora.dev>`_ for each model to see
what is supported.
"""
config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HathoraSTTService(SegmentedSTTService):
"""This service supports several different speech-to-text models hosted by Hathora.
[Documentation](https://models.hathora.dev)
"""
_settings: HathoraSTTSettings
class InputParams(BaseModel):
"""Optional input parameters for Hathora STT configuration.
@@ -72,24 +89,21 @@ class HathoraSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the parent class.
"""
params = params or HathoraSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=HathoraSTTSettings(
model=model,
language=params.language,
config=params.config,
),
**kwargs,
)
self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
params = params or HathoraSTTService.InputParams()
self._settings = {
"language": params.language,
"config": params.config,
}
self.set_model_name(model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -120,15 +134,14 @@ class HathoraSTTService(SegmentedSTTService):
url = f"{self._base_url}"
payload = {
"model": self._model,
"model": self._settings.model,
}
if self._settings["language"] is not None:
payload["language"] = self._settings["language"]
if self._settings["config"] is not None:
if self._settings.language is not None:
payload["language"] = self._settings.language
if self._settings.config is not None:
payload["model_config"] = [
{"name": option.name, "value": option.value}
for option in self._settings["config"]
{"name": option.name, "value": option.value} for option in self._settings.config
]
base64_audio = base64.b64encode(audio).decode("utf-8")
@@ -147,7 +160,7 @@ class HathoraSTTService(SegmentedSTTService):
if text: # Only yield non-empty text
# Hathora's API currently doesn't return language info
# so we default to the requested language or "en"
response_language = self._settings["language"] or "en"
response_language = self._settings.language or "en"
await self._handle_transcription(text, True, response_language)
yield TranscriptionFrame(
text,

62
src/pipecat/services/hathora/tts.py
View File

@@ -9,6 +9,7 @@
import io
import os
import wave
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional, Tuple
import aiohttp
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -45,12 +47,29 @@ def _decode_audio_payload(
return audio_bytes, fallback_sample_rate, fallback_channels
@dataclass
class HathoraTTSSettings(TTSSettings):
"""Settings for Hathora TTS service.
Parameters:
speed: Speech speed multiplier (if supported by model).
config: Some models support additional config, refer to
[docs](https://models.hathora.dev) for each model to see
what is supported.
"""
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
config: list[ConfigOption] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HathoraTTSService(TTSService):
"""This service supports several different text-to-speech models hosted by Hathora.
[Documentation](https://models.hathora.dev)
"""
_settings: HathoraTTSSettings
class InputParams(BaseModel):
"""Optional input parameters for Hathora TTS configuration.
@@ -88,23 +107,21 @@ class HathoraTTSService(TTSService):
params: Configuration parameters.
**kwargs: Additional arguments passed to the parent class.
"""
super().__init__(
sample_rate=sample_rate,
**kwargs,
)
self._model = model
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
params = params or HathoraTTSService.InputParams()
self._settings = {
"speed": params.speed,
"config": params.config,
}
self.set_model_name(model)
self.set_voice(voice_id)
super().__init__(
sample_rate=sample_rate,
settings=HathoraTTSSettings(
model=model,
voice=voice_id,
language=None, # Not applicable here
speed=params.speed,
config=params.config,
),
**kwargs,
)
self._api_key = api_key or os.getenv("HATHORA_API_KEY")
self._base_url = base_url
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -131,16 +148,15 @@ class HathoraTTSService(TTSService):
url = f"{self._base_url}"
payload = {"model": self._model, "text": text}
payload = {"model": self._settings.model, "text": text}
if self._voice_id is not None:
payload["voice"] = self._voice_id
if self._settings["speed"] is not None:
payload["speed"] = self._settings["speed"]
if self._settings["config"] is not None:
if self._settings.voice is not None:
payload["voice"] = self._settings.voice
if self._settings.speed is not None:
payload["speed"] = self._settings.speed
if self._settings.config is not None:
payload["model_config"] = [
{"name": option.name, "value": option.value}
for option in self._settings["config"]
{"name": option.name, "value": option.value} for option in self._settings.config
]
yield TTSStartedFrame(context_id=context_id)

96
src/pipecat/services/hume/tts.py
View File

@@ -6,6 +6,8 @@
import base64
import os
import warnings
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import httpx
@@ -24,7 +26,8 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import WordTTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
try:
@@ -46,7 +49,22 @@ DEFAULT_HEADERS = {
}
class HumeTTSService(WordTTSService):
@dataclass
class HumeTTSSettings(TTSSettings):
"""Settings for Hume TTS service.
Parameters:
description: Natural-language acting directions (up to 100 characters).
speed: Speaking-rate multiplier (0.5-2.0).
trailing_silence: Seconds of silence to append at the end (0-5).
"""
description: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
trailing_silence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class HumeTTSService(TTSService):
"""Hume Octave Text-to-Speech service.
Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint
@@ -61,6 +79,8 @@ class HumeTTSService(WordTTSService):
- Provides metrics for Time To First Byte (TTFB) and TTS usage.
"""
_settings: HumeTTSSettings
class InputParams(BaseModel):
"""Optional synthesis parameters for Hume TTS.
@@ -101,11 +121,21 @@ class HumeTTSService(WordTTSService):
f"Hume TTS streams at {HUME_SAMPLE_RATE} Hz; configured sample_rate={sample_rate}"
)
# WordTTSService sets push_text_frames=False by default, which we want
params = params or HumeTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
settings=HumeTTSSettings(
model=None,
voice=voice_id,
language=None, # Not applicable here
description=params.description,
speed=params.speed,
trailing_silence=params.trailing_silence,
),
**kwargs,
)
@@ -114,10 +144,6 @@ class HumeTTSService(WordTTSService):
self._http_client = httpx.AsyncClient(headers=DEFAULT_HEADERS)
self._client = AsyncHumeClient(api_key=api_key, httpx_client=self._http_client)
self._params = params or HumeTTSService.InputParams()
# Store voice in the base class (mirrors other services)
self.set_voice(voice_id)
self._audio_bytes = b""
@@ -183,7 +209,10 @@ class HumeTTSService(WordTTSService):
await self.add_word_timestamps([("Reset", 0)])
async def update_setting(self, key: str, value: Any) -> None:
"""Runtime updates via `TTSUpdateSettingsFrame`.
"""Runtime updates via key/value pair.
.. deprecated:: 0.0.104
Use ``TTSUpdateSettingsFrame(delta=HumeTTSSettings(...))`` instead.
Args:
key: The name of the setting to update. Recognized keys are:
@@ -193,20 +222,29 @@ class HumeTTSService(WordTTSService):
- "trailing_silence"
value: The new value for the setting.
"""
key_l = (key or "").lower()
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'update_setting' is deprecated, use "
"'TTSUpdateSettingsFrame(delta=HumeTTSSettings(...))' instead.",
DeprecationWarning,
stacklevel=2,
)
if key_l == "voice_id":
self.set_voice(str(value))
logger.debug(f"HumeTTSService voice_id set to: {self.voice}")
elif key_l == "description":
self._params.description = None if value is None else str(value)
elif key_l == "speed":
self._params.speed = None if value is None else float(value)
elif key_l == "trailing_silence":
self._params.trailing_silence = None if value is None else float(value)
else:
# Defer unknown keys to the base class
await super().update_setting(key, value)
key_l = (key or "").lower()
known_keys = {"voice_id", "voice", "description", "speed", "trailing_silence"}
if key_l in known_keys:
kwargs: dict[str, Any] = {}
if key_l in ("voice_id", "voice"):
kwargs["voice"] = str(value)
elif key_l == "description":
kwargs["description"] = None if value is None else str(value)
elif key_l == "speed":
kwargs["speed"] = None if value is None else float(value)
elif key_l == "trailing_silence":
kwargs["trailing_silence"] = None if value is None else float(value)
await self._update_settings(HumeTTSSettings(**kwargs))
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -226,14 +264,14 @@ class HumeTTSService(WordTTSService):
# Build the request payload
utterance_kwargs: dict[str, Any] = {
"text": text,
"voice": PostedUtteranceVoiceWithId(id=self._voice_id),
"voice": PostedUtteranceVoiceWithId(id=self._settings.voice),
}
if self._params.description is not None:
utterance_kwargs["description"] = self._params.description
if self._params.speed is not None:
utterance_kwargs["speed"] = self._params.speed
if self._params.trailing_silence is not None:
utterance_kwargs["trailing_silence"] = self._params.trailing_silence
if self._settings.description is not None:
utterance_kwargs["description"] = self._settings.description
if self._settings.speed is not None:
utterance_kwargs["speed"] = self._settings.speed
if self._settings.trailing_silence is not None:
utterance_kwargs["trailing_silence"] = self._settings.trailing_silence
utterance = PostedUtterance(**utterance_kwargs)
@@ -257,7 +295,7 @@ class HumeTTSService(WordTTSService):
# Use version "2" by default if no description is provided
# Version "1" is needed when description is used
version = "1" if self._params.description is not None else "2"
version = "1" if self._settings.description is not None else "2"
# Track the duration of this utterance based on the last timestamp
utterance_duration = 0.0

14
src/pipecat/services/image_service.py
View File

@@ -11,11 +11,12 @@ text prompts into images.
"""
from abc import abstractmethod
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from pipecat.frames.frames import Frame, TextFrame
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import ImageGenSettings
class ImageGenService(AIService):
@@ -26,13 +27,20 @@ class ImageGenService(AIService):
generation functionality using their specific AI service.
"""
def __init__(self, **kwargs):
def __init__(self, *, settings: Optional[ImageGenSettings] = None, **kwargs):
"""Initialize the image generation service.
Args:
settings: The runtime-updatable settings for the image generation service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or ImageGenSettings(),
**kwargs,
)
# Renders the image. Returns an Image object.
@abstractmethod

326
src/pipecat/services/inworld/tts.py
View File

@@ -17,7 +17,8 @@ import asyncio
import base64
import json
import uuid
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Tuple
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, List, Literal, Mapping, Optional, Tuple
import aiohttp
import websockets
@@ -28,6 +29,8 @@ from pipecat import version as pipecat_version
USER_AGENT = f"pipecat/{pipecat_version()}"
from pydantic import BaseModel
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
try:
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
@@ -48,17 +51,66 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService, WordTTSService
from pipecat.services.tts_service import AudioContextTTSService, TextAggregationMode, TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
class InworldHttpTTSService(WordTTSService):
@dataclass
class InworldTTSSettings(TTSSettings):
"""Settings for Inworld TTS services.
Parameters:
audio_encoding: Audio encoding format (e.g. LINEAR16).
audio_sample_rate: Audio sample rate in Hz.
speaking_rate: Speaking rate for speech synthesis.
temperature: Temperature for speech synthesis.
auto_mode: Whether to use auto mode. Recommended when texts are sent
in full sentences/phrases. When enabled, the server controls
flushing of buffered text to achieve minimal latency while
maintaining high quality audio output. If None (default),
automatically set based on aggregate_sentences.
apply_text_normalization: Whether to apply text normalization.
timestamp_transport_strategy: Strategy for timestamp transport ("ASYNC" or "SYNC").
"""
audio_encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaking_rate: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
auto_mode: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
apply_text_normalization: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
timestamp_transport_strategy: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {
"voice_id": "voice",
"voiceId": "voice",
"modelId": "model",
"applyTextNormalization": "apply_text_normalization",
"autoMode": "auto_mode",
"timestampTransportStrategy": "timestamp_transport_strategy",
}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "InworldTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested ``audioConfig``."""
flat = dict(settings)
nested = flat.pop("audioConfig", None)
if isinstance(nested, dict):
flat.setdefault("audio_encoding", nested.get("audioEncoding"))
flat.setdefault("audio_sample_rate", nested.get("sampleRateHertz"))
flat.setdefault("speaking_rate", nested.get("speakingRate"))
return super().from_mapping(flat)
class InworldHttpTTSService(TTSService):
"""Inworld AI HTTP-based TTS service.
Supports both streaming and non-streaming modes via the `streaming` parameter.
Outputs LINEAR16 audio at configurable sample rates with word-level timestamps.
"""
_settings: InworldTTSSettings
class InputParams(BaseModel):
"""Input parameters for Inworld TTS configuration.
@@ -70,7 +122,7 @@ class InworldHttpTTSService(WordTTSService):
temperature: Optional[float] = None
speaking_rate: Optional[float] = None
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = None
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = "ASYNC"
def __init__(
self,
@@ -98,15 +150,28 @@ class InworldHttpTTSService(WordTTSService):
params: Input parameters for Inworld TTS configuration.
**kwargs: Additional arguments passed to the parent class.
"""
params = params or InworldHttpTTSService.InputParams()
super().__init__(
push_text_frames=False,
push_stop_frames=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=None, # Not applicable for HTTP TTS
apply_text_normalization=None, # Not applicable for HTTP TTS
),
**kwargs,
)
params = params or InworldHttpTTSService.InputParams()
self._api_key = api_key
self._session = aiohttp_session
self._streaming = streaming
@@ -117,27 +182,8 @@ class InworldHttpTTSService(WordTTSService):
else:
self._base_url = "https://api.inworld.ai/tts/v1/voice"
self._settings = {
"voiceId": voice_id,
"modelId": model,
"audioConfig": {
"audioEncoding": encoding,
"sampleRateHertz": 0,
},
}
if params.temperature is not None:
self._settings["temperature"] = params.temperature
if params.speaking_rate is not None:
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
self._cumulative_time = 0.0
self.set_voice(voice_id)
self.set_model_name(model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -153,7 +199,7 @@ class InworldHttpTTSService(WordTTSService):
frame: The start frame.
"""
await super().start(frame)
self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
async def stop(self, frame: EndFrame):
"""Stop the Inworld TTS service.
@@ -232,20 +278,27 @@ class InworldHttpTTSService(WordTTSService):
"""
logger.debug(f"{self}: Generating TTS [{text}] (streaming={self._streaming})")
audio_config = {
"audioEncoding": self._settings.audio_encoding,
"sampleRateHertz": self._settings.audio_sample_rate,
}
if self._settings.speaking_rate is not None:
audio_config["speakingRate"] = self._settings.speaking_rate
payload = {
"text": text,
"voiceId": self._settings["voiceId"],
"modelId": self._settings["modelId"],
"audioConfig": self._settings["audioConfig"],
"voiceId": self._settings.voice,
"modelId": self._settings.model,
"audioConfig": audio_config,
}
if "temperature" in self._settings:
payload["temperature"] = self._settings["temperature"]
if self._settings.temperature is not None:
payload["temperature"] = self._settings.temperature
# Use WORD timestamps for simplicity and correct spacing/capitalization
payload["timestampType"] = self._timestamp_type
if "timestampTransportStrategy" in self._settings:
payload["timestampTransportStrategy"] = self._settings["timestampTransportStrategy"]
if self._settings.timestamp_transport_strategy is not None:
payload["timestampTransportStrategy"] = self._settings.timestamp_transport_strategy
request_id = str(uuid.uuid4())
headers = {
@@ -411,7 +464,7 @@ class InworldHttpTTSService(WordTTSService):
)
class InworldTTSService(AudioContextWordTTSService):
class InworldTTSService(AudioContextTTSService):
"""Inworld AI WebSocket-based TTS service.
Uses bidirectional WebSocket for lower latency streaming. Supports multiple
@@ -419,6 +472,8 @@ class InworldTTSService(AudioContextWordTTSService):
with word-level timestamps.
"""
_settings: InworldTTSSettings
class InputParams(BaseModel):
"""Input parameters for Inworld WebSocket TTS configuration.
@@ -442,7 +497,7 @@ class InworldTTSService(AudioContextWordTTSService):
max_buffer_delay_ms: Optional[int] = None
buffer_char_threshold: Optional[int] = None
auto_mode: Optional[bool] = True
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = None
timestamp_transport_strategy: Optional[Literal["ASYNC", "SYNC"]] = "ASYNC"
def __init__(
self,
@@ -454,7 +509,8 @@ class InworldTTSService(AudioContextWordTTSService):
sample_rate: Optional[int] = None,
encoding: str = "LINEAR16",
params: InputParams = None,
aggregate_sentences: bool = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
append_trailing_space: bool = True,
**kwargs: Any,
):
@@ -468,48 +524,45 @@ class InworldTTSService(AudioContextWordTTSService):
sample_rate: Audio sample rate in Hz.
encoding: Audio encoding format.
params: Input parameters for Inworld WebSocket TTS configuration.
aggregate_sentences: Whether to aggregate sentences before synthesis.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
append_trailing_space: Whether to append a trailing space to text before sending to TTS.
**kwargs: Additional arguments passed to the parent class.
"""
params = params or InworldTTSService.InputParams()
super().__init__(
push_text_frames=False,
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
append_trailing_space=append_trailing_space,
settings=InworldTTSSettings(
model=model,
voice=voice_id,
language=None,
audio_encoding=encoding,
audio_sample_rate=0,
speaking_rate=params.speaking_rate,
temperature=params.temperature,
apply_text_normalization=params.apply_text_normalization,
timestamp_transport_strategy=params.timestamp_transport_strategy,
auto_mode=params.auto_mode if params.auto_mode is not None else aggregate_sentences,
),
**kwargs,
)
params = params or InworldTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings: Dict[str, Any] = {
"voiceId": voice_id,
"modelId": model,
"audioConfig": {
"audioEncoding": encoding,
"sampleRateHertz": 0,
},
}
self._timestamp_type = "WORD"
if params.temperature is not None:
self._settings["temperature"] = params.temperature
if params.speaking_rate is not None:
self._settings["audioConfig"]["speakingRate"] = params.speaking_rate
if params.apply_text_normalization is not None:
self._settings["applyTextNormalization"] = params.apply_text_normalization
if params.timestamp_transport_strategy is not None:
self._settings["timestampTransportStrategy"] = params.timestamp_transport_strategy
if params.auto_mode is not None:
self._settings["autoMode"] = params.auto_mode
else:
self._settings["autoMode"] = aggregate_sentences
self._buffer_settings = {
"maxBufferDelayMs": params.max_buffer_delay_ms,
"bufferCharThreshold": params.buffer_char_threshold,
@@ -517,7 +570,6 @@ class InworldTTSService(AudioContextWordTTSService):
self._receive_task = None
self._keepalive_task = None
self._context_id = None
# Track cumulative time across generations for monotonic timestamps within a turn.
# When auto_mode is enabled, the server controls generations and timestamps reset
@@ -527,9 +579,6 @@ class InworldTTSService(AudioContextWordTTSService):
# Track the end time of the last word in the current generation
self._generation_end_time = 0.0
self.set_voice(voice_id)
self.set_model_name(model)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -545,7 +594,7 @@ class InworldTTSService(AudioContextWordTTSService):
frame: The start frame.
"""
await super().start(frame)
self._settings["audioConfig"]["sampleRateHertz"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -573,9 +622,10 @@ class InworldTTSService(AudioContextWordTTSService):
keeping the context open for subsequent text. The context is only
closed on interruption, disconnect, or end of session.
"""
if self._context_id and self._websocket:
logger.trace(f"Flushing audio for context {self._context_id}")
await self._send_flush(self._context_id)
context_id = self.get_active_audio_context_id()
if context_id and self._websocket:
logger.trace(f"Flushing audio for context {context_id}")
await self._send_flush(context_id)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame and handle state changes.
@@ -633,29 +683,23 @@ class InworldTTSService(AudioContextWordTTSService):
return word_times
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle an interruption from the Inworld WebSocket TTS service.
Args:
frame: The interruption frame.
direction: The direction of the interruption.
"""
old_context_id = self._context_id
logger.trace(f"{self}: Handling interruption, old context: {old_context_id}")
await super()._handle_interruption(frame, direction)
if old_context_id and self._websocket:
logger.trace(f"{self}: Closing context {old_context_id} due to interruption")
async def _close_context(self, context_id: str):
if context_id and self._websocket:
logger.info(f"{self}: Closing context {context_id} due to interruption or completion")
try:
await self._send_close_context(old_context_id)
await self._send_close_context(context_id)
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
self._context_id = None
self._cumulative_time = 0.0
self._generation_end_time = 0.0
logger.trace(f"{self}: Interruption handled, context reset to None")
async def on_audio_context_interrupted(self, context_id: str):
"""Callback invoked when an audio context has been interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Callback invoked when an audio context has been completed."""
await self._close_context(context_id)
def _get_websocket(self):
"""Get the websocket for the Inworld WebSocket TTS service.
@@ -701,6 +745,21 @@ class InworldTTSService(AudioContextWordTTSService):
await self._disconnect_websocket()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to the Inworld WebSocket TTS service.
@@ -736,9 +795,10 @@ class InworldTTSService(AudioContextWordTTSService):
if self._websocket:
logger.debug("Disconnecting from Inworld WebSocket TTS")
if self._context_id:
context_id = self.get_active_audio_context_id()
if context_id:
try:
await self._send_close_context(self._context_id)
await self._send_close_context(context_id)
except Exception:
pass
await self._websocket.close()
@@ -746,7 +806,7 @@ class InworldTTSService(AudioContextWordTTSService):
except Exception as e:
await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
finally:
self._context_id = None
await self.remove_active_audio_context()
self._websocket = None
self._cumulative_time = 0.0
self._generation_end_time = 0.0
@@ -772,7 +832,7 @@ class InworldTTSService(AudioContextWordTTSService):
]
logger.debug(
f"{self}: Received message types={msg_types}, ctx_id={ctx_id}, "
f"current_ctx={self._context_id}, available={self.audio_context_available(ctx_id) if ctx_id else 'N/A'}"
f"current_ctx={self.get_active_audio_context_id()}, available={self.audio_context_available(ctx_id) if ctx_id else 'N/A'}"
)
# Check for errors
@@ -784,7 +844,9 @@ class InworldTTSService(AudioContextWordTTSService):
# Handle "Context not found" error (code 5)
# This can happen when a keepalive message is sent but no context is available.
if error_code == 5 and "not found" in error_msg.lower():
logger.debug(f"{self}: Context {ctx_id or self._context_id} not found.")
logger.debug(
f"{self}: Context {ctx_id or self.get_active_audio_context_id()} not found."
)
continue
# For other errors, push error frame
@@ -799,11 +861,9 @@ class InworldTTSService(AudioContextWordTTSService):
# If the context isn't available but matches our current context ID,
# recreate it (handles race conditions during interruption recovery).
if ctx_id and not self.audio_context_available(ctx_id):
if self._context_id == ctx_id:
logger.trace(
f"{self}: Recreating audio context for current context: {self._context_id}"
)
await self.create_audio_context(self._context_id)
if self.get_active_audio_context_id() == ctx_id:
logger.trace(f"{self}: Recreating audio context for current context: {ctx_id}")
await self.create_audio_context(ctx_id)
else:
# This is a message from an old/closed context - skip it
logger.trace(f"{self}: Skipping message from unavailable context: {ctx_id}")
@@ -849,8 +909,8 @@ class InworldTTSService(AudioContextWordTTSService):
logger.trace(f"{self}: Context closed on server: {ctx_id}")
await self.stop_ttfb_metrics()
# Only reset if this is our current context
if ctx_id == self._context_id:
self._context_id = None
if ctx_id == self.get_active_audio_context_id():
self.reset_active_audio_context()
if ctx_id and self.audio_context_available(ctx_id):
await self.remove_audio_context(ctx_id)
await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)], ctx_id)
@@ -862,12 +922,13 @@ class InworldTTSService(AudioContextWordTTSService):
await asyncio.sleep(KEEPALIVE_SLEEP)
try:
if self._websocket and self._websocket.state is State.OPEN:
if self._context_id:
context_id = self.get_active_audio_context_id()
if context_id:
keepalive_message = {
"send_text": {"text": ""},
"contextId": self._context_id,
"contextId": context_id,
}
logger.trace(f"Sending keepalive for context {self._context_id}")
logger.trace(f"Sending keepalive for context {context_id}")
else:
keepalive_message = {"send_text": {"text": ""}}
logger.trace("Sending keepalive without context")
@@ -882,22 +943,29 @@ class InworldTTSService(AudioContextWordTTSService):
Args:
context_id: The context ID.
"""
audio_config = {
"audioEncoding": self._settings.audio_encoding,
"sampleRateHertz": self._settings.audio_sample_rate,
}
if self._settings.speaking_rate is not None:
audio_config["speakingRate"] = self._settings.speaking_rate
create_config: Dict[str, Any] = {
"voiceId": self._settings["voiceId"],
"modelId": self._settings["modelId"],
"audioConfig": self._settings["audioConfig"],
"voiceId": self._settings.voice,
"modelId": self._settings.model,
"audioConfig": audio_config,
}
if "temperature" in self._settings:
create_config["temperature"] = self._settings["temperature"]
if "applyTextNormalization" in self._settings:
create_config["applyTextNormalization"] = self._settings["applyTextNormalization"]
if "autoMode" in self._settings:
create_config["autoMode"] = self._settings["autoMode"]
if "timestampTransportStrategy" in self._settings:
create_config["timestampTransportStrategy"] = self._settings[
"timestampTransportStrategy"
]
if self._settings.temperature is not None:
create_config["temperature"] = self._settings.temperature
if self._settings.apply_text_normalization is not None:
create_config["applyTextNormalization"] = self._settings.apply_text_normalization
if self._settings.auto_mode is not None:
create_config["autoMode"] = self._settings.auto_mode
if self._settings.timestamp_transport_strategy is not None:
create_config["timestampTransportStrategy"] = (
self._settings.timestamp_transport_strategy
)
# Set buffer settings for timely audio generation.
# Use provided values or defaults that work well for streaming LLM output.
@@ -938,20 +1006,6 @@ class InworldTTSService(AudioContextWordTTSService):
msg = {"close_context": {}, "contextId": context_id}
await self.send_with_retry(json.dumps(msg), self._report_error)
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request in case we don't have one already in progress.
Returns:
A unique string identifier for the TTS context.
"""
# If a context ID does not exist, create a new one.
# If an ID exists, continue using the current ID.
# When interruptions happen, user speech results in
# an interruption, which resets the context ID.
if not self._context_id:
return str(uuid.uuid4())
return self._context_id
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio for the given text using the Inworld WebSocket TTS service.
@@ -970,19 +1024,13 @@ class InworldTTSService(AudioContextWordTTSService):
await self._connect()
try:
if not self._context_id:
if not self.has_active_audio_context():
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
self._context_id = context_id
logger.trace(f"{self}: Creating new context {self._context_id}")
await self.create_audio_context(self._context_id)
await self._send_context(self._context_id)
elif not self.audio_context_available(self._context_id):
# Context exists on server but local tracking was removed
logger.trace(f"{self}: Recreating local audio context {self._context_id}")
await self.create_audio_context(self._context_id)
await self.create_audio_context(context_id)
await self._send_context(context_id)
await self._send_text(self._context_id, text)
await self._send_text(context_id, text)
await self.start_tts_usage_metrics(text)
except Exception as e:

30
src/pipecat/services/kokoro/tts.py
View File

@@ -7,6 +7,7 @@
"""Kokoro TTS service implementation using kokoro-onnx."""
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import AsyncGenerator, Optional
@@ -22,6 +23,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -87,6 +89,17 @@ def language_to_kokoro_language(language: Language) -> str:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class KokoroTTSSettings(TTSSettings):
"""Settings for the Kokoro TTS service.
Parameters:
lang_code: Kokoro language code for synthesis.
"""
lang_code: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class KokoroTTSService(TTSService):
"""Kokoro TTS service implementation.
@@ -94,6 +107,8 @@ class KokoroTTSService(TTSService):
Automatically downloads model files on first use.
"""
_settings: KokoroTTSSettings
class InputParams(BaseModel):
"""Input parameters for Kokoro TTS configuration.
@@ -122,11 +137,18 @@ class KokoroTTSService(TTSService):
**kwargs: Additional arguments passed to parent `TTSService`.
"""
super().__init__(**kwargs)
params = params or KokoroTTSService.InputParams()
self._voice_id = voice_id
super().__init__(
settings=KokoroTTSSettings(
model=None,
voice=voice_id,
language=language_to_kokoro_language(params.language),
lang_code=language_to_kokoro_language(params.language),
),
**kwargs,
)
self._lang_code = language_to_kokoro_language(params.language)
model = Path(model_path) if model_path else KOKORO_CACHE_DIR / "kokoro-v1.0.onnx"
@@ -161,7 +183,7 @@ class KokoroTTSService(TTSService):
yield TTSStartedFrame(context_id=context_id)
stream = self._kokoro.create_stream(
text, voice=self._voice_id, lang=self._lang_code, speed=1.0
text, voice=self._settings.voice, lang=self._lang_code, speed=1.0
)
async for samples, sample_rate in stream:

82
src/pipecat/services/llm_service.py
View File

@@ -44,6 +44,7 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
UserImageRequestFrame,
)
@@ -58,8 +59,10 @@ from pipecat.processors.aggregators.llm_response import (
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import LLMSettings
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionLLMServiceMixin
from pipecat.utils.context.llm_context_summarization import (
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMContextSummarizationUtil,
)
@@ -172,12 +175,18 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
logger.info(f"Starting {len(function_calls)} function calls")
"""
_settings: LLMSettings
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
# However, subclasses should override this with a more specific adapter when necessary.
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
def __init__(
self, run_in_parallel: bool = True, function_call_timeout_secs: float = 10.0, **kwargs
self,
run_in_parallel: bool = True,
function_call_timeout_secs: float = 10.0,
settings: Optional[LLMSettings] = None,
**kwargs,
):
"""Initialize the LLM service.
@@ -186,10 +195,17 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
Defaults to True.
function_call_timeout_secs: Timeout in seconds for deferred function calls.
Defaults to 10.0 seconds.
settings: The runtime-updatable settings for the LLM service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or LLMSettings(),
**kwargs,
)
self._run_in_parallel = run_in_parallel
self._function_call_timeout_secs = function_call_timeout_secs
self._filter_incomplete_user_turns: bool = False
@@ -307,34 +323,30 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
await self._cancel_sequential_runner_task()
await self._cancel_summary_task()
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update LLM service settings.
Handles turn completion settings specially since they are not model
parameters and should not be passed to the underlying LLM API.
async def _update_settings(self, delta: LLMSettings) -> dict[str, Any]:
"""Apply a settings delta, handling turn-completion fields.
Args:
settings: Dictionary of settings to update.
"""
# Turn completion settings to extract (not model parameters)
turn_completion_keys = {"filter_incomplete_user_turns", "user_turn_completion_config"}
delta: An LLM settings delta.
# Handle turn completion settings
if "filter_incomplete_user_turns" in settings:
self._filter_incomplete_user_turns = settings["filter_incomplete_user_turns"]
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if "filter_incomplete_user_turns" in changed:
self._filter_incomplete_user_turns = (
self._settings.filter_incomplete_user_turns or False
)
logger.info(
f"{self}: Incomplete turn filtering {'enabled' if self._filter_incomplete_user_turns else 'disabled'}"
f"{self}: Incomplete turn filtering "
f"{'enabled' if self._filter_incomplete_user_turns else 'disabled'}"
)
# Configure the mixin with config object
if self._filter_incomplete_user_turns and "user_turn_completion_config" in settings:
self.set_user_turn_completion_config(settings["user_turn_completion_config"])
if "user_turn_completion_config" in changed and self._filter_incomplete_user_turns:
self.set_user_turn_completion_config(self._settings.user_turn_completion_config)
# Remove turn completion settings before passing to parent
settings = {k: v for k, v in settings.items() if k not in turn_completion_keys}
# Let the parent handle remaining model parameters
await super()._update_settings(settings)
return changed
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame.
@@ -349,6 +361,21 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
await self._handle_interruptions(frame)
elif isinstance(frame, LLMConfigureOutputFrame):
self._skip_tts = frame.skip_tts
elif isinstance(frame, LLMUpdateSettingsFrame):
if frame.delta is not None:
await self._update_settings(frame.delta)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via LLMUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.104, use LLMUpdateSettingsFrame(delta=LLMSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
delta = type(self._settings).from_mapping(frame.settings)
await self._update_settings(delta)
elif isinstance(frame, LLMContextSummaryRequestFrame):
await self._handle_summary_request(frame)
@@ -410,8 +437,15 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
last_index = -1
error = None
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try:
summary, last_index = await self._generate_summary(frame)
summary, last_index = await asyncio.wait_for(
self._generate_summary(frame),
timeout=timeout,
)
except asyncio.TimeoutError:
await self.push_error(error_msg=f"Context summarization timed out after {timeout}s")
except Exception as e:
error = f"Error generating context summary: {e}"
await self.push_error(error, exception=e)

54
src/pipecat/services/lmnt/tts.py
View File

@@ -7,7 +7,8 @@
"""LMNT text-to-speech service implementation."""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -71,6 +73,17 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class LmntTTSSettings(TTSSettings):
"""Settings for LMNT TTS service.
Parameters:
format: Audio output format. Defaults to "raw".
"""
format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class LmntTTSService(InterruptibleTTSService):
"""LMNT real-time text-to-speech service.
@@ -79,6 +92,8 @@ class LmntTTSService(InterruptibleTTSService):
language settings.
"""
_settings: LmntTTSSettings
def __init__(
self,
*,
@@ -103,16 +118,16 @@ class LmntTTSService(InterruptibleTTSService):
push_stop_frames=True,
pause_frame_processing=True,
sample_rate=sample_rate,
settings=LmntTTSSettings(
model=model,
voice=voice_id,
language=self.language_to_service_language(language),
format="raw",
),
**kwargs,
)
self._api_key = api_key
self.set_voice(voice_id)
self.set_model_name(model)
self._settings = {
"language": self.language_to_service_language(language),
"format": "raw", # Use raw format for direct PCM data
}
self._receive_task = None
self._context_id: Optional[str] = None
@@ -190,6 +205,23 @@ class LmntTTSService(InterruptibleTTSService):
await self._disconnect_websocket()
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Args:
delta: A :class:`TTSSettings` (or ``LmntTTSSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if changed:
await self._disconnect()
await self._connect()
return changed
async def _connect_websocket(self):
"""Connect to LMNT websocket."""
try:
@@ -201,11 +233,11 @@ class LmntTTSService(InterruptibleTTSService):
# Build initial connection message
init_msg = {
"X-API-Key": self._api_key,
"voice": self._voice_id,
"format": self._settings["format"],
"voice": self._settings.voice,
"format": self._settings.format,
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"model": self.model_name,
"language": self._settings.language,
"model": self._settings.model,
}
# Connect to LMNT's websocket directly

178
src/pipecat/services/minimax/tts.py
View File

@@ -11,7 +11,8 @@ for streaming text-to-speech synthesis.
"""
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Mapping, Optional
import aiohttp
from loguru import logger
@@ -25,6 +26,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -85,6 +87,69 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class MiniMaxTTSSettings(TTSSettings):
"""Settings for MiniMax TTS service.
Parameters:
stream: Whether to use streaming mode.
speed: Speech speed (range: 0.5 to 2.0).
volume: Speech volume (range: 0 to 10).
pitch: Pitch adjustment (range: -12 to 12).
emotion: Emotional tone (options: "happy", "sad", "angry", "fearful",
"disgusted", "surprised", "calm", "fluent").
text_normalization: Enable text normalization (Chinese/English).
latex_read: Enable LaTeX formula reading.
audio_bitrate: Audio bitrate in bps.
audio_format: Audio output format.
audio_channel: Number of audio channels.
audio_sample_rate: Audio sample rate in Hz.
language_boost: Language boost string for multilingual support.
"""
stream: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
volume: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
emotion: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
text_normalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
latex_read: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_bitrate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_channel: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
audio_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_boost: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@classmethod
def from_mapping(cls, settings: Mapping[str, Any]) -> "MiniMaxTTSSettings":
"""Construct settings from a plain dict, destructuring legacy nested dicts.
Handles ``voice_setting`` (with ``vol`` → ``volume`` rename) and
``audio_setting`` (with prefixed field mapping).
"""
flat = dict(settings)
voice = flat.pop("voice_setting", None)
if isinstance(voice, dict):
flat.setdefault("speed", voice.get("speed"))
flat.setdefault("volume", voice.get("vol"))
flat.setdefault("pitch", voice.get("pitch"))
flat.setdefault("emotion", voice.get("emotion"))
flat.setdefault("text_normalization", voice.get("text_normalization"))
flat.setdefault("latex_read", voice.get("latex_read"))
audio = flat.pop("audio_setting", None)
if isinstance(audio, dict):
flat.setdefault("audio_bitrate", audio.get("bitrate"))
flat.setdefault("audio_format", audio.get("format"))
flat.setdefault("audio_channel", audio.get("channel"))
flat.setdefault("audio_sample_rate", audio.get("sample_rate"))
return super().from_mapping(flat)
class MiniMaxHttpTTSService(TTSService):
"""Text-to-speech service using MiniMax's T2A (Text-to-Audio) API.
@@ -96,6 +161,8 @@ class MiniMaxHttpTTSService(TTSService):
https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643
"""
_settings: MiniMaxTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for MiniMax TTS.
@@ -160,41 +227,40 @@ class MiniMaxHttpTTSService(TTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or MiniMaxHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=MiniMaxTTSSettings(
model=model,
voice=voice_id,
language=None,
stream=True,
speed=params.speed,
volume=params.volume,
pitch=params.pitch,
language_boost=None,
emotion=None,
text_normalization=None,
latex_read=None,
audio_bitrate=128000,
audio_format="pcm",
audio_channel=1,
audio_sample_rate=0,
),
**kwargs,
)
self._api_key = api_key
self._group_id = group_id
self._base_url = f"{base_url}?GroupId={group_id}"
self._session = aiohttp_session
self._model_name = model
self._voice_id = voice_id
# Create voice settings
self._settings = {
"stream": True,
"voice_setting": {
"speed": params.speed,
"vol": params.volume,
"pitch": params.pitch,
},
"audio_setting": {
"bitrate": 128000,
"format": "pcm",
"channel": 1,
},
}
# Set voice and model
self.set_voice(voice_id)
self.set_model_name(model)
# Add language boost if provided
if params.language:
service_lang = self.language_to_service_language(params.language)
if service_lang:
self._settings["language_boost"] = service_lang
self._settings.language_boost = service_lang
# Add optional emotion if provided
if params.emotion:
@@ -210,7 +276,7 @@ class MiniMaxHttpTTSService(TTSService):
"fluent",
]
if params.emotion in supported_emotions:
self._settings["voice_setting"]["emotion"] = params.emotion
self._settings.emotion = params.emotion
else:
logger.warning(
f"Unsupported emotion: {params.emotion}. Supported emotions: {supported_emotions}"
@@ -226,15 +292,15 @@ class MiniMaxHttpTTSService(TTSService):
"Parameter `english_normalization` is deprecated and will be removed in a future version. Use `text_normalization` instead.",
DeprecationWarning,
)
self._settings["voice_setting"]["text_normalization"] = params.english_normalization
self._settings.text_normalization = params.english_normalization
# Add text_normalization if provided (corrected parameter name)
if params.text_normalization is not None:
self._settings["voice_setting"]["text_normalization"] = params.text_normalization
self._settings.text_normalization = params.text_normalization
# Add latex_read if provided
if params.latex_read is not None:
self._settings["voice_setting"]["latex_read"] = params.latex_read
self._settings.latex_read = params.latex_read
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -255,24 +321,6 @@ class MiniMaxHttpTTSService(TTSService):
"""
return language_to_minimax_language(language)
def set_model_name(self, model: str):
"""Set the TTS model to use.
Args:
model: The model name to use for synthesis.
"""
self._model_name = model
def set_voice(self, voice: str):
"""Set the voice to use.
Args:
voice: The voice identifier to use for synthesis.
"""
self._voice_id = voice
if "voice_setting" in self._settings:
self._settings["voice_setting"]["voice_id"] = voice
async def start(self, frame: StartFrame):
"""Start the MiniMax TTS service.
@@ -280,7 +328,7 @@ class MiniMaxHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["audio_setting"]["sample_rate"] = self.sample_rate
self._settings.audio_sample_rate = self.sample_rate
logger.debug(f"MiniMax TTS initialized with sample_rate: {self.sample_rate}")
@traced_tts
@@ -302,10 +350,38 @@ class MiniMaxHttpTTSService(TTSService):
"Authorization": f"Bearer {self._api_key}",
}
# Build voice_setting dict for API
voice_setting = {
"voice_id": self._settings.voice,
"speed": self._settings.speed,
"vol": self._settings.volume,
"pitch": self._settings.pitch,
}
if self._settings.emotion is not None:
voice_setting["emotion"] = self._settings.emotion
if self._settings.text_normalization is not None:
voice_setting["text_normalization"] = self._settings.text_normalization
if self._settings.latex_read is not None:
voice_setting["latex_read"] = self._settings.latex_read
# Build audio_setting dict for API
audio_setting = {
"bitrate": self._settings.audio_bitrate,
"format": self._settings.audio_format,
"channel": self._settings.audio_channel,
"sample_rate": self._settings.audio_sample_rate,
}
# Create payload from settings
payload = self._settings.copy()
payload["model"] = self._model_name
payload["text"] = text
payload = {
"stream": self._settings.stream,
"voice_setting": voice_setting,
"audio_setting": audio_setting,
"model": self._settings.model,
"text": text,
}
if self._settings.language_boost is not None:
payload["language_boost"] = self._settings.language_boost
try:
await self.start_ttfb_metrics()

18
src/pipecat/services/mistral/llm.py
View File

@@ -180,24 +180,24 @@ class MistralLLMService(OpenAILLMService):
fixed_messages = self._apply_mistral_fixups(params_from_context["messages"])
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"messages": fixed_messages,
"tools": params_from_context["tools"],
"tool_choice": params_from_context["tool_choice"],
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
}
# Handle Mistral-specific parameter mapping
# Mistral uses "random_seed" instead of "seed"
if self._settings["seed"]:
params["random_seed"] = self._settings["seed"]
if self._settings.seed:
params["random_seed"] = self._settings.seed
# Add any extra parameters
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params

15
src/pipecat/services/moondream/vision.py
View File

@@ -11,6 +11,7 @@ for image analysis and description generation.
"""
import asyncio
from dataclasses import dataclass
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
VisionFullResponseStartFrame,
VisionTextFrame,
)
from pipecat.services.settings import VisionSettings
from pipecat.services.vision_service import VisionService
try:
@@ -60,6 +62,15 @@ def detect_device():
return torch.device("cpu"), torch.float32
@dataclass
class MoondreamSettings(VisionSettings):
"""Settings for the Moondream vision service.
Parameters:
model: Moondream model identifier.
"""
class MoondreamService(VisionService):
"""Moondream vision-language model service.
@@ -79,9 +90,7 @@ class MoondreamService(VisionService):
use_cpu: Whether to force CPU usage instead of hardware acceleration.
**kwargs: Additional arguments passed to the parent VisionService.
"""
super().__init__(**kwargs)
self.set_model_name(model)
super().__init__(settings=MoondreamSettings(model=model), **kwargs)
if not use_cpu:
device, dtype = detect_device()

112
src/pipecat/services/neuphonic/tts.py
View File

@@ -13,7 +13,8 @@ text-to-speech API for real-time audio synthesis.
import asyncio
import base64
import json
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
import aiohttp
from loguru import logger
@@ -34,7 +35,8 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -72,6 +74,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class NeuphonicTTSSettings(TTSSettings):
"""Settings for Neuphonic TTS service.
Parameters:
speed: Speech speed multiplier. Defaults to 1.0.
encoding: Audio encoding format.
sampling_rate: Audio sample rate.
"""
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
encoding: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sampling_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NeuphonicTTSService(InterruptibleTTSService):
"""Neuphonic real-time text-to-speech service using WebSocket streaming.
@@ -80,6 +97,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
parameters for high-quality speech generation.
"""
_settings: NeuphonicTTSSettings
class InputParams(BaseModel):
"""Input parameters for Neuphonic TTS configuration.
@@ -100,7 +119,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
sample_rate: Optional[int] = 22050,
encoding: str = "pcm_linear",
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize the Neuphonic TTS service.
@@ -112,28 +132,35 @@ class NeuphonicTTSService(InterruptibleTTSService):
sample_rate: Audio sample rate in Hz. Defaults to 22050.
encoding: Audio encoding format. Defaults to "pcm_linear".
params: Additional input parameters for TTS configuration.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
params = params or NeuphonicTTSService.InputParams()
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_stop_frames=True,
stop_frame_timeout_s=2.0,
sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
language=self.language_to_service_language(params.language),
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
voice=voice_id,
),
**kwargs,
)
params = params or NeuphonicTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = {
"lang_code": self.language_to_service_language(params.language),
"speed": params.speed,
"encoding": encoding,
"sampling_rate": sample_rate,
}
self.set_voice(voice_id)
self._cumulative_time = 0
@@ -160,15 +187,14 @@ class NeuphonicTTSService(InterruptibleTTSService):
"""
return language_to_neuphonic_lang_code(language)
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect with new configuration."""
if "voice_id" in settings:
self.set_voice(settings["voice_id"])
await super()._update_settings(settings)
await self._disconnect()
await self._connect()
logger.info(f"Switching TTS to settings: [{self._settings}]")
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect with new configuration."""
changed = await super()._update_settings(delta)
if changed:
await self._disconnect()
await self._connect()
logger.info(f"Switching TTS to settings: [{self._settings}]")
return changed
async def start(self, frame: StartFrame):
"""Start the Neuphonic TTS service.
@@ -266,8 +292,11 @@ class NeuphonicTTSService(InterruptibleTTSService):
logger.debug("Connecting to Neuphonic")
tts_config = {
**self._settings,
"voice_id": self._voice_id,
"lang_code": self._settings.language,
"speed": self._settings.speed,
"encoding": self._settings.encoding,
"sampling_rate": self._settings.sampling_rate,
"voice_id": self._settings.voice,
}
query_params = []
@@ -275,7 +304,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
if value is not None:
query_params.append(f"{key}={value}")
url = f"{self._url}/speak/{self._settings['lang_code']}"
url = f"{self._url}/speak/{self._settings.language}"
if query_params:
url += f"?{'&'.join(query_params)}"
@@ -384,6 +413,8 @@ class NeuphonicHttpTTSService(TTSService):
HTTP-based communication over WebSocket connections.
"""
_settings: NeuphonicTTSSettings
class InputParams(BaseModel):
"""Input parameters for Neuphonic HTTP TTS configuration.
@@ -419,17 +450,24 @@ class NeuphonicHttpTTSService(TTSService):
params: Additional input parameters for TTS configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NeuphonicHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NeuphonicTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(params.language) or "en",
speed=params.speed,
encoding=encoding,
sampling_rate=sample_rate,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = url.rstrip("/")
self._lang_code = self.language_to_service_language(params.language) or "en"
self._speed = params.speed
self._encoding = encoding
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -513,7 +551,7 @@ class NeuphonicHttpTTSService(TTSService):
"""
logger.debug(f"Generating TTS: [{text}]")
url = f"{self._base_url}/sse/speak/{self._lang_code}"
url = f"{self._base_url}/sse/speak/{self._settings.language}"
headers = {
"X-API-KEY": self._api_key,
@@ -522,14 +560,14 @@ class NeuphonicHttpTTSService(TTSService):
payload = {
"text": text,
"lang_code": self._lang_code,
"encoding": self._encoding,
"lang_code": self._settings.language,
"encoding": self._settings.encoding,
"sampling_rate": self.sample_rate,
"speed": self._speed,
"speed": self._settings.speed,
}
if self._voice_id:
payload["voice_id"] = self._voice_id
if self._settings.voice:
payload["voice_id"] = self._settings.voice
try:
await self.start_ttfb_metrics()

208
src/pipecat/services/nvidia/stt.py
View File

@@ -8,7 +8,8 @@
import asyncio
from concurrent.futures import CancelledError as FuturesCancelledError
from typing import AsyncGenerator, List, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, List, Mapping, Optional
from loguru import logger
from pydantic import BaseModel
@@ -22,6 +23,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import NVIDIA_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService, STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -89,6 +91,32 @@ def language_to_nvidia_riva_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class NvidiaSTTSettings(STTSettings):
"""Settings for the NVIDIA Riva streaming STT service."""
pass
@dataclass
class NvidiaSegmentedSTTSettings(STTSettings):
"""Settings for the NVIDIA Riva segmented STT service.
Parameters:
profanity_filter: Whether to filter profanity from results.
automatic_punctuation: Whether to add automatic punctuation.
verbatim_transcripts: Whether to return verbatim transcripts.
boosted_lm_words: List of words to boost in language model.
boosted_lm_score: Score boost for specified words.
"""
profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
automatic_punctuation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
verbatim_transcripts: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
boosted_lm_words: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
boosted_lm_score: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NvidiaSTTService(STTService):
"""Real-time speech-to-text service using NVIDIA Riva streaming ASR.
@@ -97,6 +125,8 @@ class NvidiaSTTService(STTService):
processing for low-latency applications.
"""
_settings: NvidiaSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for NVIDIA Riva STT service.
@@ -134,19 +164,21 @@ class NvidiaSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSTTService.InputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSTTSettings(
model=model_function_map.get("model_name"),
language=params.language,
),
**kwargs,
)
self._server = server
self._api_key = api_key
self._use_ssl = use_ssl
self._profanity_filter = False
self._automatic_punctuation = True
self._no_verbatim_transcripts = False
self._language_code = params.language
self._boosted_lm_words = None
self._boosted_lm_score = 4.0
self._start_history = -1
self._start_threshold = -1.0
self._stop_history = -1
@@ -156,17 +188,6 @@ class NvidiaSTTService(STTService):
self._custom_configuration = ""
self._function_id = model_function_map.get("function_id")
self._settings = {
"language": str(params.language),
"profanity_filter": self._profanity_filter,
"automatic_punctuation": self._automatic_punctuation,
"verbatim_transcripts": not self._no_verbatim_transcripts,
"boosted_lm_words": self._boosted_lm_words,
"boosted_lm_score": self._boosted_lm_score,
}
self.set_model_name(model_function_map.get("model_name"))
self._asr_service = None
self._queue = None
self._config = None
@@ -186,22 +207,18 @@ class NvidiaSTTService(STTService):
config = riva.client.StreamingRecognitionConfig(
config=riva.client.RecognitionConfig(
encoding=riva.client.AudioEncoding.LINEAR_PCM,
language_code=self._language_code,
language_code=self._settings.language,
model="",
max_alternatives=1,
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=not self._no_verbatim_transcripts,
profanity_filter=False,
enable_automatic_punctuation=True,
verbatim_transcripts=True,
sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
interim_results=True,
)
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
)
riva.client.add_endpoint_parameters_to_config(
config,
self._start_history,
@@ -226,18 +243,31 @@ class NvidiaSTTService(STTService):
async def set_model(self, model: str):
"""Set the ASR model for transcription.
.. deprecated:: 0.0.104
Model cannot be changed after initialization for NVIDIA Riva streaming STT.
Set model and function id in the constructor instead, e.g.::
NvidiaSTTService(
api_key=...,
model_function_map={"function_id": "<UUID>", "model_name": "<model_name>"},
)
Args:
model: Model name to set.
Note:
Model cannot be changed after initialization. Use model_function_map
parameter in constructor instead.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated. Model cannot be changed after initialization"
" for NVIDIA Riva streaming STT. Set model and function id in the"
" constructor instead, e.g.:"
" NvidiaSTTService(api_key=..., model_function_map="
"{'function_id': '<UUID>', 'model_name': '<model_name>'})",
DeprecationWarning,
stacklevel=2,
)
async def start(self, frame: StartFrame):
"""Start the NVIDIA Riva STT service and initialize streaming configuration.
@@ -254,7 +284,7 @@ class NvidiaSTTService(STTService):
if not self._thread_task:
self._thread_task = self.create_task(self._thread_task_handler())
logger.debug(f"Initialized NvidiaSTTService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaSTTService with model: {self._settings.model}")
async def stop(self, frame: EndFrame):
"""Stop the NVIDIA Riva STT service and clean up resources.
@@ -318,14 +348,14 @@ class NvidiaSTTService(STTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language_code,
self._settings.language,
result=result,
)
)
await self._handle_transcription(
transcript=transcript,
is_final=result.is_final,
language=self._language_code,
language=self._settings.language,
)
else:
await self.push_frame(
@@ -333,7 +363,7 @@ class NvidiaSTTService(STTService):
transcript,
self._user_id,
time_now_iso8601(),
self._language_code,
self._settings.language,
result=result,
)
)
@@ -386,6 +416,8 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
audio buffering and speech detection.
"""
_settings: NvidiaSegmentedSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for NVIDIA Riva segmented STT service.
@@ -433,30 +465,29 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
params = params or NvidiaSegmentedSTTService.InputParams()
# Set model name
self.set_model_name(model_function_map.get("model_name"))
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=NvidiaSegmentedSTTSettings(
model=model_function_map.get("model_name"),
language=self.language_to_service_language(params.language or Language.EN_US)
or "en-US",
profanity_filter=params.profanity_filter,
automatic_punctuation=params.automatic_punctuation,
verbatim_transcripts=params.verbatim_transcripts,
boosted_lm_words=params.boosted_lm_words,
boosted_lm_score=params.boosted_lm_score,
),
**kwargs,
)
# Initialize NVIDIA Riva settings
self._api_key = api_key
self._server = server
self._use_ssl = use_ssl
self._function_id = model_function_map.get("function_id")
self._model_name = model_function_map.get("model_name")
# Store the language as a Language enum and as a string
self._language_enum = params.language or Language.EN_US
self._language = self.language_to_service_language(self._language_enum) or "en-US"
# Configure transcription parameters
self._profanity_filter = params.profanity_filter
self._automatic_punctuation = params.automatic_punctuation
self._verbatim_transcripts = params.verbatim_transcripts
self._boosted_lm_words = params.boosted_lm_words
self._boosted_lm_score = params.boosted_lm_score
# Voice activity detection thresholds (use NVIDIA Riva defaults)
self._start_history = -1
@@ -467,10 +498,8 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
self._stop_threshold_eou = -1.0
self._custom_configuration = ""
# Create NVIDIA Riva client
self._config = None
self._asr_service = None
self._settings = {"language": self._language_enum}
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to NVIDIA Riva's language code.
@@ -498,21 +527,25 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
auth = riva.client.Auth(None, self._use_ssl, self._server, metadata)
self._asr_service = riva.client.ASRService(auth)
def _get_language_code(self) -> str:
"""Get the current NVIDIA Riva language code string."""
return self._settings.language or "en-US"
def _create_recognition_config(self):
"""Create the NVIDIA Riva ASR recognition configuration."""
# Create base configuration
config = riva.client.RecognitionConfig(
language_code=self._language, # Now using the string, not a tuple
language_code=self._get_language_code(),
max_alternatives=1,
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=self._verbatim_transcripts,
profanity_filter=self._settings.profanity_filter,
enable_automatic_punctuation=self._settings.automatic_punctuation,
verbatim_transcripts=self._settings.verbatim_transcripts,
)
# Add word boosting if specified
if self._boosted_lm_words:
if self._settings.boosted_lm_words:
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
config, self._settings.boosted_lm_words, self._settings.boosted_lm_score
)
# Add voice activity detection parameters
@@ -540,22 +573,6 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
"""
return True
async def set_model(self, model: str):
"""Set the ASR model for transcription.
Args:
model: Model name to set.
Note:
Model cannot be changed after initialization. Use model_function_map
parameter in constructor instead.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
async def start(self, frame: StartFrame):
"""Initialize the service when the pipeline starts.
@@ -565,22 +582,23 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
await super().start(frame)
self._initialize_client()
self._config = self._create_recognition_config()
logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self._settings.model}")
async def set_language(self, language: Language):
"""Set the language for the STT service.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and sync internal state.
Args:
language: Target language for transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._language_enum = language
self._language = self.language_to_service_language(language) or "en-US"
self._settings["language"] = language
delta: A :class:`STTSettings` (or ``NvidiaSegmentedSTTSettings``) delta.
# Update configuration with new language
if self._config:
self._config.language_code = self._language
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if changed:
self._config = self._create_recognition_config()
return changed
@traced_stt
async def _handle_transcription(
@@ -633,11 +651,11 @@ class NvidiaSegmentedSTTService(SegmentedSTTService):
text,
self._user_id,
time_now_iso8601(),
self._language_enum,
self._settings.language,
)
transcription_found = True
await self._handle_transcription(text, True, self._language_enum)
await self._handle_transcription(text, True, self._settings.language)
if not transcription_found:
logger.debug(f"{self}: No transcription results found in NVIDIA Riva response")

85
src/pipecat/services/nvidia/tts.py
View File

@@ -12,7 +12,8 @@ gRPC API for high-quality speech synthesis.
import asyncio
import os
from typing import AsyncGenerator, AsyncIterator, Generator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, AsyncIterator, Generator, Mapping, Optional
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -30,6 +31,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
@@ -42,6 +44,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class NvidiaTTSSettings(TTSSettings):
"""Settings for NVIDIA Riva TTS service.
Parameters:
quality: Audio quality setting (0-100).
"""
quality: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class NvidiaTTSService(TTSService):
"""NVIDIA Riva text-to-speech service.
@@ -50,6 +63,8 @@ class NvidiaTTSService(TTSService):
configurable quality settings.
"""
_settings: NvidiaTTSSettings
class InputParams(BaseModel):
"""Input parameters for Riva TTS configuration.
@@ -88,36 +103,66 @@ class NvidiaTTSService(TTSService):
use_ssl: Whether to use SSL for the NVIDIA Riva server. Defaults to True.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NvidiaTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=NvidiaTTSSettings(
model=model_function_map.get("model_name"),
voice=voice_id,
language=params.language,
quality=params.quality,
),
**kwargs,
)
self._server = server
self._api_key = api_key
self._voice_id = voice_id
self._language_code = params.language
self._quality = params.quality
self._function_id = model_function_map.get("function_id")
self._use_ssl = use_ssl
self.set_model_name(model_function_map.get("model_name"))
self.set_voice(voice_id)
self._service = None
self._config = None
async def set_model(self, model: str):
"""Attempt to set the TTS model.
"""Set the TTS model.
Note: Model cannot be changed after initialization for Riva service.
.. deprecated:: 0.0.104
Model cannot be changed after initialization for NVIDIA Riva TTS.
Set model and function id in the constructor instead, e.g.::
NvidiaTTSService(
api_key=...,
model_function_map={"function_id": "<UUID>", "model_name": "<model_name>"},
)
Args:
model: The model name to set (operation not supported).
model: The model name to set.
"""
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated. Model cannot be changed after initialization"
" for NVIDIA Riva TTS. Set model and function id in the constructor"
" instead, e.g.: NvidiaTTSService(api_key=..., model_function_map="
"{'function_id': '<UUID>', 'model_name': '<model_name>'})",
DeprecationWarning,
stacklevel=2,
)
async def _update_settings(self, delta: NvidiaTTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: reconnect gRPC client to apply changed settings.
self._warn_unhandled_updated_settings(changed)
return changed
def _initialize_client(self):
if self._service is not None:
@@ -150,7 +195,7 @@ class NvidiaTTSService(TTSService):
await super().start(frame)
self._initialize_client()
self._config = self._create_synthesis_config()
logger.debug(f"Initialized NvidiaTTSService with model: {self.model_name}")
logger.debug(f"Initialized NvidiaTTSService with model: {self._settings.model}")
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -167,11 +212,11 @@ class NvidiaTTSService(TTSService):
def read_audio_responses() -> Generator[rtts.SynthesizeSpeechResponse, None, None]:
responses = self._service.synthesize_online(
text,
self._voice_id,
self._language_code,
self._settings.voice,
self._settings.language,
sample_rate_hz=self.sample_rate,
zero_shot_audio_prompt_file=None,
zero_shot_quality=self._quality,
zero_shot_quality=self._settings.quality,
custom_dictionary={},
)
return responses

100
src/pipecat/services/openai/base_llm.py
View File

@@ -10,7 +10,8 @@ import asyncio
import base64
import json
from contextlib import asynccontextmanager
from typing import Any, Dict, List, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Mapping, Optional
import httpx
from loguru import logger
@@ -32,7 +33,6 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -42,9 +42,24 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
from pipecat.services.settings import LLMSettings, _NotGiven
from pipecat.utils.tracing.service_decorators import traced_llm
@dataclass
class OpenAILLMSettings(LLMSettings):
"""Settings for OpenAI-compatible LLM services.
Parameters:
max_completion_tokens: Maximum completion tokens to generate.
service_tier: Service tier to use (e.g., "auto", "flex", "priority").
"""
max_completion_tokens: int | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
service_tier: str | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
class BaseOpenAILLMService(LLMService):
"""Base class for all services that use the AsyncOpenAI client.
@@ -55,6 +70,8 @@ class BaseOpenAILLMService(LLMService):
configurations.
"""
_settings: OpenAILLMSettings
class InputParams(BaseModel):
"""Input parameters for OpenAI model configuration.
@@ -116,24 +133,28 @@ class BaseOpenAILLMService(LLMService):
retry_on_timeout: Whether to retry the request once if it times out.
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs)
params = params or BaseOpenAILLMService.InputParams()
self._settings = {
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,
"seed": params.seed,
"temperature": params.temperature,
"top_p": params.top_p,
"max_tokens": params.max_tokens,
"max_completion_tokens": params.max_completion_tokens,
"service_tier": params.service_tier,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
super().__init__(
settings=OpenAILLMSettings(
model=model,
frequency_penalty=params.frequency_penalty,
presence_penalty=params.presence_penalty,
seed=params.seed,
temperature=params.temperature,
top_p=params.top_p,
top_k=None,
max_tokens=params.max_tokens,
max_completion_tokens=params.max_completion_tokens,
service_tier=params.service_tier,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
extra=params.extra if isinstance(params.extra, dict) else {},
),
**kwargs,
)
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self.set_model_name(model)
self._full_model_name: str = ""
self._client = self.create_client(
api_key=api_key,
@@ -247,23 +268,23 @@ class BaseOpenAILLMService(LLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"service_tier": self._settings["service_tier"],
"frequency_penalty": self._settings.frequency_penalty,
"presence_penalty": self._settings.presence_penalty,
"seed": self._settings.seed,
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
"max_completion_tokens": self._settings.max_completion_tokens,
"service_tier": self._settings.service_tier,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params
async def run_inference(
@@ -375,20 +396,29 @@ class BaseOpenAILLMService(LLMService):
else self._stream_chat_completions_universal_context(context)
)
# Ensure stream is closed on cancellation/exception to prevent socket
# leaks. OpenAI's AsyncStream uses close(), async generators use aclose().
# Ensure stream and its async iterator are closed on cancellation/exception
# to prevent socket leaks and uvloop crashes. Closing the iterator first
# cascades cleanup through nested async generators (httpx/httpcore internals),
# preventing uvloop's broken asyncgen finalizer from firing on Python 3.12+
# (MagicStack/uvloop#699).
@asynccontextmanager
async def _closing(stream):
chunk_iter = stream.__aiter__()
try:
yield stream
yield chunk_iter
finally:
if hasattr(stream, "aclose"):
await stream.aclose()
elif hasattr(stream, "close"):
# Close the iterator first to cascade cleanup through
# nested async generators (httpx/httpcore internals).
if hasattr(chunk_iter, "aclose"):
await chunk_iter.aclose()
# Then close the stream to release HTTP resources.
if hasattr(stream, "close"):
await stream.close()
elif hasattr(stream, "aclose"):
await stream.aclose()
async with _closing(chunk_stream):
async for chunk in chunk_stream:
async with _closing(chunk_stream) as chunk_iter:
async for chunk in chunk_iter:
if chunk.usage:
cached_tokens = (
chunk.usage.prompt_tokens_details.cached_tokens
@@ -508,8 +538,6 @@ class BaseOpenAILLMService(LLMService):
# NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
# LLMContext with it
context = OpenAILLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

16
src/pipecat/services/openai/image.py
View File

@@ -11,6 +11,7 @@ for creating images from text prompts.
"""
import io
from dataclasses import dataclass
from typing import AsyncGenerator, Literal, Optional
import aiohttp
@@ -24,6 +25,16 @@ from pipecat.frames.frames import (
URLImageRawFrame,
)
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings
@dataclass
class OpenAIImageGenSettings(ImageGenSettings):
"""Settings for the OpenAI image generation service.
Parameters:
model: DALL-E model identifier.
"""
class OpenAIImageGenService(ImageGenService):
@@ -52,8 +63,7 @@ class OpenAIImageGenService(ImageGenService):
image_size: Target size for generated images.
model: DALL-E model to use for generation. Defaults to "dall-e-3".
"""
super().__init__()
self.set_model_name(model)
super().__init__(settings=OpenAIImageGenSettings(model=model))
self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
self._aiohttp_session = aiohttp_session
@@ -70,7 +80,7 @@ class OpenAIImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
image = await self._client.images.generate(
prompt=prompt, model=self.model_name, n=1, size=self._image_size
prompt=prompt, model=self._settings.model, n=1, size=self._image_size
)
image_url = image.data[0].url

102
src/pipecat/services/openai/realtime/llm.py
View File

@@ -10,8 +10,8 @@ import base64
import io
import json
import time
from dataclasses import dataclass
from typing import Optional
from dataclasses import dataclass, field
from typing import Any, Optional
from loguru import logger
from PIL import Image
@@ -59,6 +59,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt
@@ -90,6 +91,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class OpenAIRealtimeLLMSettings(LLMSettings):
"""Settings for OpenAI Realtime LLM services.
Parameters:
session_properties: OpenAI Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class OpenAIRealtimeLLMService(LLMService):
"""OpenAI Realtime LLM service providing real-time audio and text communication.
@@ -98,6 +112,8 @@ class OpenAIRealtimeLLMService(LLMService):
management, and real-time transcription.
"""
_settings: OpenAIRealtimeLLMSettings
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
adapter_class = OpenAIRealtimeLLMAdapter
@@ -105,7 +121,7 @@ class OpenAIRealtimeLLMService(LLMService):
self,
*,
api_key: str,
model: str = "gpt-realtime",
model: str = "gpt-realtime-1.5",
base_url: str = "wss://api.openai.com/v1/realtime",
session_properties: Optional[events.SessionProperties] = None,
start_audio_paused: bool = False,
@@ -155,16 +171,26 @@ class OpenAIRealtimeLLMService(LLMService):
# Build WebSocket URL with model query parameter
# Source: https://platform.openai.com/docs/guides/realtime-websocket
full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs)
super().__init__(
base_url=full_url,
settings=OpenAIRealtimeLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = full_url
self.set_model_name(model)
# Initialize session_properties
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused
self._video_frame_detail = video_frame_detail
@@ -227,12 +253,12 @@ class OpenAIRealtimeLLMService(LLMService):
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
modalities = self._session_properties.output_modalities or ["audio", "text"]
modalities = self._settings.session_properties.output_modalities or ["audio", "text"]
return modality in modalities
def _get_enabled_modalities(self) -> list[str]:
"""Get the list of enabled modalities."""
modalities = self._session_properties.output_modalities or ["audio", "text"]
modalities = self._settings.session_properties.output_modalities or ["audio", "text"]
# API only supports single modality responses: either ["text"] or ["audio"]
if "audio" in modalities:
return ["audio"]
@@ -305,9 +331,9 @@ class OpenAIRealtimeLLMService(LLMService):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
turn_detection_disabled = (
self._session_properties.audio
and self._session_properties.audio.input
and self._session_properties.audio.input.turn_detection is False
self._settings.session_properties.audio
and self._settings.session_properties.audio.input
and self._settings.session_properties.audio.input.turn_detection is False
)
if turn_detection_disabled:
await self.send_client_event(events.InputAudioBufferClearEvent())
@@ -327,9 +353,9 @@ class OpenAIRealtimeLLMService(LLMService):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
turn_detection_disabled = (
self._session_properties.audio
and self._session_properties.audio.input
and self._session_properties.audio.input.turn_detection is False
self._settings.session_properties.audio
and self._settings.session_properties.audio.input
and self._settings.session_properties.audio.input.turn_detection is False
)
if turn_detection_disabled:
await self.send_client_event(events.InputAudioBufferCommitEvent())
@@ -397,6 +423,16 @@ class OpenAIRealtimeLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.delta path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.delta is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -424,11 +460,8 @@ class OpenAIRealtimeLLMService(LLMService):
await self._handle_bot_stopped_speaking()
elif isinstance(frame, LLMMessagesAppendFrame):
await self._handle_messages_append(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
await self.push_frame(frame, direction)
@@ -513,8 +546,16 @@ class OpenAIRealtimeLLMService(LLMService):
# treat a send-side error as fatal.
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
settings = self._session_properties
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update if needed."""
changed = await super()._update_settings(delta)
if "session_properties" in changed:
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - {"session_properties"})
return changed
async def _send_session_update(self):
settings = self._settings.session_properties
adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
@@ -577,15 +618,18 @@ class OpenAIRealtimeLLMService(LLMService):
await self._handle_evt_function_call_arguments_done(evt)
elif evt.type == "error":
if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt):
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
if evt.error.code == "response_cancel_not_active":
logger.debug(f"{self} {evt.error.message}")
else:
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
@traced_openai_realtime(operation="llm_setup")
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
# to configure the session properties.
await self._update_settings()
await self._send_session_update()
async def _handle_evt_session_updated(self, evt):
# If this is our first context frame, run the LLM
@@ -868,7 +912,7 @@ class OpenAIRealtimeLLMService(LLMService):
await self.send_client_event(evt)
# Send new settings if needed
await self._update_settings()
await self._send_session_update()
# We're done configuring the LLM for this session
self._llm_needs_conversation_setup = False

71
src/pipecat/services/openai/stt.py
View File

@@ -16,7 +16,8 @@ Provides two STT services:
import base64
import json
from typing import AsyncGenerator, Literal, Optional, Union
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Literal, Optional, Union
from loguru import logger
@@ -34,6 +35,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription
@@ -98,24 +100,24 @@ class OpenAISTTService(BaseWhisperSTTService):
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"language": self._language,
"model": self._settings.model,
"language": self._settings.language,
}
if self._include_prob_metrics:
# GPT-4o-transcribe models only support logprobs (not verbose_json)
if self.model_name in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"):
if self._settings.model in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"):
kwargs["response_format"] = "json"
kwargs["include"] = ["logprobs"]
else:
# Whisper models support verbose_json
kwargs["response_format"] = "verbose_json"
if self._prompt is not None:
kwargs["prompt"] = self._prompt
if self._settings.prompt is not None:
kwargs["prompt"] = self._settings.prompt
if self._temperature is not None:
kwargs["temperature"] = self._temperature
if self._settings.temperature is not None:
kwargs["temperature"] = self._settings.temperature
return await self._client.audio.transcriptions.create(**kwargs)
@@ -123,6 +125,17 @@ class OpenAISTTService(BaseWhisperSTTService):
_OPENAI_SAMPLE_RATE = 24000
@dataclass
class OpenAIRealtimeSTTSettings(STTSettings):
"""Settings for the OpenAI Realtime STT service.
Parameters:
prompt: Optional prompt text to guide transcription style.
"""
prompt: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class OpenAIRealtimeSTTService(WebsocketSTTService):
"""OpenAI Realtime Speech-to-Text service using WebSocket transcription sessions.
@@ -156,6 +169,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
)
"""
_settings: OpenAIRealtimeSTTSettings
def __init__(
self,
*,
@@ -206,14 +221,17 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=OpenAIRealtimeSTTSettings(
model=model,
language=language,
prompt=prompt,
),
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self.set_model_name(model)
self._language_code = self._language_to_code(language) if language else None
self._prompt = prompt
self._turn_detection = turn_detection
self._noise_reduction = noise_reduction
@@ -248,19 +266,31 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
"""
return True
async def set_language(self, language: Language):
"""Set the language for speech recognition.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta and send session update if needed.
If the session is already active, sends an updated configuration
to the server.
Keeps ``_language_code`` and ``_prompt`` in sync with settings
and sends a ``session.update`` to the server when the session is active.
Args:
language: The language to use for speech recognition.
delta: A :class:`STTSettings` (or ``OpenAIRealtimeSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
"""
self._language_code = self._language_to_code(language)
changed = await super()._update_settings(delta)
if not changed:
return changed
if "prompt" in changed and isinstance(self._settings, OpenAIRealtimeSTTSettings):
self._prompt = self._settings.prompt
if self._session_ready:
await self._send_session_update()
return changed
async def start(self, frame: StartFrame):
"""Start the service and establish WebSocket connection.
@@ -405,10 +435,13 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
async def _send_session_update(self):
"""Send ``session.update`` to configure the transcription session."""
transcription: dict = {"model": self.model_name}
transcription: dict = {"model": self._settings.model}
if self._language_code:
transcription["language"] = self._language_code
language_code = (
self._language_to_code(self._settings.language) if self._settings.language else None
)
if language_code:
transcription["language"] = language_code
if self._prompt:
transcription["prompt"] = self._prompt

60
src/pipecat/services/openai/tts.py
View File

@@ -10,6 +10,7 @@ This module provides integration with OpenAI's text-to-speech API for
generating high-quality synthetic speech from text input.
"""
from dataclasses import dataclass, field
from typing import AsyncGenerator, Dict, Literal, Optional
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -60,6 +62,19 @@ VALID_VOICES: Dict[str, ValidVoice] = {
}
@dataclass
class OpenAITTSSettings(TTSSettings):
"""Settings for OpenAI TTS service.
Parameters:
instructions: Instructions to guide voice synthesis behavior.
speed: Voice speed control (0.25 to 4.0, default 1.0).
"""
instructions: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speed: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class OpenAITTSService(TTSService):
"""OpenAI Text-to-Speech service that generates audio from text.
@@ -68,6 +83,8 @@ class OpenAITTSService(TTSService):
speech synthesis with streaming audio output.
"""
_settings: OpenAITTSSettings
OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz
class InputParams(BaseModel):
@@ -115,12 +132,6 @@ class OpenAITTSService(TTSService):
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
self.set_model_name(model)
self.set_voice(voice)
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
if instructions or speed:
import warnings
@@ -132,10 +143,18 @@ class OpenAITTSService(TTSService):
stacklevel=2,
)
self._settings = {
"instructions": params.instructions if params else instructions,
"speed": params.speed if params else speed,
}
super().__init__(
sample_rate=sample_rate,
settings=OpenAITTSSettings(
model=model,
voice=voice,
instructions=params.instructions if params else instructions,
speed=params.speed if params else speed,
),
**kwargs,
)
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -145,15 +164,6 @@ class OpenAITTSService(TTSService):
"""
return True
async def set_model(self, model: str):
"""Set the TTS model to use.
Args:
model: The model name to use for text-to-speech synthesis.
"""
logger.info(f"Switching TTS model to: [{model}]")
self.set_model_name(model)
async def start(self, frame: StartFrame):
"""Start the OpenAI TTS service.
@@ -185,16 +195,16 @@ class OpenAITTSService(TTSService):
# Setup API parameters
create_params = {
"input": text,
"model": self.model_name,
"voice": VALID_VOICES[self._voice_id],
"model": self._settings.model,
"voice": VALID_VOICES[self._settings.voice],
"response_format": "pcm",
}
if self._settings["instructions"]:
create_params["instructions"] = self._settings["instructions"]
if self._settings.instructions:
create_params["instructions"] = self._settings.instructions
if self._settings["speed"]:
create_params["speed"] = self._settings["speed"]
if self._settings.speed:
create_params["speed"] = self._settings.speed
async with self._client.audio.speech.with_streaming_response.create(
**create_params

88
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -10,7 +10,7 @@ import base64
import json
import time
import warnings
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Optional
from loguru import logger
@@ -54,6 +54,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai.llm import OpenAIContextAggregatorPair
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt
@@ -91,6 +92,19 @@ class CurrentAudioResponse:
total_size: int = 0
@dataclass
class OpenAIRealtimeBetaLLMSettings(LLMSettings):
"""Settings for OpenAI Realtime Beta LLM services.
Parameters:
session_properties: OpenAI Realtime session configuration.
"""
session_properties: events.SessionProperties | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
class OpenAIRealtimeBetaLLMService(LLMService):
"""OpenAI Realtime Beta LLM service providing real-time audio and text communication.
@@ -103,6 +117,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
management, and real-time transcription.
"""
_settings: OpenAIRealtimeBetaLLMSettings
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
adapter_class = OpenAIRealtimeLLMAdapter
@@ -140,15 +156,26 @@ class OpenAIRealtimeBetaLLMService(LLMService):
)
full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs)
super().__init__(
base_url=full_url,
settings=OpenAIRealtimeBetaLLMSettings(
model=model,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
session_properties=session_properties or events.SessionProperties(),
),
**kwargs,
)
self.api_key = api_key
self.base_url = full_url
self.set_model_name(model)
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._send_transcription_frames = send_transcription_frames
self._websocket = None
@@ -187,12 +214,12 @@ class OpenAIRealtimeBetaLLMService(LLMService):
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
modalities = self._session_properties.modalities or ["audio", "text"]
modalities = self._settings.session_properties.modalities or ["audio", "text"]
return modality in modalities
def _get_enabled_modalities(self) -> list[str]:
"""Get the list of enabled modalities."""
return self._session_properties.modalities or ["audio", "text"]
return self._settings.session_properties.modalities or ["audio", "text"]
async def retrieve_conversation_item(self, item_id: str):
"""Retrieve a conversation item by ID from the server.
@@ -259,7 +286,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_interruption(self):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
if self._session_properties.turn_detection is False:
if self._settings.session_properties.turn_detection is False:
await self.send_client_event(events.InputAudioBufferClearEvent())
await self.send_client_event(events.ResponseCancelEvent())
await self._truncate_current_audio_response()
@@ -276,7 +303,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_user_stopped_speaking(self, frame):
# None and False are different. Check for False. None means we're using OpenAI's
# built-in turn detection defaults.
if self._session_properties.turn_detection is False:
if self._settings.session_properties.turn_detection is False:
await self.send_client_event(events.InputAudioBufferCommitEvent())
await self.send_client_event(events.ResponseCreateEvent())
@@ -342,6 +369,16 @@ class OpenAIRealtimeBetaLLMService(LLMService):
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
# Backward-compatible dict path: frame.settings contains SessionProperties
# fields, not our Settings fields, so we construct SessionProperties
# directly. The frame.delta path falls through to super, which calls
# _update_settings → our override handles the rest.
if isinstance(frame, LLMUpdateSettingsFrame) and frame.delta is None:
self._settings.session_properties = events.SessionProperties(**frame.settings)
await self._send_session_update()
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
@@ -377,11 +414,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._handle_messages_append(frame)
elif isinstance(frame, RealtimeMessagesUpdateFrame):
self._context = frame.context
elif isinstance(frame, LLMUpdateSettingsFrame):
self._session_properties = events.SessionProperties(**frame.settings)
await self._update_settings()
elif isinstance(frame, LLMSetToolsFrame):
await self._update_settings()
await self._send_session_update()
elif isinstance(frame, RealtimeFunctionCallResultFrame):
await self._handle_function_call_result(frame.result_frame)
@@ -456,8 +490,15 @@ class OpenAIRealtimeBetaLLMService(LLMService):
# treat a send-side error as fatal.
await self.push_error(error_msg=f"Error sending client event: {e}", exception=e)
async def _update_settings(self):
settings = self._session_properties
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update if needed."""
changed = await super()._update_settings(delta)
if "session_properties" in changed:
await self._send_session_update()
return changed
async def _send_session_update(self):
settings = self._settings.session_properties
# tools given in the context override the tools in the session properties
if self._context and self._context.tools:
settings.tools = self._context.tools
@@ -503,15 +544,18 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._handle_evt_audio_transcript_delta(evt)
elif evt.type == "error":
if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt):
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
if evt.error.code == "response_cancel_not_active":
logger.debug(f"{self} {evt.error.message}")
else:
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
@traced_openai_realtime(operation="llm_setup")
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
# to configure the session properties.
await self._update_settings()
await self._send_session_update()
async def _handle_evt_session_updated(self, evt):
# If this is our first context frame, run the LLM
@@ -750,7 +794,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self._context.llm_needs_initial_messages = False
if self._context.llm_needs_settings_update:
await self._update_settings()
await self._send_session_update()
self._context.llm_needs_settings_update = False
logger.debug(f"Creating response: {self._context.get_messages_for_logging()}")

3
src/pipecat/services/openrouter/llm.py
View File

@@ -72,8 +72,7 @@ class OpenRouterLLMService(OpenAILLMService):
Transformed parameters ready for the API call.
"""
params = super().build_chat_completion_params(params_from_context)
model = getattr(self, "model_name", getattr(self, "model", "")).lower()
if "gemini" in model:
if "gemini" in self._settings.model.lower():
messages = params.get("messages", [])
if not messages:
return params

24
src/pipecat/services/perplexity/llm.py
View File

@@ -11,8 +11,6 @@ an OpenAI-compatible interface. It handles Perplexity's unique token usage
reporting patterns while maintaining compatibility with the Pipecat framework.
"""
from openai import NOT_GIVEN
from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -66,22 +64,22 @@ class PerplexityLLMService(OpenAILLMService):
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"messages": params_from_context["messages"],
}
# Add OpenAI-compatible parameters if they're set
if self._settings["frequency_penalty"] is not NOT_GIVEN:
params["frequency_penalty"] = self._settings["frequency_penalty"]
if self._settings["presence_penalty"] is not NOT_GIVEN:
params["presence_penalty"] = self._settings["presence_penalty"]
if self._settings["temperature"] is not NOT_GIVEN:
params["temperature"] = self._settings["temperature"]
if self._settings["top_p"] is not NOT_GIVEN:
params["top_p"] = self._settings["top_p"]
if self._settings["max_tokens"] is not NOT_GIVEN:
params["max_tokens"] = self._settings["max_tokens"]
if self._settings.frequency_penalty is not None:
params["frequency_penalty"] = self._settings.frequency_penalty
if self._settings.presence_penalty is not None:
params["presence_penalty"] = self._settings.presence_penalty
if self._settings.temperature is not None:
params["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
params["top_p"] = self._settings.top_p
if self._settings.max_tokens is not None:
params["max_tokens"] = self._settings.max_tokens
return params

49
src/pipecat/services/piper/tts.py
View File

@@ -7,8 +7,9 @@
"""Piper TTS service implementation."""
import asyncio
from dataclasses import dataclass
from pathlib import Path
from typing import AsyncGenerator, AsyncIterator, Optional
from typing import Any, AsyncGenerator, AsyncIterator, Optional
import aiohttp
from loguru import logger
@@ -19,6 +20,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import TTSSettings
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -31,6 +33,13 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class PiperTTSSettings(TTSSettings):
"""Settings for Piper TTS service."""
pass
class PiperTTSService(TTSService):
"""Piper TTS service implementation.
@@ -39,6 +48,8 @@ class PiperTTSService(TTSService):
match the configured sample rate.
"""
_settings: PiperTTSSettings
def __init__(
self,
*,
@@ -58,9 +69,10 @@ class PiperTTSService(TTSService):
use_cuda: Use CUDA for GPU-accelerated inference.
**kwargs: Additional arguments passed to the parent `TTSService`.
"""
super().__init__(**kwargs)
self._voice_id = voice_id
super().__init__(
settings=PiperTTSSettings(model=None, voice=voice_id, language=None),
**kwargs,
)
download_dir = download_dir or Path.cwd()
@@ -85,6 +97,18 @@ class PiperTTSService(TTSService):
"""
return True
async def _update_settings(self, delta: PiperTTSSettings) -> dict[str, Any]:
"""Apply a settings delta.
Settings are stored but not applied to the active connection.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: voice changes would require re-downloading and loading the model.
self._warn_unhandled_updated_settings(changed)
return changed
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Piper.
@@ -143,6 +167,13 @@ class PiperTTSService(TTSService):
# $ uv pip install "piper-tts[http]"
# $ uv run python -m piper.http_server -m en_US-ryan-high
#
@dataclass
class PiperHttpTTSSettings(TTSSettings):
"""Settings for Piper HTTP TTS service."""
pass
class PiperHttpTTSService(TTSService):
"""Piper HTTP TTS service implementation.
@@ -151,6 +182,8 @@ class PiperHttpTTSService(TTSService):
rates and automatic WAV header removal.
"""
_settings: PiperHttpTTSSettings
def __init__(
self,
*,
@@ -167,7 +200,10 @@ class PiperHttpTTSService(TTSService):
voice_id: Piper voice model identifier (e.g. `en_US-ryan-high`).
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(**kwargs)
super().__init__(
settings=PiperHttpTTSSettings(model=None, voice=voice_id, language=None),
**kwargs,
)
if base_url.endswith("/"):
logger.warning("Base URL ends with a slash, this is not allowed.")
@@ -175,7 +211,6 @@ class PiperHttpTTSService(TTSService):
self._base_url = base_url
self._session = aiohttp_session
self._model_id = voice_id
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -205,7 +240,7 @@ class PiperHttpTTSService(TTSService):
data = {
"text": text,
"voice": self._model_id,
"voice": self._settings.voice,
}
async with self._session.post(self._base_url, json=data, headers=headers) as response:

13
src/pipecat/services/playht/__init__.py
View File

@@ -1,13 +0,0 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import sys
from pipecat.services import DeprecatedModuleProxy
from .tts import *
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "playht", "playht.tts")

651
src/pipecat/services/playht/tts.py
View File

@@ -1,651 +0,0 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""PlayHT text-to-speech service implementations.
This module provides integration with PlayHT's text-to-speech API
supporting both WebSocket streaming and HTTP-based synthesis.
"""
import io
import json
import struct
import uuid
import warnings
from typing import AsyncGenerator, Optional
import aiohttp
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use PlayHTTTSService, you need to `pip install pipecat-ai[playht]`.")
raise Exception(f"Missing module: {e}")
def language_to_playht_language(language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding PlayHT language code, or None if not supported.
"""
LANGUAGE_MAP = {
Language.AF: "afrikans",
Language.AM: "amharic",
Language.AR: "arabic",
Language.BN: "bengali",
Language.BG: "bulgarian",
Language.CA: "catalan",
Language.CS: "czech",
Language.DA: "danish",
Language.DE: "german",
Language.EL: "greek",
Language.EN: "english",
Language.ES: "spanish",
Language.FR: "french",
Language.GL: "galician",
Language.HE: "hebrew",
Language.HI: "hindi",
Language.HR: "croatian",
Language.HU: "hungarian",
Language.ID: "indonesian",
Language.IT: "italian",
Language.JA: "japanese",
Language.KO: "korean",
Language.MS: "malay",
Language.NL: "dutch",
Language.PL: "polish",
Language.PT: "portuguese",
Language.RU: "russian",
Language.SQ: "albanian",
Language.SR: "serbian",
Language.SV: "swedish",
Language.TH: "thai",
Language.TL: "tagalog",
Language.TR: "turkish",
Language.UK: "ukrainian",
Language.UR: "urdu",
Language.XH: "xhosa",
Language.ZH: "mandarin",
}
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
class PlayHTTTSService(InterruptibleTTSService):
"""PlayHT WebSocket-based text-to-speech service.
.. deprecated:: 0.0.88
This class is deprecated and will be removed in a future version.
PlayHT is shutting down their API on December 31st, 2025.
Provides real-time text-to-speech synthesis using PlayHT's WebSocket API.
Supports streaming audio generation with configurable voice engines and
language settings.
"""
class InputParams(BaseModel):
"""Input parameters for PlayHT TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
seed: Random seed for voice consistency.
"""
language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0
seed: Optional[int] = None
def __init__(
self,
*,
api_key: str,
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini",
sample_rate: Optional[int] = None,
output_format: str = "wav",
params: Optional[InputParams] = None,
**kwargs,
):
"""Initialize the PlayHT WebSocket TTS service.
Args:
api_key: PlayHT API key for authentication.
user_id: PlayHT user ID for authentication.
voice_url: URL of the voice to use for synthesis.
voice_engine: Voice engine to use. Defaults to "Play3.0-mini".
sample_rate: Audio sample rate. If None, uses default.
output_format: Audio output format. Defaults to "wav".
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
super().__init__(
pause_frame_processing=True,
sample_rate=sample_rate,
**kwargs,
)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"PlayHT is shutting down their API on December 31st, 2025. "
"'PlayHTTTSService' is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
params = params or PlayHTTTSService.InputParams()
self._api_key = api_key
self._user_id = user_id
self._websocket_url = None
self._receive_task = None
self._context_id = None
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "english",
"output_format": output_format,
"voice_engine": voice_engine,
"speed": params.speed,
"seed": params.seed,
}
self.set_model_name(voice_engine)
self.set_voice(voice_url)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as PlayHT service supports metrics generation.
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT service language format.
Args:
language: The language to convert.
Returns:
The PlayHT-specific language code, or None if not supported.
"""
return language_to_playht_language(language)
async def start(self, frame: StartFrame):
"""Start the PlayHT TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
await self._connect()
async def stop(self, frame: EndFrame):
"""Stop the PlayHT TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
"""Cancel the PlayHT TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame)
await self._disconnect()
async def _connect(self):
"""Connect to PlayHT WebSocket and start receive task."""
await super()._connect()
await self._connect_websocket()
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self):
"""Disconnect from PlayHT WebSocket and clean up tasks."""
await super()._disconnect()
if self._receive_task:
await self.cancel_task(self._receive_task)
self._receive_task = None
await self._disconnect_websocket()
async def _connect_websocket(self):
"""Connect to PlayHT websocket."""
try:
if self._websocket and self._websocket.state is State.OPEN:
return
logger.debug("Connecting to PlayHT")
if not self._websocket_url:
await self._get_websocket_url()
if not isinstance(self._websocket_url, str):
raise ValueError("WebSocket URL is not a string")
self._websocket = await websocket_connect(self._websocket_url)
await self._call_event_handler("on_connected")
except ValueError as e:
logger.error(f"{self} initialization error: {e}")
self._websocket = None
await self._call_event_handler("on_connection_error", f"{e}")
except Exception as e:
await self.push_error(error_msg=f"Error connecting: {e}", exception=e)
self._websocket = None
await self._call_event_handler("on_connection_error", f"{e}")
async def _disconnect_websocket(self):
"""Disconnect from PlayHT websocket."""
try:
await self.stop_all_metrics()
if self._websocket:
logger.debug("Disconnecting from PlayHT")
await self._websocket.close()
except Exception as e:
await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e)
finally:
self._context_id = None
self._websocket = None
await self._call_event_handler("on_disconnected")
async def _get_websocket_url(self):
"""Retrieve WebSocket URL from PlayHT API."""
async with aiohttp.ClientSession() as session:
async with session.post(
"https://api.play.ht/api/v4/websocket-auth",
headers={
"Authorization": f"Bearer {self._api_key}",
"X-User-Id": self._user_id,
"Content-Type": "application/json",
},
) as response:
if response.status in (200, 201):
data = await response.json()
# Handle the new response format with multiple URLs
if "websocket_urls" in data:
# Select URL based on voice_engine
if self._settings["voice_engine"] in data["websocket_urls"]:
self._websocket_url = data["websocket_urls"][
self._settings["voice_engine"]
]
else:
raise ValueError(
f"Unsupported voice engine: {self._settings['voice_engine']}"
)
else:
raise ValueError("Invalid response: missing websocket_urls")
else:
raise Exception(f"Failed to get WebSocket URL: {response.status}")
def _get_websocket(self):
"""Get the WebSocket connection if available."""
if self._websocket:
return self._websocket
raise Exception("Websocket not connected")
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request in case we don't have one already in progress.
Returns:
A unique string identifier for the TTS context.
"""
# If a context ID does not exist, create a new one.
# If an ID exists, continue using the current ID.
# When interruptions happen, user speech results in
# an interruption, which resets the context ID.
if not self._context_id:
return str(uuid.uuid4())
return self._context_id
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by stopping metrics and clearing request ID."""
await super()._handle_interruption(frame, direction)
await self.stop_all_metrics()
self._context_id = None
async def _receive_messages(self):
"""Receive messages from PlayHT websocket."""
async for message in self._get_websocket():
if isinstance(message, bytes):
# Skip the WAV header message
if message.startswith(b"RIFF"):
continue
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(message, self.sample_rate, 1, context_id=self._context_id)
await self.push_frame(frame)
else:
logger.debug(f"Received text message: {message}")
try:
msg = json.loads(message)
if msg.get("type") == "start":
# Handle start of stream
logger.debug(f"Started processing request: {msg.get('request_id')}")
elif msg.get("type") == "end":
# Handle end of stream
if "request_id" in msg and msg["request_id"] == self._context_id:
await self.push_frame(TTSStoppedFrame(context_id=self._context_id))
self._context_id = None
elif "error" in msg:
await self.push_error(error_msg=f"Error: {msg['error']}")
except json.JSONDecodeError:
logger.error(f"Invalid JSON message: {message}")
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text using PlayHT's WebSocket API.
Args:
text: The text to synthesize into speech.
context_id: The context ID for tracking audio frames.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]")
try:
# Reconnect if the websocket is closed
if not self._websocket or self._websocket.state is State.CLOSED:
await self._connect()
if not self._context_id:
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
self._context_id = context_id
tts_command = {
"text": text,
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"speed": self._settings["speed"],
"seed": self._settings["seed"],
"request_id": self._context_id,
}
try:
await self._get_websocket().send(json.dumps(tts_command))
await self.start_tts_usage_metrics(text)
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
yield TTSStoppedFrame(context_id=context_id)
await self._disconnect()
await self._connect()
return
# The actual audio frames will be handled in _receive_task_handler
yield None
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
class PlayHTHttpTTSService(TTSService):
"""PlayHT HTTP-based text-to-speech service.
.. deprecated:: 0.0.88
This class is deprecated and will be removed in a future version.
PlayHT is shutting down their API on December 31st, 2025.
Provides text-to-speech synthesis using PlayHT's HTTP API for simpler,
non-streaming synthesis. Suitable for use cases where streaming is not
required and simpler integration is preferred.
"""
class InputParams(BaseModel):
"""Input parameters for PlayHT HTTP TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
seed: Random seed for voice consistency.
"""
language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0
seed: Optional[int] = None
def __init__(
self,
*,
api_key: str,
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini",
protocol: Optional[str] = None,
output_format: str = "wav",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
**kwargs,
):
"""Initialize the PlayHT HTTP TTS service.
Args:
api_key: PlayHT API key for authentication.
user_id: PlayHT user ID for authentication.
voice_url: URL of the voice to use for synthesis.
voice_engine: Voice engine to use. Defaults to "Play3.0-mini".
protocol: Protocol to use ("http" or "ws").
.. deprecated:: 0.0.80
This parameter no longer has any effect and will be removed in a future version.
Use PlayHTTTSService for WebSocket or PlayHTHttpTTSService for HTTP.
output_format: Audio output format. Defaults to "wav".
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
# Warn about deprecated protocol parameter if explicitly provided
if protocol:
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"The 'protocol' parameter is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"PlayHT is shutting down their API on December 31st, 2025. "
"'PlayHTHttpTTSService' is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
params = params or PlayHTHttpTTSService.InputParams()
self._user_id = user_id
self._api_key = api_key
# Check if voice_engine contains protocol information (backward compatibility)
if "-http" in voice_engine:
# Extract the base engine name
voice_engine = voice_engine.replace("-http", "")
elif "-ws" in voice_engine:
# Extract the base engine name
voice_engine = voice_engine.replace("-ws", "")
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "english",
"output_format": output_format,
"voice_engine": voice_engine,
"speed": params.speed,
"seed": params.seed,
}
self.set_model_name(voice_engine)
self.set_voice(voice_url)
async def start(self, frame: StartFrame):
"""Start the PlayHT HTTP TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as PlayHT HTTP service supports metrics generation.
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT service language format.
Args:
language: The language to convert.
Returns:
The PlayHT-specific language code, or None if not supported.
"""
return language_to_playht_language(language)
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text using PlayHT's HTTP API.
Args:
text: The text to synthesize into speech.
context_id: The context ID for tracking audio frames.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]")
try:
await self.start_ttfb_metrics()
# Prepare the request payload
payload = {
"text": text,
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
"sample_rate": self.sample_rate,
"language": self._settings["language"],
}
# Add optional parameters if they exist
if self._settings["speed"] is not None:
payload["speed"] = self._settings["speed"]
if self._settings["seed"] is not None:
payload["seed"] = self._settings["seed"]
headers = {
"Authorization": f"Bearer {self._api_key}",
"X-User-Id": self._user_id,
"Content-Type": "application/json",
"Accept": "*/*",
}
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame(context_id=context_id)
async with aiohttp.ClientSession() as session:
async with session.post(
"https://api.play.ht/api/v2/tts/stream",
headers=headers,
json=payload,
) as response:
if response.status not in (200, 201):
error_text = await response.text()
raise Exception(f"PlayHT API error {response.status}: {error_text}")
in_header = True
buffer = b""
CHUNK_SIZE = self.chunk_size
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
if len(chunk) == 0:
continue
# Skip the RIFF header
if in_header:
buffer += chunk
if len(buffer) <= 36:
continue
else:
fh = io.BytesIO(buffer)
fh.seek(36)
(data, size) = struct.unpack("<4sI", fh.read(8))
while data != b"data":
fh.read(size)
(data, size) = struct.unpack("<4sI", fh.read(8))
# Extract audio data after header
audio_data = buffer[fh.tell() :]
if len(audio_data) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio_data, self.sample_rate, 1, context_id=context_id
)
yield frame
in_header = False
elif len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
chunk, self.sample_rate, 1, context_id=context_id
)
yield frame
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
finally:
await self.stop_ttfb_metrics()
yield TTSStoppedFrame(context_id=context_id)

81
src/pipecat/services/resembleai/tts.py
View File

@@ -8,7 +8,8 @@
import base64
import json
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, ClassVar, Dict, Optional
from loguru import logger
@@ -17,16 +18,13 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
InterruptionFrame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import AudioContextTTSService
from pipecat.utils.tracing.service_decorators import traced_tts
try:
@@ -38,7 +36,27 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
class ResembleAITTSService(AudioContextWordTTSService):
@dataclass
class ResembleAITTSSettings(TTSSettings):
"""Settings for Resemble AI TTS service.
Parameters:
precision: PCM bit depth (PCM_32, PCM_24, PCM_16, or MULAW).
output_format: Audio format (wav or mp3).
resemble_sample_rate: Audio sample rate sent to the API.
"""
precision: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_format: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
resemble_sample_rate: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {
"voice_id": "voice",
"sample_rate": "resemble_sample_rate",
}
class ResembleAITTSService(AudioContextTTSService):
"""Resemble AI TTS service with WebSocket streaming and word timestamps.
Provides text-to-speech using Resemble AI's streaming WebSocket API.
@@ -46,6 +64,8 @@ class ResembleAITTSService(AudioContextWordTTSService):
multiple simultaneous synthesis requests with proper interruption support.
"""
_settings: ResembleAITTSSettings
def __init__(
self,
*,
@@ -70,17 +90,21 @@ class ResembleAITTSService(AudioContextWordTTSService):
"""
super().__init__(
sample_rate=sample_rate,
reuse_context_id_within_turn=False,
supports_word_timestamps=True,
settings=ResembleAITTSSettings(
model=None,
voice=voice_id,
language=None,
precision=precision,
output_format=output_format,
resemble_sample_rate=sample_rate,
),
**kwargs,
)
self._api_key = api_key
self._voice_id = voice_id
self._url = url
self._settings = {
"precision": precision,
"output_format": output_format,
"sample_rate": sample_rate,
}
self._websocket = None
self._request_id_counter = 0
@@ -101,8 +125,6 @@ class ResembleAITTSService(AudioContextWordTTSService):
self._jitter_buffer_bytes = 44100 # ~1000ms at 22050Hz to handle 400ms+ network gaps
self._playback_started: dict[str, bool] = {} # Track if we've started playback per request
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -121,13 +143,13 @@ class ResembleAITTSService(AudioContextWordTTSService):
JSON string containing the request payload.
"""
msg = {
"voice_uuid": self._voice_id,
"voice_uuid": self._settings.voice,
"data": text,
"binary_response": False, # Use JSON frames to get timestamps
"request_id": self._request_id_counter, # ResembleAI only accepts number
"output_format": self._settings["output_format"],
"sample_rate": self._settings["sample_rate"],
"precision": self._settings["precision"],
"output_format": self._settings.output_format,
"sample_rate": self._settings.resemble_sample_rate,
"precision": self._settings.precision,
"no_audio_header": True,
}
@@ -141,7 +163,7 @@ class ResembleAITTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.resemble_sample_rate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -223,16 +245,19 @@ class ResembleAITTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by stopping current synthesis.
Args:
frame: The interruption frame.
direction: The direction of frame processing.
"""
await super()._handle_interruption(frame, direction)
async def on_audio_context_interrupted(self, context_id: str):
"""Stop metrics when the bot is interrupted."""
await self.stop_all_metrics()
async def on_audio_context_completed(self, context_id: str):
"""Stop metrics after the Resemble AI context finishes playing.
No close message is needed: Resemble AI signals completion with an
``audio_end`` message (handled in ``_process_messages``), after which
the server-side context is already closed.
"""
pass
async def flush_audio(self):
"""Flush any pending audio and finalize the current context."""
logger.trace(f"{self}: flushing audio")

515
src/pipecat/services/rime/tts.py
View File

@@ -12,8 +12,8 @@ using Rime's API for streaming and batch audio synthesis.
import base64
import json
import uuid
from typing import Any, AsyncGenerator, Mapping, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, ClassVar, Dict, Optional
import aiohttp
from loguru import logger
@@ -31,9 +31,11 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import (
AudioContextWordTTSService,
AudioContextTTSService,
InterruptibleTTSService,
TextAggregationMode,
TTSService,
)
from pipecat.transcriptions.language import Language, resolve_language
@@ -69,7 +71,67 @@ def language_to_rime_language(language: Language) -> str:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
class RimeTTSService(AudioContextWordTTSService):
@dataclass
class RimeTTSSettings(TTSSettings):
"""Settings for Rime WS JSON and HTTP TTS services.
Parameters:
audioFormat: Audio output format.
samplingRate: Audio sample rate.
segment: Text segmentation mode ("immediate", "bySentence", "never").
speedAlpha: Speech speed multiplier (mistv2 only).
reduceLatency: Whether to reduce latency at potential quality cost (mistv2 only).
pauseBetweenBrackets: Whether to add pauses between bracketed content (mistv2 only).
phonemizeBetweenBrackets: Whether to phonemize bracketed content (mistv2 only).
noTextNormalization: Whether to disable text normalization (mistv2 only).
saveOovs: Whether to save out-of-vocabulary words (mistv2 only).
inlineSpeedAlpha: Inline speed control markup.
repetition_penalty: Token repetition penalty (arcana only, 1.0-2.0).
temperature: Sampling temperature (arcana only, 0.0-1.0).
top_p: Cumulative probability threshold (arcana only, 0.0-1.0).
"""
audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speedAlpha: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
reduceLatency: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pauseBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
phonemizeBetweenBrackets: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
noTextNormalization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
saveOovs: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
inlineSpeedAlpha: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"}
@dataclass
class RimeNonJsonTTSSettings(TTSSettings):
"""Settings for Rime non-JSON WS TTS service.
Parameters:
audioFormat: Audio output format.
samplingRate: Audio sample rate.
segment: Text segmentation mode ("immediate", "bySentence", "never").
repetition_penalty: Token repetition penalty (1.0-2.0).
temperature: Sampling temperature (0.0-1.0).
top_p: Cumulative probability threshold (0.0-1.0).
"""
audioFormat: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
samplingRate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
segment: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
repetition_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"speaker": "voice"}
class RimeTTSService(AudioContextTTSService):
"""Text-to-Speech service using Rime's websocket API.
Uses Rime's websocket JSON API to convert text to speech with word-level timing
@@ -77,6 +139,8 @@ class RimeTTSService(AudioContextWordTTSService):
within a turn.
"""
_settings: RimeTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime TTS service.
@@ -118,7 +182,8 @@ class RimeTTSService(AudioContextWordTTSService):
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
text_aggregator: Optional[BaseTextAggregator] = None,
aggregate_sentences: Optional[bool] = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
**kwargs,
):
"""Initialize Rime TTS service.
@@ -135,17 +200,48 @@ class RimeTTSService(AudioContextWordTTSService):
.. deprecated:: 0.0.95
Use an LLMTextProcessor before the TTSService for custom text aggregation.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
text_aggregation_mode: How to aggregate incoming text before synthesis.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
**kwargs: Additional arguments passed to parent class.
"""
# Initialize with parent class settings for proper frame handling
params = params or RimeTTSService.InputParams()
super().__init__(
text_aggregation_mode=text_aggregation_mode,
aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
pause_frame_processing=True,
supports_word_timestamps=True,
append_trailing_space=True,
sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
voice=voice_id,
audioFormat="pcm",
samplingRate=0, # updated in start()
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
inlineSpeedAlpha=None, # Not applicable here
# Arcana params
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
# Mistv2 params
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=params.no_text_normalization,
saveOovs=params.save_oovs,
),
**kwargs,
)
@@ -155,19 +251,15 @@ class RimeTTSService(AudioContextWordTTSService):
# The preferred way of taking advantage of Rime spelling is
# to use an LLMTextProcessor and/or a text_transformer to identify
# and insert these tags for the purpose of the TTS service alone.
self._text_aggregator = SkipTagsAggregator([("spell(", ")")])
self._params = params or RimeTTSService.InputParams()
self._text_aggregator = SkipTagsAggregator(
[("spell(", ")")], aggregation_type=self._text_aggregation_mode
)
# Store service configuration
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._model = model
self._settings = self._build_settings()
# State tracking
self._context_id = None # Tracks current turn
self._receive_task = None
self._cumulative_time = 0 # Accumulates time across messages
self._extra_msg_fields = {} # Extra fields for next message
@@ -191,60 +283,49 @@ class RimeTTSService(AudioContextWordTTSService):
"""
return language_to_rime_language(language)
def _build_settings(self) -> dict:
"""Build query params for the WebSocket URL based on the current model and params.
def _build_ws_params(self) -> dict[str, Any]:
"""Build query params for the WebSocket URL from current settings.
Returns:
Dictionary of query parameters. Only explicitly-set values are included.
Dictionary of query parameters for the WebSocket URL.
Only explicitly-set values are included. Boolean mistv2 params
are serialized with ``json.dumps()`` for the wire format.
"""
settings = {
"speaker": self._voice_id,
"modelId": self._model,
"audioFormat": "pcm",
"samplingRate": self.sample_rate or 0,
params: dict[str, Any] = {
"speaker": self._settings.voice,
"modelId": self._settings.model,
"audioFormat": self._settings.audioFormat,
"samplingRate": self._settings.samplingRate,
}
if self._params.language:
settings["lang"] = self.language_to_service_language(self._params.language) or "eng"
if self._params.segment is not None:
settings["segment"] = self._params.segment
if self._settings.language is not None:
params["lang"] = self._settings.language
if self._settings.segment is not None:
params["segment"] = self._settings.segment
if self._settings.speedAlpha is not None:
params["speedAlpha"] = self._settings.speedAlpha
if self._model == "arcana":
if self._params.repetition_penalty is not None:
settings["repetition_penalty"] = self._params.repetition_penalty
if self._params.temperature is not None:
settings["temperature"] = self._params.temperature
if self._params.top_p is not None:
settings["top_p"] = self._params.top_p
if self._settings.model == "arcana":
if self._settings.repetition_penalty is not None:
params["repetition_penalty"] = self._settings.repetition_penalty
if self._settings.temperature is not None:
params["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
params["top_p"] = self._settings.top_p
else: # mistv2/mist
if self._params.speed_alpha is not None:
settings["speedAlpha"] = self._params.speed_alpha
if self._params.reduce_latency is not None:
settings["reduceLatency"] = self._params.reduce_latency
if self._params.pause_between_brackets is not None:
settings["pauseBetweenBrackets"] = json.dumps(self._params.pause_between_brackets)
if self._params.phonemize_between_brackets is not None:
settings["phonemizeBetweenBrackets"] = json.dumps(
self._params.phonemize_between_brackets
if self._settings.reduceLatency is not None:
params["reduceLatency"] = self._settings.reduceLatency
if self._settings.pauseBetweenBrackets is not None:
params["pauseBetweenBrackets"] = json.dumps(self._settings.pauseBetweenBrackets)
if self._settings.phonemizeBetweenBrackets is not None:
params["phonemizeBetweenBrackets"] = json.dumps(
self._settings.phonemizeBetweenBrackets
)
if self._params.no_text_normalization is not None:
settings["noTextNormalization"] = json.dumps(self._params.no_text_normalization)
if self._params.save_oovs is not None:
settings["saveOovs"] = json.dumps(self._params.save_oovs)
if self._settings.noTextNormalization is not None:
params["noTextNormalization"] = json.dumps(self._settings.noTextNormalization)
if self._settings.saveOovs is not None:
params["saveOovs"] = json.dumps(self._settings.saveOovs)
return settings
async def set_model(self, model: str):
"""Update the TTS model and reconnect.
Args:
model: The model name to use for synthesis.
"""
self._model = model
self._settings = self._build_settings()
await super().set_model(model)
if self._websocket:
await self._disconnect()
await self._connect()
return params
# A set of Rime-specific helpers for text transformations
def SPELL(text: str) -> str:
@@ -271,75 +352,23 @@ class RimeTTSService(AudioContextWordTTSService):
self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)])
return f"[{text}]"
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if necessary.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if necessary.
Since all settings are WebSocket URL query parameters,
any setting change requires reconnecting to apply the new values.
"""
prev_settings = self._settings.copy()
await super()._update_settings(settings)
changed = await super()._update_settings(delta)
needs_reconnect = False
if "voice" in settings or "voice_id" in settings:
self._settings["speaker"] = self._voice_id
if prev_settings.get("speaker") != self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
needs_reconnect = True
if "model" in settings:
self._settings = self._build_settings()
needs_reconnect = True
if "language" in settings:
new_lang = self.language_to_service_language(settings["language"])
if new_lang and new_lang != prev_settings.get("lang"):
logger.info(f"Updating language to: [{new_lang}]")
self._settings["lang"] = new_lang
needs_reconnect = True
# Arcana params
for key, settings_key in [
("repetition_penalty", "repetition_penalty"),
("temperature", "temperature"),
("top_p", "top_p"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 params
for key, settings_key in [
("speed_alpha", "speedAlpha"),
("reduce_latency", "reduceLatency"),
]:
if key in settings and settings[key] != prev_settings.get(settings_key):
self._settings[settings_key] = settings[key]
needs_reconnect = True
# Mistv2 boolean params (need json.dumps)
for key, settings_key in [
("pause_between_brackets", "pauseBetweenBrackets"),
("phonemize_between_brackets", "phonemizeBetweenBrackets"),
("no_text_normalization", "noTextNormalization"),
("save_oovs", "saveOovs"),
]:
if key in settings and json.dumps(settings[key]) != prev_settings.get(settings_key):
self._settings[settings_key] = json.dumps(settings[key])
needs_reconnect = True
if "segment" in settings and settings["segment"] != prev_settings.get("segment"):
self._settings["segment"] = settings["segment"]
needs_reconnect = True
if needs_reconnect and self._websocket:
if changed and self._websocket:
await self._disconnect()
await self._connect()
return changed
def _build_msg(self, text: str = "") -> dict:
"""Build JSON message for Rime API."""
msg = {"text": text, "contextId": self._context_id}
msg = {"text": text, "contextId": self.get_active_audio_context_id()}
if self._extra_msg_fields:
msg |= self._extra_msg_fields
self._extra_msg_fields = {}
@@ -360,7 +389,7 @@ class RimeTTSService(AudioContextWordTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings = self._build_settings()
self._settings.samplingRate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -406,7 +435,8 @@ class RimeTTSService(AudioContextWordTTSService):
if self._websocket and self._websocket.state is State.OPEN:
return
params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None)
ws_params = self._build_ws_params()
params = "&".join(f"{k}={v}" for k, v in ws_params.items() if v is not None)
url = f"{self._url}?{params}"
headers = {"Authorization": f"Bearer {self._api_key}"}
self._websocket = await websocket_connect(url, additional_headers=headers)
@@ -427,7 +457,7 @@ class RimeTTSService(AudioContextWordTTSService):
except Exception as e:
await self.push_error(error_msg=f"Error disconnecting: {e}", exception=e)
finally:
self._context_id = None
await self.remove_active_audio_context()
self._websocket = None
await self._call_event_handler("on_disconnected")
@@ -437,13 +467,24 @@ class RimeTTSService(AudioContextWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
"""Handle interruption by clearing current context."""
await super()._handle_interruption(frame, direction)
async def _close_context(self, context_id: str):
"""Clear the Rime speech queue and stop metrics."""
await self.stop_all_metrics()
if self._context_id:
if context_id:
await self._get_websocket().send(json.dumps(self._build_clear_msg()))
self._context_id = None
async def on_audio_context_interrupted(self, context_id: str):
"""Clear the Rime speech queue and stop metrics when the bot is interrupted."""
await self._close_context(context_id)
async def on_audio_context_completed(self, context_id: str):
"""Clear server-side state and stop metrics after the Rime context finishes playing.
Rime does not send a server-side completion signal (e.g. ``done`` / ``end_of_stream`` /
``audio_end``), so we explicitly send a ``clear`` message to clean up
any residual server-side state once all audio has been delivered.
"""
await self._close_context(context_id)
def _calculate_word_times(self, words: list, starts: list, ends: list) -> list:
"""Calculate word timing pairs with proper spacing and punctuation.
@@ -474,28 +515,15 @@ class RimeTTSService(AudioContextWordTTSService):
return word_pairs
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request in case we don't have one already in progress.
Returns:
A unique string identifier for the TTS context.
"""
# If a context ID does not exist, create a new one.
# If an ID exists, continue using the current ID.
# When interruptions happen, user speech results in
# an interruption, which resets the context ID.
if not self._context_id:
return str(uuid.uuid4())
return self._context_id
async def flush_audio(self):
"""Flush any pending audio synthesis."""
if not self._context_id or not self._websocket:
context_id = self.get_active_audio_context_id()
if not context_id or not self._websocket:
return
logger.trace(f"{self}: flushing audio")
await self._get_websocket().send(json.dumps({"operation": "flush"}))
self._context_id = None
self.reset_active_audio_context()
async def _receive_messages(self):
"""Process incoming websocket messages."""
@@ -537,7 +565,7 @@ class RimeTTSService(AudioContextWordTTSService):
await self.push_frame(TTSStoppedFrame())
await self.stop_all_metrics()
await self.push_error(error_msg=f"Error: {msg['message']}")
self._context_id = None
self.reset_active_audio_context()
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push frame and handle end-of-turn conditions.
@@ -568,12 +596,11 @@ class RimeTTSService(AudioContextWordTTSService):
await self._connect()
try:
if not self._context_id:
if not self.has_active_audio_context():
await self.start_ttfb_metrics()
yield TTSStartedFrame(context_id=context_id)
self._cumulative_time = 0
self._context_id = context_id
await self.create_audio_context(self._context_id)
await self.create_audio_context(context_id)
msg = self._build_msg(text=text)
await self._get_websocket().send(json.dumps(msg))
@@ -596,6 +623,8 @@ class RimeHttpTTSService(TTSService):
Suitable for use cases where streaming is not required.
"""
_settings: RimeTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime HTTP TTS service.
@@ -637,27 +666,36 @@ class RimeHttpTTSService(TTSService):
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RimeHttpTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=RimeTTSSettings(
model=model,
language=self.language_to_service_language(params.language)
if params.language
else "eng",
audioFormat="pcm",
samplingRate=0,
segment=None,
speedAlpha=params.speed_alpha,
reduceLatency=params.reduce_latency,
pauseBetweenBrackets=params.pause_between_brackets,
phonemizeBetweenBrackets=params.phonemize_between_brackets,
noTextNormalization=None,
saveOovs=None,
inlineSpeedAlpha=params.inline_speed_alpha if params.inline_speed_alpha else None,
repetition_penalty=None,
temperature=None,
top_p=None,
voice=voice_id,
),
**kwargs,
)
self._api_key = api_key
self._session = aiohttp_session
self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = {
"lang": self.language_to_service_language(params.language)
if params.language
else "eng",
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": params.pause_between_brackets,
"phonemizeBetweenBrackets": params.phonemize_between_brackets,
}
self.set_voice(voice_id)
self.set_model_name(model)
if params.inline_speed_alpha:
self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -697,10 +735,18 @@ class RimeHttpTTSService(TTSService):
"Content-Type": "application/json",
}
payload = self._settings.copy()
payload = {
"lang": self._settings.language,
"speedAlpha": self._settings.speedAlpha,
"reduceLatency": self._settings.reduceLatency,
"pauseBetweenBrackets": self._settings.pauseBetweenBrackets,
"phonemizeBetweenBrackets": self._settings.phonemizeBetweenBrackets,
}
if self._settings.inlineSpeedAlpha is not None:
payload["inlineSpeedAlpha"] = self._settings.inlineSpeedAlpha
payload["text"] = text
payload["speaker"] = self._voice_id
payload["modelId"] = self._model_name
payload["speaker"] = self._settings.voice
payload["modelId"] = self._settings.model
payload["samplingRate"] = self.sample_rate
# Arcana does not support PCM audio
@@ -759,6 +805,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
accepts and returns non-JSON messages.
"""
_settings: RimeNonJsonTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Rime Non-JSON WebSocket TTS service.
@@ -788,7 +836,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
audio_format: str = "pcm",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
**kwargs,
):
"""Initialize Rime Non-JSON WebSocket TTS service.
@@ -801,41 +850,44 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
audio_format: Audio format to use.
sample_rate: Audio sample rate in Hz.
params: Additional configuration parameters.
aggregate_sentences: Whether to aggregate sentences within the TTSService.
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead. Set to ``TextAggregationMode.SENTENCE``
to aggregate text into sentences before synthesis, or
``TextAggregationMode.TOKEN`` to stream tokens directly for lower latency.
text_aggregation_mode: How to aggregate text before synthesis.
**kwargs: Additional arguments passed to parent class.
"""
params = params or RimeNonJsonTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_stop_frames=True,
pause_frame_processing=True,
append_trailing_space=True,
settings=RimeNonJsonTTSSettings(
voice=voice_id,
model=model,
audioFormat=audio_format,
samplingRate=sample_rate,
language=self.language_to_service_language(params.language)
if params.language
else None,
segment=params.segment,
repetition_penalty=params.repetition_penalty,
temperature=params.temperature,
top_p=params.top_p,
),
**kwargs,
)
params = params or RimeNonJsonTTSService.InputParams()
self._api_key = api_key
self._url = url
self._voice_id = voice_id
self._model = model
self._settings = {
"speaker": voice_id,
"modelId": model,
"audioFormat": audio_format,
"samplingRate": sample_rate,
}
if params.language:
self._settings["lang"] = self.language_to_service_language(params.language)
if params.segment is not None:
self._settings["segment"] = params.segment
if params.repetition_penalty is not None:
self._settings["repetition_penalty"] = params.repetition_penalty
if params.temperature is not None:
self._settings["temperature"] = params.temperature
if params.top_p is not None:
self._settings["top_p"] = params.top_p
# Add any extra parameters for future compatibility
if params.extra:
self._settings.update(params.extra)
self._settings.extra.update(params.extra)
self._receive_task = None
self._context_id: Optional[str] = None
@@ -867,7 +919,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["samplingRate"] = self.sample_rate
self._settings.samplingRate = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -911,8 +963,26 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
try:
if self._websocket and self._websocket.state is State.OPEN:
return
# Build URL with query parameters (only non-None values)
params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None)
# Build URL with query parameters (only given, non-None values)
settings_dict = {
"speaker": self._settings.voice,
"modelId": self._settings.model,
"audioFormat": self._settings.audioFormat,
"samplingRate": self._settings.samplingRate,
}
if self._settings.language is not None:
settings_dict["lang"] = self._settings.language
if self._settings.segment is not None:
settings_dict["segment"] = self._settings.segment
if self._settings.repetition_penalty is not None:
settings_dict["repetition_penalty"] = self._settings.repetition_penalty
if self._settings.temperature is not None:
settings_dict["temperature"] = self._settings.temperature
if self._settings.top_p is not None:
settings_dict["top_p"] = self._settings.top_p
# Include extras
settings_dict.update(self._settings.extra)
params = "&".join(f"{k}={v}" for k, v in settings_dict.items() if v is not None)
url = f"{self._url}?{params}"
headers = {"Authorization": f"Bearer {self._api_key}"}
self._websocket = await websocket_connect(
@@ -1006,68 +1076,17 @@ class RimeNonJsonTTSService(InterruptibleTTSService):
except Exception as e:
yield ErrorFrame(error=f"Unknown error occurred: {e}")
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if necessary.
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and reconnect if necessary.
Since all settings are WebSocket URL query parameters,
any setting change requires reconnecting to apply the new values.
"""
needs_reconnect = False
changed = await super()._update_settings(delta)
# Track previous values from self._settings only
prev_settings = self._settings.copy()
# Let parent class handle standard settings (voice, model, language)
await super()._update_settings(settings)
# Check if voice changed and update settings dict
if "voice" in settings or "voice_id" in settings:
self._settings["speaker"] = self._voice_id
if prev_settings.get("speaker") != self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
needs_reconnect = True
# Check if model changed and update settings dict
if "model" in settings:
self._settings["modelId"] = self._model
if prev_settings.get("modelId") != self._model:
logger.info(f"Switching TTS model to: [{self._model}]")
needs_reconnect = True
# Handle language explicitly
if "language" in settings:
new_lang = self.language_to_service_language(settings["language"])
if new_lang and new_lang != prev_settings.get("lang"):
logger.info(f"Updating language to: [{new_lang}]")
self._settings["lang"] = new_lang
needs_reconnect = True
# Check other parameters
for key in ["segment", "repetition_penalty", "temperature", "top_p"]:
if key in settings and settings[key] != prev_settings.get(key):
logger.info(f"Updating {key} to: [{settings[key]}]")
self._settings[key] = settings[key]
needs_reconnect = True
# Handle extra parameters
for key, value in settings.items():
if key not in [
"voice",
"voice_id",
"model",
"language",
"segment",
"repetition_penalty",
"temperature",
"top_p",
]:
if value != prev_settings.get(key):
logger.info(f"Updating extra parameter {key} to: [{value}]")
self._settings[key] = value
needs_reconnect = True
# Reconnect if any setting changed
if needs_reconnect:
if changed:
logger.debug("Settings changed, reconnecting WebSocket with new parameters")
await self._disconnect()
await self._connect()
return changed

12
src/pipecat/services/sambanova/llm.py
View File

@@ -84,19 +84,19 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore
Dictionary of parameters for the chat completion request.
"""
params = {
"model": self.model_name,
"model": self._settings.model,
"stream": True,
"stream_options": {"include_usage": True},
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"max_tokens": self._settings["max_tokens"],
"max_completion_tokens": self._settings["max_completion_tokens"],
"temperature": self._settings.temperature,
"top_p": self._settings.top_p,
"max_tokens": self._settings.max_tokens,
"max_completion_tokens": self._settings.max_completion_tokens,
}
# Messages, tools, tool_choice
params.update(params_from_context)
params.update(self._settings["extra"])
params.update(self._settings.extra)
return params
@traced_llm # type: ignore

2
src/pipecat/services/sambanova/stt.py
View File

@@ -72,7 +72,7 @@ class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore
# Build kwargs dict with only set parameters
kwargs = {
"file": ("audio.wav", audio, "audio/wav"),
"model": self.model_name,
"model": self._settings.model,
"response_format": "json",
"language": self._language,
}

209
src/pipecat/services/sarvam/stt.py
View File

@@ -12,8 +12,8 @@ can handle multiple audio formats for Indian language speech recognition.
"""
import base64
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Literal, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Literal, Optional
from loguru import logger
from pydantic import BaseModel
@@ -32,6 +32,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.sarvam._sdk import sdk_headers
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, is_given
from pipecat.services.stt_latency import SARVAM_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -119,10 +120,10 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = {
use_translate_method=True,
),
"saaras:v3": ModelConfig(
supports_prompt=True,
supports_prompt=False,
supports_mode=True,
supports_language=True,
default_language="en-IN",
default_language="unknown",
default_mode="transcribe",
use_translate_endpoint=False,
use_translate_method=False,
@@ -130,6 +131,23 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = {
}
@dataclass
class SarvamSTTSettings(STTSettings):
"""Settings for the Sarvam STT service.
Parameters:
prompt: Optional prompt to guide transcription/translation style.
mode: Mode of operation (transcribe, translate, verbatim, etc.).
vad_signals: Enable VAD signals in response.
high_vad_sensitivity: Enable high VAD sensitivity.
"""
prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
mode: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SarvamSTTService(STTService):
"""Sarvam speech-to-text service.
@@ -148,6 +166,8 @@ class SarvamSTTService(STTService):
...
"""
_settings: SarvamSTTSettings
class InputParams(BaseModel):
"""Configuration parameters for Sarvam STT service.
@@ -155,9 +175,9 @@ class SarvamSTTService(STTService):
language: Target language for transcription.
- saarika:v2.5: Defaults to "unknown" (auto-detect supported)
- saaras:v2.5: Not used (auto-detects language)
- saaras:v3: Defaults to "en-IN"
- saaras:v3: Defaults to "unknown" (auto-detect supported)
prompt: Optional prompt to guide transcription/translation style/context.
Only applicable to saaras models (v2.5 and v3). Defaults to None.
Only applicable to saaras:v2.5. Defaults to None.
mode: Mode of operation for saaras:v3 models only. Options: transcribe, translate,
verbatim, translit, codemix. Defaults to "transcribe" for saaras:v3.
vad_signals: Enable VAD signals in response. Defaults to None.
@@ -190,7 +210,7 @@ class SarvamSTTService(STTService):
model: Sarvam model to use for transcription. Allowed values:
- "saarika:v2.5": Standard STT model
- "saaras:v2.5": STT-Translate model (auto-detects language, supports prompts)
- "saaras:v3": Advanced STT model (supports mode and prompts)
- "saaras:v3": Advanced STT model (supports mode)
sample_rate: Audio sample rate. Defaults to 16000 if not specified.
input_audio_codec: Audio codec/format of the input file. Defaults to "wav".
params: Configuration parameters for Sarvam STT service.
@@ -220,32 +240,28 @@ class SarvamSTTService(STTService):
f"Model '{model}' does not support language parameter (auto-detects language)."
)
# Resolve mode default from model config
mode = params.mode if params.mode is not None else self._config.default_mode
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=keepalive_timeout,
keepalive_interval=keepalive_interval,
settings=SarvamSTTSettings(
model=model,
language=params.language,
prompt=params.prompt,
mode=mode,
vad_signals=params.vad_signals,
high_vad_sensitivity=params.high_vad_sensitivity,
),
**kwargs,
)
self.set_model_name(model)
self._api_key = api_key
self._language_code: Optional[Language] = params.language
# Set language string: use provided language or model's default
if params.language:
self._language_string = language_to_sarvam_language(params.language)
else:
self._language_string = self._config.default_language
self._prompt = params.prompt
# Set mode: use provided mode or model's default
self._mode = params.mode if params.mode is not None else self._config.default_mode
# Store connection parameters
self._vad_signals = params.vad_signals
self._high_vad_sensitivity = params.high_vad_sensitivity
self._input_audio_codec = input_audio_codec
# Initialize Sarvam SDK client
@@ -263,7 +279,7 @@ class SarvamSTTService(STTService):
self._socket_client = None
self._receive_task = None
if self._vad_signals:
if params.vad_signals:
self._register_event_handler("on_speech_started")
self._register_event_handler("on_speech_stopped")
self._register_event_handler("on_utterance_end")
@@ -281,6 +297,12 @@ class SarvamSTTService(STTService):
"""
return language_to_sarvam_language(language)
def _get_language_string(self) -> Optional[str]:
"""Resolve the current language setting to a Sarvam language code string."""
if self._settings.language:
return language_to_sarvam_language(self._settings.language)
return self._config.default_language
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -298,50 +320,91 @@ class SarvamSTTService(STTService):
await super().process_frame(frame, direction)
# Only handle VAD frames when not using Sarvam's VAD signals
if not self._vad_signals:
if not self._settings.vad_signals:
if isinstance(frame, VADUserStartedSpeakingFrame):
await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
if self._socket_client:
await self._socket_client.flush()
async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta, validate, sync state, and reconnect.
Args:
language: The language to use for speech recognition.
delta: A :class:`STTSettings` (or ``SarvamSTTSettings``) delta.
Returns:
Dict mapping changed field names to their previous values.
Raises:
ValueError: If called on a model that auto-detects language.
ValueError: If a setting is not supported by the current model.
"""
if not self._config.supports_language:
raise ValueError(
f"Model '{self.model_name}' does not support language parameter "
"(auto-detects language)."
)
# Validate against model capabilities before applying
if is_given(delta.language) and delta.language is not None:
if not self._config.supports_language:
raise ValueError(
f"Model '{self._settings.model}' does not support language parameter "
"(auto-detects language)."
)
logger.info(f"Switching STT language to: [{language}]")
self._language_code = language
self._language_string = language_to_sarvam_language(language)
await self._disconnect()
await self._connect()
if isinstance(delta, SarvamSTTSettings):
if is_given(delta.prompt) and delta.prompt is not None:
if not self._config.supports_prompt:
raise ValueError(
f"Model '{self._settings.model}' does not support prompt parameter."
)
if is_given(delta.mode) and delta.mode is not None:
if not self._config.supports_mode:
raise ValueError(
f"Model '{self._settings.model}' does not support mode parameter."
)
changed = await super()._update_settings(delta)
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# if not changed:
# return changed
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def set_prompt(self, prompt: Optional[str]):
"""Set the transcription/translation prompt and reconnect.
.. deprecated::
Use ``STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...))`` instead.
Args:
prompt: Prompt text to guide transcription/translation style/context.
Pass None to clear/disable prompt.
Only applicable to models that support prompts.
"""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
f"{self.__class__.__name__}.set_prompt() is deprecated. "
"Use STTUpdateSettingsFrame(SarvamSTTSettings(prompt=...)) instead.",
DeprecationWarning,
stacklevel=2,
)
if not self._config.supports_prompt:
if prompt is not None:
raise ValueError(f"Model '{self.model_name}' does not support prompt parameter.")
raise ValueError(
f"Model '{self._settings.model}' does not support prompt parameter."
)
# If prompt is None and model doesn't support prompts, silently return (no-op)
return
logger.info(f"Updating {self.model_name} prompt.")
self._prompt = prompt
logger.info(f"Updating {self._settings.model} prompt.")
self._settings.prompt = prompt
await self._disconnect()
await self._connect()
@@ -422,38 +485,58 @@ class SarvamSTTService(STTService):
try:
# Build common connection parameters
connect_kwargs = {
"model": self.model_name,
"model": self._settings.model,
"sample_rate": str(self.sample_rate),
}
# Enable flush signal when using Pipecat's VAD (not Sarvam's) so that
# the flush() call on user-stopped-speaking is honored by the server.
if not self._vad_signals:
if not self._settings.vad_signals:
connect_kwargs["flush_signal"] = "true"
# Only send vad parameters when explicitly set (avoid overriding server defaults)
if self._vad_signals is not None:
connect_kwargs["vad_signals"] = "true" if self._vad_signals else "false"
if self._high_vad_sensitivity is not None:
if self._settings.vad_signals is not None:
connect_kwargs["vad_signals"] = "true" if self._settings.vad_signals else "false"
if self._settings.high_vad_sensitivity is not None:
connect_kwargs["high_vad_sensitivity"] = (
"true" if self._high_vad_sensitivity else "false"
"true" if self._settings.high_vad_sensitivity else "false"
)
# Add language_code for models that support it
if self._language_string is not None:
connect_kwargs["language_code"] = self._language_string
language_string = self._get_language_string()
if language_string is not None:
connect_kwargs["language_code"] = language_string
# Add mode for models that support it
if self._config.supports_mode and self._mode is not None:
connect_kwargs["mode"] = self._mode
if self._config.supports_mode and self._settings.mode is not None:
connect_kwargs["mode"] = self._settings.mode
# Prompt support differs across sarvamai versions. Prefer connect-time prompt
# when available and gracefully degrade if the SDK doesn't accept it.
if self._settings.prompt is not None and self._config.supports_prompt:
connect_kwargs["prompt"] = self._settings.prompt
def _connect_with_sdk_headers(connect_fn, **kwargs):
# Different SDK versions may use different kwarg names.
for header_kw in ("headers", "additional_headers", "extra_headers"):
# If prompt is unsupported at connect-time, retry without it.
attempts = [kwargs]
if "prompt" in kwargs:
attempts.append({k: v for k, v in kwargs.items() if k != "prompt"})
last_type_error = None
for attempt_kwargs in attempts:
for header_kw in ("headers", "additional_headers", "extra_headers"):
try:
return connect_fn(**attempt_kwargs, **{header_kw: self._sdk_headers})
except TypeError as e:
last_type_error = e
try:
return connect_fn(**kwargs, **{header_kw: self._sdk_headers})
except TypeError:
pass
return connect_fn(**attempt_kwargs)
except TypeError as e:
last_type_error = e
if last_type_error is not None:
raise last_type_error
return connect_fn(**kwargs)
# Choose the appropriate endpoint based on model configuration
@@ -471,9 +554,11 @@ class SarvamSTTService(STTService):
# Enter the async context manager
self._socket_client = await self._websocket_context.__aenter__()
# Set prompt if provided (only for models that support prompts)
if self._prompt is not None and self._config.supports_prompt:
await self._socket_client.set_prompt(self._prompt)
# Fallback for SDKs that support runtime prompt updates.
if self._settings.prompt is not None and self._config.supports_prompt:
prompt_setter = getattr(self._socket_client, "set_prompt", None)
if callable(prompt_setter):
await prompt_setter(self._settings.prompt)
# Register event handler for incoming messages
def _message_handler(message):
@@ -571,10 +656,12 @@ class SarvamSTTService(STTService):
# Prefer language from message (auto-detected for translate models). Fallback to configured.
if language_code:
language = self._map_language_code_to_enum(language_code)
elif self._language_string:
language = self._map_language_code_to_enum(self._language_string)
else:
language = Language.HI_IN
language_string = self._get_language_string()
if language_string:
language = self._map_language_code_to_enum(language_string)
else:
language = Language.HI_IN
# Emit utterance end event
await self._call_event_handler("on_utterance_end")

281
src/pipecat/services/sarvam/tts.py
View File

@@ -40,9 +40,9 @@ See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for full API
import asyncio
import base64
import json
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple
from typing import Any, AsyncGenerator, ClassVar, Dict, List, Optional, Tuple
import aiohttp
from loguru import logger
@@ -62,7 +62,8 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.sarvam._sdk import sdk_headers
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import InterruptibleTTSService, TextAggregationMode, TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -244,6 +245,80 @@ def language_to_sarvam_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
@dataclass
class SarvamHttpTTSSettings(TTSSettings):
"""Settings for Sarvam HTTP TTS service.
Parameters:
language: Sarvam language code.
enable_preprocessing: Whether to enable text preprocessing. Defaults to False.
**Note:** Always enabled for bulbul:v3-beta (cannot be disabled).
pace: Speech pace multiplier. Defaults to 1.0.
- bulbul:v2: Range 0.3 to 3.0
- bulbul:v3-beta: Range 0.5 to 2.0
pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0).
Lower values = more deterministic, higher = more random. Defaults to 0.6.
**Note:** Only supported for bulbul:v3-beta. Ignored for v2.
sample_rate: Audio sample rate.
"""
language: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
sarvam_sample_rate: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class SarvamTTSSettings(TTSSettings):
"""Settings for Sarvam WebSocket TTS service.
Parameters:
language: Sarvam language code (e.g. ``"hi-IN"``). Uses the standard
``TTSSettings.language`` field.
speech_sample_rate: Audio sample rate as string.
enable_preprocessing: Enable text preprocessing. Defaults to False.
**Note:** Always enabled for bulbul:v3-beta.
min_buffer_size: Minimum characters to buffer before generating audio.
Lower values reduce latency but may affect quality. Defaults to 50.
max_chunk_length: Maximum characters processed in a single chunk.
Controls memory usage and processing efficiency. Defaults to 150.
output_audio_codec: Audio codec format. Options: linear16, mulaw, alaw,
opus, flac, aac, wav, mp3. Defaults to "linear16".
output_audio_bitrate: Audio bitrate (32k, 64k, 96k, 128k, 192k).
Defaults to "128k".
pace: Speech pace multiplier. Defaults to 1.0.
- bulbul:v2: Range 0.3 to 3.0
- bulbul:v3-beta: Range 0.5 to 2.0
pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
loudness: Volume multiplier (0.3 to 3.0). Defaults to 1.0.
**Note:** Only supported for bulbul:v2. Ignored for v3 models.
temperature: Controls output randomness for bulbul:v3-beta (0.01 to 1.0).
Lower = more deterministic, higher = more random. Defaults to 0.6.
**Note:** Only supported for bulbul:v3-beta. Ignored for v2.
"""
_aliases: ClassVar[Dict[str, str]] = {"target_language_code": "language"}
speech_sample_rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_preprocessing: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_buffer_size: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_chunk_length: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_audio_codec: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
output_audio_bitrate: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pace: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pitch: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
loudness: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SarvamHttpTTSService(TTSService):
"""Text-to-Speech service using Sarvam AI's API.
@@ -296,6 +371,8 @@ class SarvamHttpTTSService(TTSService):
)
"""
_settings: SarvamHttpTTSSettings
class InputParams(BaseModel):
"""Input parameters for Sarvam TTS configuration.
@@ -383,18 +460,12 @@ class SarvamHttpTTSService(TTSService):
if sample_rate is None:
sample_rate = self._config.default_sample_rate
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or SarvamHttpTTSService.InputParams()
# Set default voice based on model if not specified
if voice_id is None:
voice_id = self._config.default_speaker
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Validate and clamp pace to model's valid range
pace = params.pace
pace_min, pace_max = self._config.pace_range
@@ -402,37 +473,49 @@ class SarvamHttpTTSService(TTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = {
"language": (
self.language_to_service_language(params.language) if params.language else "en-IN"
super().__init__(
sample_rate=sample_rate,
settings=SarvamHttpTTSSettings(
language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
),
"enable_preprocessing": (
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
"pace": pace,
"model": model,
}
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
# Add parameters based on model support
if self._config.supports_pitch:
self._settings["pitch"] = params.pitch
self._settings.pitch = params.pitch
elif params.pitch != 0.0:
logger.warning(f"pitch parameter is ignored for {model}")
if self._config.supports_loudness:
self._settings["loudness"] = params.loudness
self._settings.loudness = params.loudness
elif params.loudness != 1.0:
logger.warning(f"loudness parameter is ignored for {model}")
if self._config.supports_temperature:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
elif params.temperature != 0.6:
logger.warning(f"temperature parameter is ignored for {model}")
self.set_model_name(model)
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -459,7 +542,7 @@ class SarvamHttpTTSService(TTSService):
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
self._settings.sarvam_sample_rate = self.sample_rate
@traced_tts
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
@@ -480,21 +563,25 @@ class SarvamHttpTTSService(TTSService):
# Build payload with common parameters
payload = {
"text": text,
"target_language_code": self._settings["language"],
"speaker": self._voice_id,
"target_language_code": self._settings.language,
"speaker": self._settings.voice,
"sample_rate": self.sample_rate,
"enable_preprocessing": self._settings["enable_preprocessing"],
"model": self._model_name,
"pace": self._settings.get("pace", 1.0),
"enable_preprocessing": self._settings.enable_preprocessing,
"model": self._settings.model,
"pace": self._settings.pace if self._settings.pace is not None else 1.0,
}
# Add model-specific parameters based on config
if self._config.supports_pitch:
payload["pitch"] = self._settings.get("pitch", 0.0)
payload["pitch"] = self._settings.pitch if self._settings.pitch is not None else 0.0
if self._config.supports_loudness:
payload["loudness"] = self._settings.get("loudness", 1.0)
payload["loudness"] = (
self._settings.loudness if self._settings.loudness is not None else 1.0
)
if self._config.supports_temperature:
payload["temperature"] = self._settings.get("temperature", 0.6)
payload["temperature"] = (
self._settings.temperature if self._settings.temperature is not None else 0.6
)
headers = {
"api-subscription-key": self._api_key,
@@ -605,6 +692,8 @@ class SarvamTTSService(InterruptibleTTSService):
See https://docs.sarvam.ai/api-reference-docs/text-to-speech/stream for API details.
"""
_settings: SarvamTTSSettings
class InputParams(BaseModel):
"""Configuration parameters for Sarvam TTS WebSocket service.
@@ -696,7 +785,8 @@ class SarvamTTSService(InterruptibleTTSService):
model: str = "bulbul:v2",
voice_id: Optional[str] = None,
url: str = "wss://api.sarvam.ai/text-to-speech/ws",
aggregate_sentences: Optional[bool] = True,
aggregate_sentences: Optional[bool] = None,
text_aggregation_mode: Optional[TextAggregationMode] = None,
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
**kwargs,
@@ -710,7 +800,12 @@ class SarvamTTSService(InterruptibleTTSService):
- "bulbul:v3-beta": Advanced model with temperature control
voice_id: Speaker voice ID. If None, uses model-appropriate default.
url: WebSocket URL for the TTS backend (default production URL).
aggregate_sentences: Merge multiple sentences into one audio chunk (default True).
aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead.
text_aggregation_mode: How to aggregate text before synthesis.
sample_rate: Output audio sample rate in Hz (8000, 16000, 22050, 24000).
If None, uses model-specific default.
params: Optional input parameters to override defaults.
@@ -729,26 +824,11 @@ class SarvamTTSService(InterruptibleTTSService):
if sample_rate is None:
sample_rate = self._config.default_sample_rate
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or SarvamTTSService.InputParams()
# Set default voice based on model if not specified
if voice_id is None:
voice_id = self._config.default_speaker
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
self.set_model_name(model)
self.set_voice(voice_id)
params = params or SarvamTTSService.InputParams()
# Validate and clamp pace to model's valid range
pace = params.pace
@@ -757,37 +837,57 @@ class SarvamTTSService(InterruptibleTTSService):
logger.warning(f"Pace {pace} is outside model range ({pace_min}-{pace_max}). Clamping.")
pace = max(pace_min, min(pace_max, pace))
# Build base settings
self._settings = {
"target_language_code": (
self.language_to_service_language(params.language) if params.language else "en-IN"
# Initialize parent class first
super().__init__(
aggregate_sentences=aggregate_sentences,
text_aggregation_mode=text_aggregation_mode,
push_text_frames=True,
pause_frame_processing=True,
push_stop_frames=True,
sample_rate=sample_rate,
settings=SarvamTTSSettings(
language=(
self.language_to_service_language(params.language)
if params.language
else "en-IN"
),
speech_sample_rate=str(sample_rate),
enable_preprocessing=(
True
if self._config.preprocessing_always_enabled
else params.enable_preprocessing
),
min_buffer_size=params.min_buffer_size,
max_chunk_length=params.max_chunk_length,
output_audio_codec=params.output_audio_codec,
output_audio_bitrate=params.output_audio_bitrate,
pace=pace,
pitch=None,
loudness=None,
temperature=None,
model=model,
voice=voice_id,
),
"speaker": voice_id,
"speech_sample_rate": str(sample_rate),
"enable_preprocessing": (
True if self._config.preprocessing_always_enabled else params.enable_preprocessing
),
"min_buffer_size": params.min_buffer_size,
"max_chunk_length": params.max_chunk_length,
"output_audio_codec": params.output_audio_codec,
"output_audio_bitrate": params.output_audio_bitrate,
"pace": pace,
"model": model,
}
**kwargs,
)
# WebSocket endpoint URL with model query parameter
self._websocket_url = f"{url}?model={model}"
self._api_key = api_key
# Add parameters based on model support
if self._config.supports_pitch:
self._settings["pitch"] = params.pitch
self._settings.pitch = params.pitch
elif params.pitch != 0.0:
logger.warning(f"pitch parameter is ignored for {model}")
if self._config.supports_loudness:
self._settings["loudness"] = params.loudness
self._settings.loudness = params.loudness
elif params.loudness != 1.0:
logger.warning(f"loudness parameter is ignored for {model}")
if self._config.supports_temperature:
self._settings["temperature"] = params.temperature
self._settings.temperature = params.temperature
elif params.temperature != 0.6:
logger.warning(f"temperature parameter is ignored for {model}")
@@ -823,7 +923,7 @@ class SarvamTTSService(InterruptibleTTSService):
await super().start(frame)
# WebSocket API expects sample rate as string
self._settings["speech_sample_rate"] = str(self.sample_rate)
self._settings.speech_sample_rate = str(self.sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -870,14 +970,15 @@ class SarvamTTSService(InterruptibleTTSService):
if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
await self.flush_audio()
async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a settings delta and resend config if voice changed."""
changed = await super()._update_settings(delta)
if changed:
await self._send_config()
return changed
async def _connect(self):
"""Connect to Sarvam WebSocket and start background tasks."""
await super()._connect()
@@ -934,9 +1035,27 @@ class SarvamTTSService(InterruptibleTTSService):
"""Send initial configuration message."""
if not self._websocket:
raise Exception("WebSocket not connected")
self._settings["speaker"] = self._voice_id
logger.debug(f"Config being sent is {self._settings}")
config_message = {"type": "config", "data": self._settings}
# Build config dict for the API
config_data = {
"target_language_code": self._settings.language,
"speaker": self._settings.voice,
"speech_sample_rate": self._settings.speech_sample_rate,
"enable_preprocessing": self._settings.enable_preprocessing,
"min_buffer_size": self._settings.min_buffer_size,
"max_chunk_length": self._settings.max_chunk_length,
"output_audio_codec": self._settings.output_audio_codec,
"output_audio_bitrate": self._settings.output_audio_bitrate,
"pace": self._settings.pace,
"model": self._settings.model,
}
if self._settings.pitch is not None:
config_data["pitch"] = self._settings.pitch
if self._settings.loudness is not None:
config_data["loudness"] = self._settings.loudness
if self._settings.temperature is not None:
config_data["temperature"] = self._settings.temperature
logger.debug(f"Config being sent is {config_data}")
config_message = {"type": "config", "data": config_data}
try:
await self._websocket.send(json.dumps(config_message))

433
src/pipecat/services/settings.py Normal file
View File

@@ -0,0 +1,433 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Settings infrastructure for Pipecat AI services.
Each service type has a settings dataclass (``LLMSettings``, ``TTSSettings``,
``STTSettings``, or a service-specific subclass). The same class is used in
two distinct modes:
**Store mode** — the service's ``self._settings`` object that holds the full
current state. Every field must have a real value; ``NOT_GIVEN`` is never
valid here. Services that don't support an inherited field should set it to
``None``. ``validate_complete()`` (called automatically in
``AIService.start()``) enforces this invariant.
**Delta mode** — a sparse update object carried by an
``*UpdateSettingsFrame``. Only the fields the caller wants to change are set;
all others remain at their default of ``NOT_GIVEN``. ``apply_update()``
merges a delta into a store, skipping any ``NOT_GIVEN`` fields.
Key helpers:
- ``NOT_GIVEN`` / ``is_given()`` — sentinel and check for "field not provided
in this delta".
- ``apply_update(delta)`` — merge a delta into a store, returning changed
fields.
- ``from_mapping(dict)`` — build a delta from a plain dict (for backward
compatibility with dict-based ``*UpdateSettingsFrame``).
- ``validate_complete()`` — assert that a store has no ``NOT_GIVEN`` fields.
- ``extra`` dict — overflow for service-specific keys that don't map to a
declared field.
"""
from __future__ import annotations
import copy
from dataclasses import dataclass, field, fields
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Mapping, Optional, Type, TypeVar
from loguru import logger
from pipecat.transcriptions.language import Language
if TYPE_CHECKING:
from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionConfig
# ---------------------------------------------------------------------------
# NOT_GIVEN sentinel
# ---------------------------------------------------------------------------
class _NotGiven:
"""Sentinel meaning "this field was not included in the delta".
``NOT_GIVEN`` is distinct from ``None`` (which is a valid stored value,
typically meaning "this service doesn't support this field"). Every
settings field defaults to ``NOT_GIVEN`` so that delta-mode objects are
sparse by default and ``apply_update`` can skip untouched fields.
``NOT_GIVEN`` must never appear in a store-mode object — see
``validate_complete()``.
"""
_instance: Optional[_NotGiven] = None
def __new__(cls) -> _NotGiven:
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __repr__(self) -> str:
return "NOT_GIVEN"
def __bool__(self) -> bool:
return False
NOT_GIVEN: _NotGiven = _NotGiven()
"""Singleton sentinel meaning "this field was not included in the delta".
Valid only in delta-mode settings objects. Must never appear in a service's
``self._settings`` (store mode) — use ``None`` instead for unsupported fields.
"""
def is_given(value: Any) -> bool:
"""Check whether a delta field was explicitly provided.
Typically used when processing a delta to decide whether a field
should be applied::
if is_given(delta.voice):
# caller wants to change the voice
...
For store-mode objects this always returns ``True`` (since
``validate_complete`` ensures no ``NOT_GIVEN`` fields remain).
Args:
value: The value to check.
Returns:
``True`` if *value* is anything other than ``NOT_GIVEN``.
"""
return not isinstance(value, _NotGiven)
# ---------------------------------------------------------------------------
# Base ServiceSettings
# ---------------------------------------------------------------------------
_S = TypeVar("_S", bound="ServiceSettings")
@dataclass
class ServiceSettings:
"""Base class for runtime-updatable service settings.
These settings capture the subset of a service's configuration that can
be changed **while the pipeline is running** (e.g. switching the model or
changing the voice). They are *not* meant to capture every constructor
parameter — only those that support live updates via
``*UpdateSettingsFrame``.
Every AI service type (LLM, TTS, STT) extends this with its own fields.
Each instance operates in one of two modes (see module docstring):
- **Store mode** (``self._settings``): holds the full current state.
Every field must be a real value — ``NOT_GIVEN`` is never valid.
Use ``None`` for inherited fields the service doesn't support.
Enforced at runtime by ``validate_complete()``.
- **Delta mode** (``*UpdateSettingsFrame``): a sparse update.
Only fields the caller wants to change are set; all others stay at
the default ``NOT_GIVEN`` and are skipped by ``apply_update()``.
Parameters:
model: The model identifier used by the service. Set to ``None``
in store mode if the service has no model concept.
extra: Overflow dict for service-specific keys that don't map to a
declared field.
"""
# -- common fields -------------------------------------------------------
model: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
"""AI model identifier (e.g. ``"gpt-4o"``, ``"eleven_turbo_v2_5"``).
Defaults to ``NOT_GIVEN`` for delta mode. In store mode, set to a
model string or ``None`` if the service has no model concept.
"""
extra: Dict[str, Any] = field(default_factory=dict)
"""Catch-all for service-specific keys that have no declared field."""
# -- class-level configuration -------------------------------------------
_aliases: ClassVar[Dict[str, str]] = {}
"""Map of alternative key names to canonical field names.
For example ``{"voice_id": "voice"}`` lets callers use either spelling.
Subclasses should override this as needed.
"""
# -- public API ----------------------------------------------------------
def given_fields(self) -> Dict[str, Any]:
"""Return a dict of only the fields that are not ``NOT_GIVEN``.
Primarily useful for delta-mode objects to inspect which fields were
set. For a store-mode object this returns all declared fields (since
none should be ``NOT_GIVEN``).
Skips the ``extra`` field itself but merges its entries into the
returned dict at the top level.
Returns:
Dictionary mapping field names to their provided values.
"""
result: Dict[str, Any] = {}
for f in fields(self):
if f.name == "extra":
continue
val = getattr(self, f.name)
if is_given(val):
result[f.name] = val
result.update(self.extra)
return result
def apply_update(self: _S, delta: _S) -> Dict[str, Any]:
"""Merge a delta-mode object into this store-mode object.
Only fields in *delta* that are **given** (i.e. not ``NOT_GIVEN``)
are considered. A field is "changed" if its new value differs from
the current value.
The ``extra`` dicts are merged: keys present in the delta overwrite
keys in the target.
Args:
delta: A delta-mode settings object of the same type.
Returns:
A dict mapping each changed field name to its **pre-update** value.
Use ``changed.keys()`` for the set of names, or index with
``changed["field"]`` to inspect the old value.
Examples::
# store-mode object (all fields given)
current = TTSSettings(voice="alice", language="en")
# delta-mode object (only voice is set)
delta = TTSSettings(voice="bob")
changed = current.apply_update(delta)
# changed == {"voice": "alice"}
# current.voice == "bob", current.language == "en"
"""
changed: Dict[str, Any] = {}
for f in fields(self):
if f.name == "extra":
continue
new_val = getattr(delta, f.name, NOT_GIVEN)
if not is_given(new_val):
continue
old_val = getattr(self, f.name)
if old_val != new_val:
setattr(self, f.name, new_val)
changed[f.name] = old_val
# Merge extra
for key, new_val in delta.extra.items():
old_val = self.extra.get(key, NOT_GIVEN)
if old_val != new_val:
self.extra[key] = new_val
changed[key] = old_val
return changed
@classmethod
def from_mapping(cls: Type[_S], settings: Mapping[str, Any]) -> _S:
"""Build a **delta-mode** settings object from a plain dictionary.
This exists for backward compatibility with code that passes plain
dicts via ``*UpdateSettingsFrame(settings={...})``. The returned
object is a delta: only the keys present in *settings* are set;
all other fields remain ``NOT_GIVEN``.
Keys are matched to dataclass fields by name. Keys listed in
``_aliases`` are translated to their canonical name first. Any
remaining unrecognized keys are placed into ``extra``.
Args:
settings: A dictionary of setting names to values.
Returns:
A new delta-mode settings instance.
Examples::
delta = TTSSettings.from_mapping({"voice_id": "alice", "speed": 1.2})
# delta.voice == "alice" (via alias)
# delta.language is NOT_GIVEN (not in the dict)
# delta.extra == {"speed": 1.2}
"""
field_names = {f.name for f in fields(cls)} - {"extra"}
kwargs: Dict[str, Any] = {}
extra: Dict[str, Any] = {}
for key, value in settings.items():
# Resolve aliases first
canonical = cls._aliases.get(key, key)
if canonical in field_names:
kwargs[canonical] = value
else:
extra[key] = value
instance = cls(**kwargs)
instance.extra = extra
return instance
def validate_complete(self) -> None:
"""Check that this is a valid store-mode object (no ``NOT_GIVEN`` fields).
Called automatically by ``AIService.start()`` to catch fields that a
service forgot to initialize in its ``__init__``. Can also be called
manually after constructing a store-mode settings object.
Logs a warning for each uninitialized field. Failure to initialize
all fields may or may not cause runtime issues — it depends on
whether and how the service actually reads the field — but it indicates
a deviation from expectations and should be fixed.
"""
missing = [
f.name
for f in fields(self)
if f.name != "extra" and isinstance(getattr(self, f.name), _NotGiven)
]
if missing:
names = ", ".join(missing)
logger.error(
f"{type(self).__name__}: the following fields are NOT_GIVEN: {names}. "
f"All settings fields should be initialized in the service's "
f"__init__ (use None for unsupported fields)."
)
def copy(self: _S) -> _S:
"""Return a deep copy of this settings instance.
Returns:
A new settings object with the same field values.
"""
return copy.deepcopy(self)
# ---------------------------------------------------------------------------
# Service-specific settings
# ---------------------------------------------------------------------------
@dataclass
class ImageGenSettings(ServiceSettings):
"""Runtime-updatable settings for image generation services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Image generation model identifier.
"""
@dataclass
class VisionSettings(ServiceSettings):
"""Runtime-updatable settings for vision services.
Used in both store and delta mode — see ``ServiceSettings``.
Parameters:
model: Vision model identifier.
"""
@dataclass
class LLMSettings(ServiceSettings):
"""Runtime-updatable settings for LLM services.
Used in both store and delta mode — see ``ServiceSettings``.
These fields are common across LLM providers. Not every provider supports
every field; in store mode, set unsupported fields to ``None`` (e.g. a
service that doesn't support ``seed`` should initialize it as
``seed=None``).
Parameters:
model: LLM model identifier.
temperature: Sampling temperature.
max_tokens: Maximum tokens to generate.
top_p: Nucleus sampling probability.
top_k: Top-k sampling parameter.
frequency_penalty: Frequency penalty.
presence_penalty: Presence penalty.
seed: Random seed for reproducibility.
filter_incomplete_user_turns: Enable LLM-based turn completion detection
to suppress bot responses when the user was cut off mid-thought.
See ``examples/foundational/22-filter-incomplete-turns.py`` and
``UserTurnCompletionLLMServiceMixin``.
user_turn_completion_config: Configuration for turn completion behavior
when ``filter_incomplete_user_turns`` is enabled. Controls timeouts
and prompts for incomplete turns.
"""
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_tokens: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_k: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
frequency_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
presence_penalty: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
seed: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
filter_incomplete_user_turns: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
user_turn_completion_config: UserTurnCompletionConfig | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
@dataclass
class TTSSettings(ServiceSettings):
"""Runtime-updatable settings for TTS services.
Used in both store and delta mode — see ``ServiceSettings``.
In store mode, set unsupported fields to ``None`` (e.g. ``language=None``
if the service doesn't expose a language setting).
Parameters:
model: TTS model identifier.
voice: Voice identifier or name.
language: Language for speech synthesis. The union type reflects the
*input* side: callers may pass a ``Language`` enum or a raw string
in a delta. However, the **stored** value (in store mode) is
always a service-specific string or ``None`` —
``TTSService._update_settings`` converts ``Language`` enums via
``language_to_service_language()`` before writing, and
``__init__`` methods do the same at construction time.
"""
voice: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language: Language | str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
_aliases: ClassVar[Dict[str, str]] = {"voice_id": "voice"}
@dataclass
class STTSettings(ServiceSettings):
"""Runtime-updatable settings for STT services.
Used in both store and delta mode — see ``ServiceSettings``.
In store mode, set unsupported fields to ``None`` (e.g. ``language=None``
if the service auto-detects language).
Parameters:
model: STT model identifier.
language: Language for speech recognition. The union type reflects the
*input* side: callers may pass a ``Language`` enum or a raw string
in a delta. However, the **stored** value (in store mode) is
always a service-specific string or ``None`` —
``STTService._update_settings`` converts ``Language`` enums via
``language_to_service_language()`` before writing, and
``__init__`` methods do the same at construction time.
"""
language: Language | str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)

7
src/pipecat/services/simli/video.py
View File

@@ -131,7 +131,6 @@ class SimliVideoService(FrameProcessor):
# Build SimliConfig from new parameters
# Only pass optional parameters if explicitly provided to use SimliConfig defaults
config_kwargs = {
"apiKey": api_key,
"faceId": face_id,
}
if params.max_session_length is not None:
@@ -153,10 +152,10 @@ class SimliVideoService(FrameProcessor):
config.maxIdleTime += 5
config.maxSessionLength += 5
self._simli_client = SimliClient(
api_key=api_key,
config=config,
latencyInterval=latency_interval,
simliURL=simli_url,
enable_logging=params.enable_logging or False,
enableSFU=True,
)
self._pipecat_resampler: AudioResampler = None
@@ -173,7 +172,7 @@ class SimliVideoService(FrameProcessor):
"""Start the connection to Simli service and begin processing tasks."""
try:
if not self._initialized:
await self._simli_client.Initialize()
await self._simli_client.start()
self._initialized = True
# Create task to consume and process audio and video

109
src/pipecat/services/soniox/stt.py
View File

@@ -8,7 +8,8 @@
import json
import time
from typing import AsyncGenerator, List, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, List, Optional
from loguru import logger
from pydantic import BaseModel
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import SONIOX_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
@@ -134,6 +136,35 @@ def _prepare_language_hints(
return list(set(prepared_languages))
@dataclass
class SonioxSTTSettings(STTSettings):
"""Settings for Soniox STT service.
Parameters:
audio_format: Audio format to use for transcription.
num_channels: Number of channels to use for transcription.
language_hints: List of language hints to use for transcription.
language_hints_strict: If true, strictly enforce language hints.
context: Customization for transcription. String for models with
context_version 1 and SonioxContextObject for models with
context_version 2.
enable_speaker_diarization: Whether to enable speaker diarization.
enable_language_identification: Whether to enable language identification.
client_reference_id: Client reference ID to use for transcription.
"""
audio_format: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
num_channels: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_hints: List[Language] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
language_hints_strict: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
context: SonioxContextObject | str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_speaker_diarization: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_language_identification: bool | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
client_reference_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SonioxSTTService(WebsocketSTTService):
"""Speech-to-Text service using Soniox's WebSocket API.
@@ -144,6 +175,8 @@ class SonioxSTTService(WebsocketSTTService):
For complete API documentation, see: https://soniox.com/docs/speech-to-text/api-reference/websocket-api
"""
_settings: SonioxSTTSettings
def __init__(
self,
*,
@@ -169,19 +202,30 @@ class SonioxSTTService(WebsocketSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to the STTService.
"""
params = params or SonioxInputParams()
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
keepalive_timeout=1,
keepalive_interval=5,
settings=SonioxSTTSettings(
model=params.model,
language=None,
audio_format=params.audio_format,
num_channels=params.num_channels,
language_hints=params.language_hints,
language_hints_strict=params.language_hints_strict,
context=params.context,
enable_speaker_diarization=params.enable_speaker_diarization,
enable_language_identification=params.enable_language_identification,
client_reference_id=params.client_reference_id,
),
**kwargs,
)
params = params or SonioxInputParams()
self._api_key = api_key
self._url = url
self.set_model_name(params.model)
self._params = params
self._vad_force_turn_endpoint = vad_force_turn_endpoint
self._final_transcription_buffer = []
@@ -189,6 +233,14 @@ class SonioxSTTService(WebsocketSTTService):
self._receive_task = None
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Soniox STT supports metrics generation.
"""
return True
async def start(self, frame: StartFrame):
"""Start the Soniox STT websocket connection.
@@ -198,6 +250,31 @@ class SonioxSTTService(WebsocketSTTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, delta: SonioxSTTSettings) -> dict[str, Any]:
"""Apply settings delta.
Settings are stored but not applied to the active connection.
Args:
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
# TODO: someday we could reconnect here to apply updated settings.
# Code might look something like the below:
# await self._disconnect()
# await self._connect()
self._warn_unhandled_updated_settings(changed)
return changed
async def stop(self, frame: EndFrame):
"""Stop the Soniox STT websocket connection.
@@ -233,10 +310,8 @@ class SonioxSTTService(WebsocketSTTService):
Yields:
Frame: None (transcription results come via WebSocket callbacks).
"""
await self.start_processing_metrics()
if self._websocket and self._websocket.state is State.OPEN:
await self._websocket.send(audio)
await self.stop_processing_metrics()
yield None
@@ -311,24 +386,26 @@ class SonioxSTTService(WebsocketSTTService):
# Either one or the other is required.
enable_endpoint_detection = not self._vad_force_turn_endpoint
context = self._params.context
s = self._settings
context = s.context
if isinstance(context, SonioxContextObject):
context = context.model_dump()
# Send the initial configuration message.
config = {
"api_key": self._api_key,
"model": self._model_name,
"audio_format": self._params.audio_format,
"num_channels": self._params.num_channels or 1,
"model": s.model,
"audio_format": s.audio_format,
"num_channels": s.num_channels or 1,
"enable_endpoint_detection": enable_endpoint_detection,
"sample_rate": self.sample_rate,
"language_hints": _prepare_language_hints(self._params.language_hints),
"language_hints_strict": self._params.language_hints_strict,
"language_hints": _prepare_language_hints(s.language_hints),
"language_hints_strict": s.language_hints_strict,
"context": context,
"enable_speaker_diarization": self._params.enable_speaker_diarization,
"enable_language_identification": self._params.enable_language_identification,
"client_reference_id": self._params.client_reference_id,
"enable_speaker_diarization": s.enable_speaker_diarization,
"enable_language_identification": s.enable_language_identification,
"client_reference_id": s.client_reference_id,
}
# Send the configuration message.
@@ -415,6 +492,8 @@ class SonioxSTTService(WebsocketSTTService):
# the rest will be sent as interim tokens (even final tokens).
await send_endpoint_transcript()
else:
if not self._final_transcription_buffer:
await self.start_processing_metrics()
self._final_transcription_buffer.append(token)
else:
non_final_transcription.append(token)

272
src/pipecat/services/speechmatics/stt.py
View File

@@ -8,8 +8,10 @@
import asyncio
import os
import warnings
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, AsyncGenerator
from typing import Any, AsyncGenerator, ClassVar
from dotenv import load_dotenv
from loguru import logger
@@ -31,6 +33,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import SPEECHMATICS_TTFS_P99
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -80,6 +83,83 @@ class TurnDetectionMode(str, Enum):
SMART_TURN = "smart_turn"
@dataclass
class SpeechmaticsSTTSettings(STTSettings):
"""Settings for Speechmatics STT service.
See ``SpeechmaticsSTTService.InputParams`` for detailed descriptions of each field.
Parameters:
model: The operating point / model name.
domain: Domain for Speechmatics API.
turn_detection_mode: Endpoint handling mode.
speaker_active_format: Formatter for active speaker ID.
speaker_passive_format: Formatter for passive speaker ID.
focus_speakers: List of speaker IDs to focus on.
ignore_speakers: List of speaker IDs to ignore.
focus_mode: Speaker focus mode for diarization.
known_speakers: List of known speaker labels and identifiers.
additional_vocab: List of additional vocabulary entries.
audio_encoding: Audio encoding format.
operating_point: Operating point for accuracy vs. latency.
max_delay: Maximum delay in seconds for transcription.
end_of_utterance_silence_trigger: Maximum delay for end of utterance trigger.
end_of_utterance_max_delay: Maximum delay for end of utterance.
punctuation_overrides: Punctuation overrides.
include_partials: Include partial segment fragments.
split_sentences: Emit finalized sentences mid-turn.
enable_diarization: Enable speaker diarization.
speaker_sensitivity: Diarization sensitivity.
max_speakers: Maximum number of speakers to detect.
prefer_current_speaker: Prefer current speaker ID.
extra_params: Extra parameters for the STT engine.
"""
domain: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
turn_detection_mode: TurnDetectionMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_active_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_passive_format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
focus_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
ignore_speakers: list[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
focus_mode: SpeakerFocusMode | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
known_speakers: list[SpeakerIdentifier] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
additional_vocab: list[AdditionalVocabEntry] | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
audio_encoding: AudioEncoding | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
operating_point: OperatingPoint | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
end_of_utterance_silence_trigger: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
end_of_utterance_max_delay: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
punctuation_overrides: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
include_partials: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
split_sentences: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_diarization: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
speaker_sensitivity: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_speakers: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prefer_current_speaker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
extra_params: dict[str, Any] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
#: Fields that can be updated on a live connection via the Speechmatics
#: diarization-config API — no reconnect needed.
HOT_FIELDS: ClassVar[frozenset[str]] = frozenset(
{
"focus_speakers",
"ignore_speakers",
"focus_mode",
}
)
#: Fields that are purely local (formatting templates) — no reconnect
#: and no API call needed.
LOCAL_FIELDS: ClassVar[frozenset[str]] = frozenset(
{
"speaker_active_format",
"speaker_passive_format",
}
)
class SpeechmaticsSTTService(STTService):
"""Speechmatics STT service implementation.
@@ -98,6 +178,8 @@ class SpeechmaticsSTTService(STTService):
...
"""
_settings: SpeechmaticsSTTSettings
# Export related classes as class attributes
TurnDetectionMode = TurnDetectionMode
AudioEncoding = AudioEncoding
@@ -316,8 +398,6 @@ class SpeechmaticsSTTService(STTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to STTService.
"""
super().__init__(sample_rate=sample_rate, ttfs_p99_latency=ttfs_p99_latency, **kwargs)
# Service parameters
self._api_key: str = api_key or os.getenv("SPEECHMATICS_API_KEY")
self._base_url: str = (
@@ -337,31 +417,62 @@ class SpeechmaticsSTTService(STTService):
# Deprecation check
self._check_deprecated_args(kwargs, params)
# Voice agent
# Output formatting defaults
speaker_active_format = params.speaker_active_format
if speaker_active_format is None:
speaker_active_format = (
"@{speaker_id}: {text}" if params.enable_diarization else "{text}"
)
speaker_passive_format = params.speaker_passive_format or speaker_active_format
# Settings — seeded from InputParams
settings = SpeechmaticsSTTSettings(
model=None, # Will be resolved from operating_point after config is built
language=params.language,
domain=params.domain,
turn_detection_mode=params.turn_detection_mode,
speaker_active_format=speaker_active_format,
speaker_passive_format=speaker_passive_format,
focus_speakers=params.focus_speakers,
ignore_speakers=params.ignore_speakers,
focus_mode=params.focus_mode,
known_speakers=params.known_speakers,
additional_vocab=params.additional_vocab,
audio_encoding=params.audio_encoding,
operating_point=params.operating_point,
max_delay=params.max_delay,
end_of_utterance_silence_trigger=params.end_of_utterance_silence_trigger,
end_of_utterance_max_delay=params.end_of_utterance_max_delay,
punctuation_overrides=params.punctuation_overrides,
include_partials=params.include_partials,
split_sentences=params.split_sentences,
enable_diarization=params.enable_diarization,
speaker_sensitivity=params.speaker_sensitivity,
max_speakers=params.max_speakers,
prefer_current_speaker=params.prefer_current_speaker,
extra_params=params.extra_params,
)
# Build SDK config from settings, then resolve model from operating_point
self._client: VoiceAgentClient | None = None
self._config: VoiceAgentConfig = self._prepare_config(params)
self._config: VoiceAgentConfig = self._build_config(settings)
settings.model = self._config.operating_point.value
super().__init__(
sample_rate=sample_rate,
ttfs_p99_latency=ttfs_p99_latency,
settings=settings,
**kwargs,
)
# Outbound frame queue
self._outbound_frames: asyncio.Queue[Frame] = asyncio.Queue()
# Output formatting
if params.speaker_active_format is None:
params.speaker_active_format = (
"@{speaker_id}: {text}" if params.enable_diarization else "{text}"
)
# Framework options
self._enable_vad: bool = self._config.end_of_utterance_mode not in [
EndOfUtteranceMode.FIXED,
EndOfUtteranceMode.EXTERNAL,
]
self._speaker_active_format: str = params.speaker_active_format
self._speaker_passive_format: str = (
params.speaker_passive_format or params.speaker_active_format
)
# Model + metrics
self.set_model_name(self._config.operating_point.value)
# Message queue
self._stt_msg_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
@@ -384,6 +495,64 @@ class SpeechmaticsSTTService(STTService):
await super().start(frame)
await self._connect()
async def _update_settings(self, delta: SpeechmaticsSTTSettings) -> dict[str, Any]:
"""Apply settings delta, reconnecting only when necessary.
Fields are classified into three categories (see
``SpeechmaticsSTTSettings``):
* **HOT_FIELDS** – diarization speaker settings that can be pushed
to a live Speechmatics connection without reconnecting.
* **LOCAL_FIELDS** – formatting templates evaluated locally; no
reconnect or API call needed.
* Everything else – baked into ``VoiceAgentConfig`` at connection
time and therefore require a full disconnect / reconnect.
Args:
delta: A settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
if not changed:
return changed
no_reconnect = SpeechmaticsSTTSettings.HOT_FIELDS | SpeechmaticsSTTSettings.LOCAL_FIELDS
needs_reconnect = bool(changed.keys() - no_reconnect)
if needs_reconnect:
logger.debug(f"{self} settings update requires reconnect: {changed.keys()}")
# Connection-level fields changed — rebuild the SDK config
# from the now-updated self._settings, then reconnect.
self._config = self._build_config(self._settings)
await self._disconnect()
await self._connect()
elif changed.keys() & SpeechmaticsSTTSettings.HOT_FIELDS:
logger.debug(f"{self} applying hot settings update: {changed.keys()}")
if self._config.enable_diarization:
# Only hot-updatable fields changed — push to the live session.
self._config.speaker_config.focus_speakers = self._settings.focus_speakers
self._config.speaker_config.ignore_speakers = self._settings.ignore_speakers
self._config.speaker_config.focus_mode = self._settings.focus_mode
if self._client:
self._client.update_diarization_config(self._config.speaker_config)
else:
logger.debug(
f"{self} hot settings updated but diarization not enabled: {changed.keys()}. ignoring."
)
# Diarization not enabled — the new settings will take effect
# if/when diarization is enabled, which does require a reconnect.
elif changed.keys() & SpeechmaticsSTTSettings.LOCAL_FIELDS:
logger.debug(
f"{self} local settings update, no special action required: {changed.keys()}"
)
# Only local fields changed — no need to push to the STT engine,
# the new settings will take effect immediately.
return changed
async def stop(self, frame: EndFrame):
"""Called when the session ends."""
await super().stop(frame)
@@ -494,28 +663,39 @@ class SpeechmaticsSTTService(STTService):
# CONFIGURATION
# ============================================================================
def _prepare_config(self, params: InputParams) -> VoiceAgentConfig:
"""Parse the InputParams into VoiceAgentConfig."""
# Preset
config = VoiceAgentConfigPreset.load(params.turn_detection_mode.value)
def _build_config(self, settings: SpeechmaticsSTTSettings) -> VoiceAgentConfig:
"""Build a ``VoiceAgentConfig`` from the given settings.
Used both at init time (with explicit settings, before
``super().__init__`` has run) and before reconnecting so the
connection always reflects the latest settings.
Args:
settings: Settings to build from.
"""
s = settings
# Preset from turn detection mode
config = VoiceAgentConfigPreset.load(s.turn_detection_mode.value)
# Language + domain
config.language = self._language_to_speechmatics_language(params.language)
config.domain = params.domain
config.output_locale = self._locale_to_speechmatics_locale(config.language, params.language)
language = s.language
config.language = self._language_to_speechmatics_language(language)
config.domain = s.domain if s.domain is not None else None
config.output_locale = self._locale_to_speechmatics_locale(config.language, language)
# Speaker config
config.speaker_config = SpeakerFocusConfig(
focus_speakers=params.focus_speakers,
ignore_speakers=params.ignore_speakers,
focus_mode=params.focus_mode,
focus_speakers=s.focus_speakers if s.focus_speakers is not None else [],
ignore_speakers=s.ignore_speakers if s.ignore_speakers is not None else [],
focus_mode=s.focus_mode if s.focus_mode is not None else SpeakerFocusMode.RETAIN,
)
config.known_speakers = params.known_speakers
config.known_speakers = s.known_speakers if s.known_speakers is not None else []
# Custom dictionary
config.additional_vocab = params.additional_vocab
config.additional_vocab = s.additional_vocab if s.additional_vocab is not None else []
# Advanced parameters
# Advanced parameters — only set if not None
for param in [
"operating_point",
"max_delay",
@@ -529,21 +709,20 @@ class SpeechmaticsSTTService(STTService):
"max_speakers",
"prefer_current_speaker",
]:
if getattr(params, param) is not None:
setattr(config, param, getattr(params, param))
val = getattr(s, param)
if val is not None:
setattr(config, param, val)
# Extra parameters
if isinstance(params.extra_params, dict):
for key, value in params.extra_params.items():
if isinstance(s.extra_params, dict):
for key, value in s.extra_params.items():
if hasattr(config, key):
setattr(config, key, value)
# Enable sentences
config.speech_segment_config = SpeechSegmentConfig(
emit_sentences=params.split_sentences or False
)
split = s.split_sentences if s.split_sentences is not None else False
config.speech_segment_config = SpeechSegmentConfig(emit_sentences=split or False)
# Return the complete config
return config
def update_params(
@@ -552,12 +731,23 @@ class SpeechmaticsSTTService(STTService):
) -> None:
"""Updates the speaker configuration.
.. deprecated::
Use ``STTUpdateSettingsFrame`` with
``SpeechmaticsSTTSettings(...)`` instead.
This can update the speakers to listen to or ignore during an in-flight
transcription. Only available if diarization is enabled.
Args:
params: Update parameters for the service.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"update_params() is deprecated. Use STTUpdateSettingsFrame with "
"SpeechmaticsSTTSettings(...) instead.",
DeprecationWarning,
)
# Check possible
if not self._config.enable_diarization:
raise ValueError("Diarization is not enabled")
@@ -727,9 +917,9 @@ class SpeechmaticsSTTService(STTService):
def attr_from_segment(segment: dict[str, Any]) -> dict[str, Any]:
# Formats the output text based on the speaker and defined formats from the config.
text = (
self._speaker_active_format
self._settings.speaker_active_format
if segment.get("is_active", True)
else self._speaker_passive_format
else self._settings.speaker_passive_format
).format(
**{
"speaker_id": segment.get("speaker_id", "UU"),

40
src/pipecat/services/speechmatics/tts.py
View File

@@ -7,7 +7,8 @@
"""Speechmatics TTS service integration."""
import asyncio
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from urllib.parse import urlencode
import aiohttp
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.utils.network import exponential_backoff_time
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -35,6 +37,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class SpeechmaticsTTSSettings(TTSSettings):
"""Settings for Speechmatics TTS service.
Parameters:
max_retries: Maximum number of retries for HTTP requests.
"""
max_retries: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SpeechmaticsTTSService(TTSService):
"""Speechmatics TTS service implementation.
@@ -42,6 +55,8 @@ class SpeechmaticsTTSService(TTSService):
It converts text to speech and returns raw PCM audio data for real-time playback.
"""
_settings: SpeechmaticsTTSSettings
SPEECHMATICS_SAMPLE_RATE = 16000
class InputParams(BaseModel):
@@ -80,7 +95,18 @@ class SpeechmaticsTTSService(TTSService):
f"Speechmatics TTS only supports {self.SPEECHMATICS_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or SpeechmaticsTTSService.InputParams()
super().__init__(
sample_rate=sample_rate,
settings=SpeechmaticsTTSSettings(
model=None,
voice=voice_id,
language=None,
max_retries=params.max_retries,
),
**kwargs,
)
# Service parameters
self._api_key: str = api_key
@@ -91,12 +117,6 @@ class SpeechmaticsTTSService(TTSService):
if not self._api_key:
raise ValueError("Missing Speechmatics API key")
# Default parameters
self._params = params or SpeechmaticsTTSService.InputParams()
# Set voice from constructor parameter
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
@@ -131,7 +151,7 @@ class SpeechmaticsTTSService(TTSService):
}
# Complete HTTP URL
url = _get_endpoint_url(self._base_url, self._voice_id, self.sample_rate)
url = _get_endpoint_url(self._base_url, self._settings.voice, self.sample_rate)
try:
# Start TTS TTFB metrics
@@ -159,7 +179,7 @@ class SpeechmaticsTTSService(TTSService):
attempt += 1
# Check if we've exceeded the maximum number of attempts
if attempt >= self._params.max_retries:
if attempt >= self._settings.max_retries:
raise ValueError()
# Report error frame

165
src/pipecat/services/stt_service.py
View File

@@ -9,9 +9,10 @@
import asyncio
import io
import time
import warnings
import wave
from abc import abstractmethod
from typing import Any, AsyncGenerator, Dict, Mapping, Optional
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from websockets.protocol import State
@@ -21,7 +22,6 @@ from pipecat.frames.frames import (
ErrorFrame,
Frame,
InterruptionFrame,
MetricsFrame,
ServiceSwitcherRequestMetadataFrame,
StartFrame,
STTMetadataFrame,
@@ -31,9 +31,9 @@ from pipecat.frames.frames import (
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.metrics.metrics import TTFBMetricsData
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import STTSettings, is_given
from pipecat.services.stt_latency import DEFAULT_TTFS_P99
from pipecat.services.websocket_service import WebsocketService
from pipecat.transcriptions.language import Language
@@ -75,6 +75,8 @@ class STTService(AIService):
logger.error(f"STT connection error: {error}")
"""
_settings: STTSettings
def __init__(
self,
*,
@@ -84,6 +86,7 @@ class STTService(AIService):
ttfs_p99_latency: Optional[float] = None,
keepalive_timeout: Optional[float] = None,
keepalive_interval: float = 5.0,
settings: Optional[STTSettings] = None,
**kwargs,
):
"""Initialize the STT service.
@@ -107,13 +110,20 @@ class STTService(AIService):
connection alive. None disables keepalive. Useful for services that
close idle connections (e.g. behind a ServiceSwitcher).
keepalive_interval: Seconds between idle checks when keepalive is enabled.
settings: The runtime-updatable settings for the STT service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or STTSettings(),
**kwargs,
)
self._audio_passthrough = audio_passthrough
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._settings: Dict[str, Any] = {}
self._muted: bool = False
self._user_id: str = ""
self._ttfs_p99_latency = ttfs_p99_latency
@@ -121,11 +131,10 @@ class STTService(AIService):
# STT TTFB tracking state
self._stt_ttfb_timeout = stt_ttfb_timeout
self._ttfb_timeout_task: Optional[asyncio.Task] = None
self._speech_end_time: Optional[float] = None
self._user_speaking: bool = False
self._last_transcription_time: Optional[float] = None
self._finalize_pending: bool = False
self._finalize_requested: bool = False
self._last_transcript_time: float = 0
# Keepalive state
self._keepalive_timeout = keepalive_timeout
@@ -183,18 +192,53 @@ class STTService(AIService):
async def set_model(self, model: str):
"""Set the speech recognition model.
.. deprecated:: 0.0.104
Use ``STTUpdateSettingsFrame(model=...)`` instead.
Args:
model: The name of the model to use for speech recognition.
"""
self.set_model_name(model)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated, use 'STTUpdateSettingsFrame(model=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching STT model to: [{model}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(model=model))
async def set_language(self, language: Language):
"""Set the language for speech recognition.
.. deprecated:: 0.0.104
Use ``STTUpdateSettingsFrame(language=...)`` instead.
Args:
language: The language to use for speech recognition.
"""
pass
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_language' is deprecated, use 'STTUpdateSettingsFrame(language=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching STT language to: [{language}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(language=language))
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a language to the service-specific language format.
Args:
language: The language to convert.
Returns:
The service-specific language identifier, or None if not supported.
"""
return Language(language)
@abstractmethod
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
@@ -226,20 +270,29 @@ class STTService(AIService):
await self._cancel_ttfb_timeout()
await self._cancel_keepalive_task()
async def _update_settings(self, settings: Mapping[str, Any]):
logger.info(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating STT setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
await self.set_language(value)
elif key == "language":
await self.set_language(value)
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for STT service: {key}")
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply an STT settings delta.
Handles ``model`` (via parent). Translates ``Language`` enum values
before applying so the stored value is a service-specific string.
Concrete services should override this method and handle language
changes (including any reconnect logic) based on the returned
changed-field dict.
Args:
delta: An STT settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
# Translate language *before* applying so the stored value is canonical
if is_given(delta.language) and isinstance(delta.language, Language):
converted = self.language_to_service_language(delta.language)
if converted is not None:
delta.language = converted
changed = await super()._update_settings(delta)
return changed
async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
"""Process an audio frame for speech recognition.
@@ -304,7 +357,20 @@ class STTService(AIService):
await self._handle_vad_user_stopped_speaking(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, STTUpdateSettingsFrame):
await self._update_settings(frame.settings)
if frame.delta is not None:
await self._update_settings(frame.delta)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via STTUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.104, use STTUpdateSettingsFrame(delta=STTSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
delta = type(self._settings).from_mapping(frame.settings)
await self._update_settings(delta)
elif isinstance(frame, STTMuteFrame):
self._muted = frame.mute
logger.debug(f"STT service {'muted' if frame.mute else 'unmuted'}")
@@ -327,8 +393,8 @@ class STTService(AIService):
direction: The direction to push the frame.
"""
if isinstance(frame, TranscriptionFrame):
# Store the transcription time for TTFB calculation
self._last_transcription_time = time.time()
# Store the transcript time for TTFB calculation
self._last_transcript_time = time.time()
# Set finalized from pending state and auto-reset
if self._finalize_pending:
@@ -336,14 +402,10 @@ class STTService(AIService):
self._finalize_pending = False
# If this is a finalized transcription, report TTFB immediately
if frame.finalized and self._speech_end_time is not None:
ttfb = self._last_transcription_time - self._speech_end_time
await self._emit_stt_ttfb_metric(ttfb)
if frame.finalized:
await self.stop_ttfb_metrics()
# Cancel the timeout since we've already reported
await self._cancel_ttfb_timeout()
# Clear state
self._speech_end_time = None
self._last_transcription_time = None
await super().push_frame(frame, direction)
@@ -373,8 +435,6 @@ class STTService(AIService):
while user is still speaking.
"""
await self._cancel_ttfb_timeout()
self._speech_end_time = None
self._last_transcription_time = None
async def _handle_vad_user_started_speaking(self, frame: VADUserStartedSpeakingFrame):
"""Handle VAD user started speaking frame to start tracking transcriptions.
@@ -389,6 +449,7 @@ class STTService(AIService):
self._user_speaking = True
self._finalize_requested = False
self._finalize_pending = False
self._last_transcript_time = 0
async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame):
"""Handle VAD user stopped speaking frame.
@@ -408,7 +469,8 @@ class STTService(AIService):
# Calculate the actual speech end time (current time minus VAD stop delay).
# This approximates when the last user audio was sent to the STT service,
# which we use to measure against the eventual transcription response.
self._speech_end_time = frame.timestamp - frame.stop_secs
speech_end_time = frame.timestamp - frame.stop_secs
await self.start_ttfb_metrics(start_time=speech_end_time)
# Start timeout task (any previous timeout was cancelled by VADUserStartedSpeakingFrame
# or InterruptionFrame)
@@ -417,44 +479,23 @@ class STTService(AIService):
)
async def _ttfb_timeout_handler(self):
"""Wait for timeout then report TTFB using the last transcription timestamp.
"""Wait for timeout then report TTFB using the last transcript timestamp.
This timeout allows the final transcription to arrive before we calculate
and report TTFB. If no transcription arrived, no TTFB is reported.
and report TTFB. Uses _last_transcript_time as the end time so we measure
to when the transcript actually arrived, not when the timeout fired.
If no transcription arrived, no TTFB is reported.
"""
try:
await asyncio.sleep(self._stt_ttfb_timeout)
# Report TTFB if we have both speech end time and transcription time
if self._speech_end_time is not None and self._last_transcription_time is not None:
ttfb = self._last_transcription_time - self._speech_end_time
await self._emit_stt_ttfb_metric(ttfb)
# Clear state after reporting
self._speech_end_time = None
self._last_transcription_time = None
if self._last_transcript_time > 0:
await self.stop_ttfb_metrics(end_time=self._last_transcript_time)
except asyncio.CancelledError:
# Task was cancelled (new utterance or interruption), which is expected behavior
pass
finally:
self._ttfb_timeout_task = None
async def _emit_stt_ttfb_metric(self, ttfb: float):
"""Emit STT TTFB metric if value is non-negative.
Args:
ttfb: The TTFB value in seconds.
"""
if ttfb >= 0:
logger.debug(f"{self} TTFB: {ttfb:.3f}s")
if self.metrics_enabled:
ttfb_data = TTFBMetricsData(
processor=self.name,
model=self.model_name,
value=ttfb,
)
await super().push_frame(MetricsFrame(data=[ttfb_data]))
def _create_keepalive_task(self):
"""Start the keepalive task if keepalive is enabled."""
if self._keepalive_timeout is not None:

575
src/pipecat/services/tts_service.py
View File

@@ -8,8 +8,10 @@
import asyncio
import uuid
import warnings
from abc import abstractmethod
from dataclasses import dataclass
from enum import Enum
from typing import (
Any,
AsyncGenerator,
@@ -18,7 +20,6 @@ from typing import (
Callable,
Dict,
List,
Mapping,
Optional,
Sequence,
Tuple,
@@ -38,6 +39,7 @@ from pipecat.frames.frames import (
Frame,
InterimTranscriptionFrame,
InterruptionFrame,
LLMAssistantPushAggregationFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
StartFrame,
@@ -52,6 +54,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import TTSSettings, is_given
from pipecat.services.websocket_service import WebsocketService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
@@ -65,10 +68,33 @@ class TTSContext:
"""Context information for a TTS request.
Attributes:
append_to_context: Whether this TTS output should be appended to the conversation context.
append_to_context: Whether this TTS output should be appended to the
conversation context after it is spoken.
push_assistant_aggregation: Whether to push an
``LLMAssistantPushAggregationFrame`` after the TTS has finished
speaking, forcing the assistant aggregator to commit its current
text buffer to the conversation context.
"""
append_to_context: bool = True
push_assistant_aggregation: Optional[bool] = False
class TextAggregationMode(str, Enum):
"""Controls how incoming text is aggregated before TTS synthesis.
Parameters:
SENTENCE: Buffer text until sentence boundaries are detected before synthesis.
Produces more natural speech but adds latency (~200-300ms per sentence).
TOKEN: Stream text tokens directly to TTS as they arrive.
Reduces latency but may affect speech quality depending on the TTS provider.
"""
SENTENCE = "sentence"
TOKEN = "token"
def __str__(self):
return self.value
class TTSService(AIService):
@@ -103,10 +129,13 @@ class TTSService(AIService):
logger.debug(f"TTS request: {context_id} - {text}")
"""
_settings: TTSSettings
def __init__(
self,
*,
aggregate_sentences: bool = True,
text_aggregation_mode: Optional[TextAggregationMode] = None,
aggregate_sentences: Optional[bool] = None,
# if True, TTSService will push TextFrames and LLMFullResponseEndFrames,
# otherwise subclass must do it
push_text_frames: bool = True,
@@ -125,6 +154,8 @@ class TTSService(AIService):
append_trailing_space: bool = False,
# TTS output sample rate
sample_rate: Optional[int] = None,
# if True, enables word-level timestamp tracking and synchronization
supports_word_timestamps: bool = False,
# Text aggregator to aggregate incoming tokens and decide when to push to the TTS.
text_aggregator: Optional[BaseTextAggregator] = None,
# Types of text aggregations that should not be spoken.
@@ -142,12 +173,22 @@ class TTSService(AIService):
text_filter: Optional[BaseTextFilter] = None,
# Audio transport destination of the generated frames.
transport_destination: Optional[str] = None,
settings: Optional[TTSSettings] = None,
**kwargs,
):
"""Initialize the TTS service.
Args:
text_aggregation_mode: How to aggregate incoming text before synthesis.
TextAggregationMode.SENTENCE (default) buffers until sentence boundaries,
TextAggregationMode.TOKEN streams tokens directly for lower latency.
aggregate_sentences: Whether to aggregate text into sentences before synthesis.
.. deprecated:: 0.0.104
Use ``text_aggregation_mode`` instead. Set to ``TextAggregationMode.SENTENCE``
to aggregate text into sentences before synthesis, or
``TextAggregationMode.TOKEN`` to stream tokens directly for lower latency.
push_text_frames: Whether to push TextFrames and LLMFullResponseEndFrames.
push_stop_frames: Whether to automatically push TTSStoppedFrames.
stop_frame_timeout_s: Idle time before pushing TTSStoppedFrame when push_stop_frames is True.
@@ -157,6 +198,9 @@ class TTSService(AIService):
append_trailing_space: Whether to append a trailing space to text before sending to TTS.
This helps prevent some TTS services from vocalizing trailing punctuation (e.g., "dot").
sample_rate: Output sample rate for generated audio.
supports_word_timestamps: Whether this service supports word-level timestamp tracking.
When True, enables synchronization of audio with spoken words so only spoken words
are added to the conversation context.
text_aggregator: Custom text aggregator for processing incoming text.
.. deprecated:: 0.0.95
@@ -175,10 +219,41 @@ class TTSService(AIService):
Use `text_filters` instead, which allows multiple filters.
transport_destination: Destination for generated audio frames.
settings: The runtime-updatable settings for the TTS service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
self._aggregate_sentences: bool = aggregate_sentences
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or TTSSettings(),
**kwargs,
)
# Resolve text_aggregation_mode from the new param or deprecated aggregate_sentences
if aggregate_sentences is not None:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'aggregate_sentences' is deprecated. "
"Use 'text_aggregation_mode=TextAggregationMode.SENTENCE' or "
"'text_aggregation_mode=TextAggregationMode.TOKEN' instead.",
DeprecationWarning,
stacklevel=2,
)
if text_aggregation_mode is None:
text_aggregation_mode = (
TextAggregationMode.SENTENCE
if aggregate_sentences
else TextAggregationMode.TOKEN
)
if text_aggregation_mode is None:
text_aggregation_mode = TextAggregationMode.SENTENCE
self._text_aggregation_mode: TextAggregationMode = text_aggregation_mode
self._push_text_frames: bool = push_text_frames
self._push_stop_frames: bool = push_stop_frames
self._stop_frame_timeout_s: float = stop_frame_timeout_s
@@ -188,9 +263,9 @@ class TTSService(AIService):
self._append_trailing_space: bool = append_trailing_space
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator(
aggregation_type=self._text_aggregation_mode
)
if text_aggregator:
import warnings
@@ -226,12 +301,55 @@ class TTSService(AIService):
self._processing_text: bool = False
self._tts_contexts: Dict[str, TTSContext] = {}
self._streamed_text: str = ""
self._text_aggregation_metrics_started: bool = False
# Word timestamp state (active when supports_word_timestamps=True)
self._supports_word_timestamps: bool = supports_word_timestamps
self._initial_word_timestamp: int = -1
self._initial_word_times: List[Tuple[str, float, Optional[str]]] = []
self._words_task: Optional[asyncio.Task] = None
self._llm_response_started: bool = False
self._register_event_handler("on_connected")
self._register_event_handler("on_disconnected")
self._register_event_handler("on_connection_error")
self._register_event_handler("on_tts_request")
@property
def _is_streaming_tokens(self) -> bool:
"""Whether the service is streaming tokens directly without sentence aggregation."""
return self._text_aggregation_mode == TextAggregationMode.TOKEN
async def start_tts_usage_metrics(self, text: str):
"""Record TTS usage metrics.
When streaming tokens, usage metrics are aggregated and reported at
flush time instead of per token, so individual calls are skipped.
Args:
text: The text being processed by TTS.
"""
if self._is_streaming_tokens:
return
await super().start_tts_usage_metrics(text)
async def start_text_aggregation_metrics(self):
"""Start text aggregation metrics if not already started.
Only starts the metric once per LLM response. Skipped when streaming
tokens since per-token aggregation time is not meaningful.
"""
if self._is_streaming_tokens or self._text_aggregation_metrics_started:
return
self._text_aggregation_metrics_started = True
await super().start_text_aggregation_metrics()
async def stop_text_aggregation_metrics(self):
"""Stop text aggregation metrics and reset the started flag."""
self._text_aggregation_metrics_started = False
await super().stop_text_aggregation_metrics()
@property
def sample_rate(self) -> int:
"""Get the current sample rate for audio output.
@@ -261,18 +379,42 @@ class TTSService(AIService):
async def set_model(self, model: str):
"""Set the TTS model to use.
.. deprecated:: 0.0.104
Use ``TTSUpdateSettingsFrame(model=...)`` instead.
Args:
model: The name of the TTS model.
"""
self.set_model_name(model)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_model' is deprecated, use 'TTSUpdateSettingsFrame(model=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching TTS model to: [{model}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(model=model))
def set_voice(self, voice: str):
async def set_voice(self, voice: str):
"""Set the voice for speech synthesis.
.. deprecated:: 0.0.104
Use ``TTSUpdateSettingsFrame(voice=...)`` instead.
Args:
voice: The voice identifier or name.
"""
self._voice_id = voice
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'set_voice' is deprecated, use 'TTSUpdateSettingsFrame(voice=...)' instead.",
DeprecationWarning,
stacklevel=2,
)
logger.info(f"Switching TTS voice to: [{voice}]")
settings_cls = type(self._settings)
await self._update_settings(settings_cls(voice=voice))
def create_context_id(self) -> str:
"""Generate a unique context ID for a TTS request.
@@ -324,15 +466,6 @@ class TTSService(AIService):
return text + " "
return text
async def update_setting(self, key: str, value: Any):
"""Update a service-specific setting.
Args:
key: The setting key to update.
value: The new value for the setting.
"""
pass
async def flush_audio(self):
"""Flush any buffered audio data."""
pass
@@ -347,6 +480,8 @@ class TTSService(AIService):
self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
if self._push_stop_frames and not self._stop_frame_task:
self._stop_frame_task = self.create_task(self._stop_frame_handler())
if self._supports_word_timestamps:
self._create_words_task()
async def stop(self, frame: EndFrame):
"""Stop the TTS service.
@@ -358,6 +493,8 @@ class TTSService(AIService):
if self._stop_frame_task:
await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
if self._words_task:
await self._stop_words_task()
async def cancel(self, frame: CancelFrame):
"""Cancel the TTS service.
@@ -369,6 +506,8 @@ class TTSService(AIService):
if self._stop_frame_task:
await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
if self._words_task:
await self._stop_words_task()
def add_text_transformer(
self,
@@ -403,22 +542,26 @@ class TTSService(AIService):
if not (agg_type == aggregation_type and func == transform_function)
]
async def _update_settings(self, settings: Mapping[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating TTS setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
self._settings[key] = self.language_to_service_language(value)
elif key == "model":
self.set_model_name(value)
elif key == "voice" or key == "voice_id":
self.set_voice(value)
elif key == "text_filter":
for filter in self._text_filters:
await filter.update_settings(value)
else:
logger.warning(f"Unknown setting for TTS service: {key}")
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
"""Apply a TTS settings delta.
Translates language to service-specific value before applying.
Args:
delta: A TTS settings delta.
Returns:
Dict mapping changed field names to their previous values.
"""
# Translate language *before* applying so the stored value is canonical
if is_given(delta.language) and isinstance(delta.language, Language):
converted = self.language_to_service_language(delta.language)
if converted is not None:
delta.language = converted
changed = await super()._update_settings(delta)
return changed
async def say(self, text: str):
"""Immediately speak the provided text.
@@ -465,10 +608,14 @@ class TTSService(AIService):
and not isinstance(frame, InterimTranscriptionFrame)
and not isinstance(frame, TranscriptionFrame)
):
await self.start_text_aggregation_metrics()
await self._process_text_frame(frame)
elif isinstance(frame, InterruptionFrame):
await self._handle_interruption(frame, direction)
await self.push_frame(frame, direction)
elif isinstance(frame, LLMFullResponseStartFrame):
self._llm_response_started = True
await self.push_frame(frame, direction)
elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
# We pause processing incoming frames if the LLM response included
# text (it might be that it's only a function calling response). We
@@ -477,9 +624,17 @@ class TTSService(AIService):
# Flush any remaining text (including text waiting for lookahead)
remaining = await self._text_aggregator.flush()
# Stop the aggregation metric (no-op if already stopped on first sentence).
await self.stop_text_aggregation_metrics()
if remaining:
await self._push_tts_frames(AggregatedTextFrame(remaining.text, remaining.type))
# Log accumulated streamed text and emit aggregated usage metric.
if self._streamed_text:
logger.debug(f"{self}: Generating TTS [{self._streamed_text}]")
await super().start_tts_usage_metrics(self._streamed_text)
self._streamed_text = ""
# Reset aggregator state
self._processing_text = False
if isinstance(frame, LLMFullResponseEndFrame):
@@ -487,13 +642,19 @@ class TTSService(AIService):
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
# Flush any pending audio so the TTS service closes the current context.
if self._supports_word_timestamps:
await self.flush_audio()
elif isinstance(frame, TTSSpeakFrame):
# Store if we were processing text or not so we can set it back.
processing_text = self._processing_text
# If we are not receiving text from the LLM, we can assume that the SpeakFrame should be automatically added to the context
push_assistant_aggregation = frame.append_to_context and not self._llm_response_started
# Assumption: text in TTSSpeakFrame does not include inter-frame spaces
await self._push_tts_frames(
AggregatedTextFrame(frame.text, AggregationType.SENTENCE),
append_tts_text_to_context=frame.append_to_context,
push_assistant_aggregation=push_assistant_aggregation,
)
# We pause processing incoming frames because we are sending data to
# the TTS. We pause to avoid audio overlapping.
@@ -501,7 +662,20 @@ class TTSService(AIService):
await self.flush_audio()
self._processing_text = processing_text
elif isinstance(frame, TTSUpdateSettingsFrame):
await self._update_settings(frame.settings)
if frame.delta is not None:
await self._update_settings(frame.delta)
elif frame.settings:
# Backward-compatible path: convert legacy dict to settings object.
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Passing a dict via TTSUpdateSettingsFrame(settings={...}) is deprecated "
"since 0.0.104, use TTSUpdateSettingsFrame(delta=TTSSettings(...)) instead.",
DeprecationWarning,
stacklevel=2,
)
delta = type(self._settings).from_mapping(frame.settings)
await self._update_settings(delta)
elif isinstance(frame, BotStoppedSpeakingFrame):
await self._maybe_resume_frame_processing()
await self.push_frame(frame, direction)
@@ -612,6 +786,12 @@ class TTSService(AIService):
for filter in self._text_filters:
await filter.handle_interruption()
self._llm_response_started = False
self._streamed_text = ""
self._text_aggregation_metrics_started = False
if self._supports_word_timestamps:
await self.reset_word_timestamps()
async def _maybe_pause_frame_processing(self):
if self._processing_text and self._pause_frame_processing:
await self.pause_processing_frames()
@@ -621,32 +801,25 @@ class TTSService(AIService):
await self.resume_processing_frames()
async def _process_text_frame(self, frame: TextFrame):
text: Optional[str] = None
includes_inter_frame_spaces: bool = False
if not self._aggregate_sentences:
text = frame.text
includes_inter_frame_spaces = frame.includes_inter_frame_spaces
aggregated_by = "token"
if text:
logger.trace(f"Pushing TTS frames for text: {text}, {aggregated_by}")
await self._push_tts_frames(
AggregatedTextFrame(text, aggregated_by), includes_inter_frame_spaces
)
else:
async for aggregate in self._text_aggregator.aggregate(frame.text):
text = aggregate.text
aggregated_by = aggregate.type
logger.trace(f"Pushing TTS frames for text: {text}, {aggregated_by}")
await self._push_tts_frames(
AggregatedTextFrame(text, aggregated_by), includes_inter_frame_spaces
)
async for aggregate in self._text_aggregator.aggregate(frame.text):
includes_inter_frame_spaces = (
frame.includes_inter_frame_spaces
if aggregate.type == AggregationType.TOKEN
else False
)
if aggregate.type != AggregationType.TOKEN:
# Stop the aggregation metric on the first sentence only.
await self.stop_text_aggregation_metrics()
await self._push_tts_frames(
AggregatedTextFrame(aggregate.text, aggregate.type), includes_inter_frame_spaces
)
async def _push_tts_frames(
self,
src_frame: AggregatedTextFrame,
includes_inter_frame_spaces: Optional[bool] = False,
append_tts_text_to_context: Optional[bool] = True,
push_assistant_aggregation: Optional[bool] = False,
):
type = src_frame.aggregated_by
text = src_frame.text
@@ -670,7 +843,15 @@ class TTSService(AIService):
# or when we received an LLMFullResponseEndFrame
self._processing_text = True
await self.start_processing_metrics()
# Accumulate text for a single debug log at flush time when streaming tokens.
if self._is_streaming_tokens:
self._streamed_text += text
# Skip per-token processing metrics when streaming. The per-token
# processing time is just websocket send overhead (~0.1ms) and not
# meaningful. TTFB captures the important timing for streaming TTS.
if not self._is_streaming_tokens:
await self.start_processing_metrics()
# Process all filters.
for filter in self._text_filters:
@@ -678,7 +859,8 @@ class TTSService(AIService):
text = await filter.filter(text)
if not text.strip():
await self.stop_processing_metrics()
if not self._is_streaming_tokens:
await self.stop_processing_metrics()
return
# Create context ID and store metadata
@@ -705,7 +887,8 @@ class TTSService(AIService):
self._tts_contexts[context_id] = TTSContext(
append_to_context=append_tts_text_to_context
if append_tts_text_to_context is not None
else True
else True,
push_assistant_aggregation=push_assistant_aggregation,
)
# Apply any final text preparation (e.g., trailing space)
@@ -716,7 +899,8 @@ class TTSService(AIService):
await self.process_generator(self.run_tts(prepared_text, context_id))
await self.stop_processing_metrics()
if not self._is_streaming_tokens:
await self.stop_processing_metrics()
if self._push_text_frames:
# In TTS services that support word timestamps, the TTSTextFrames
@@ -733,6 +917,8 @@ class TTSService(AIService):
if append_tts_text_to_context is not None:
frame.append_to_context = append_tts_text_to_context
await self.push_frame(frame)
if push_assistant_aggregation:
await self.push_frame(LLMAssistantPushAggregationFrame())
async def _stop_frame_handler(self):
has_started = False
@@ -750,25 +936,9 @@ class TTSService(AIService):
await self.push_frame(TTSStoppedFrame())
has_started = False
class WordTTSService(TTSService):
"""Base class for TTS services that support word timestamps.
Word timestamps are useful to synchronize audio with text of the spoken
words. This way only the spoken words are added to the conversation context.
"""
def __init__(self, **kwargs):
"""Initialize the Word TTS service.
Args:
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(**kwargs)
self._initial_word_timestamp = -1
self._initial_word_times = []
self._words_task = None
self._llm_response_started: bool = False
#
# Word timestamp methods (active when supports_word_timestamps=True)
#
async def start_word_timestamps(self):
"""Start tracking word timestamps from the current time."""
@@ -803,55 +973,9 @@ class WordTTSService(TTSService):
else:
await self._add_word_timestamps(word_times_with_context)
async def start(self, frame: StartFrame):
"""Start the word TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame)
self._create_words_task()
async def stop(self, frame: EndFrame):
"""Stop the word TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame)
await self._stop_words_task()
async def cancel(self, frame: CancelFrame):
"""Cancel the word TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame)
await self._stop_words_task()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with word timestamp awareness.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction)
if isinstance(frame, LLMFullResponseStartFrame):
self._llm_response_started = True
elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
await self.flush_audio()
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
self._llm_response_started = False
await self.reset_word_timestamps()
def _create_words_task(self):
if not self._words_task:
self._words_queue = asyncio.Queue()
self._words_queue: asyncio.Queue = asyncio.Queue()
self._words_task = self.create_task(self._words_task_handler())
async def _stop_words_task(self):
@@ -878,6 +1002,9 @@ class WordTTSService(TTSService):
frame = TTSStoppedFrame()
frame.pts = last_pts
frame.context_id = context_id
if context_id in self._tts_contexts:
if self._tts_contexts[context_id].push_assistant_aggregation:
await self.push_frame(LLMAssistantPushAggregationFrame())
else:
# Assumption: word-by-word text frames don't include spaces, so
# we can rely on the default includes_inter_frame_spaces=False
@@ -893,6 +1020,23 @@ class WordTTSService(TTSService):
self._words_queue.task_done()
class WordTTSService(TTSService):
"""Deprecated. Use TTSService with supports_word_timestamps=True instead.
.. deprecated:: 0.0.104
Word timestamp functionality has been moved to TTSService. Pass
``supports_word_timestamps=True`` to TTSService (or any subclass) instead.
"""
def __init__(self, **kwargs):
"""Initialize the Word TTS service.
Args:
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(supports_word_timestamps=True, **kwargs)
class WebsocketTTSService(TTSService, WebsocketService):
"""Base class for websocket-based TTS services.
@@ -965,10 +1109,12 @@ class InterruptibleTTSService(WebsocketTTSService):
self._bot_speaking = False
class WebsocketWordTTSService(WordTTSService, WebsocketService):
"""Base class for websocket-based TTS services that support word timestamps.
class WebsocketWordTTSService(WebsocketTTSService):
"""Deprecated. Use WebsocketTTSService with supports_word_timestamps=True instead.
Combines word timestamp functionality with websocket connectivity.
.. deprecated:: 0.0.104
Word timestamp functionality has been moved to TTSService. Pass
``supports_word_timestamps=True`` to WebsocketTTSService instead.
"""
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
@@ -978,53 +1124,26 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService):
reconnect_on_error: Whether to automatically reconnect on websocket errors.
**kwargs: Additional arguments passed to parent classes.
"""
WordTTSService.__init__(self, **kwargs)
WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
async def _report_error(self, error: ErrorFrame):
await self._call_event_handler("on_connection_error", error.error)
await self.push_error_frame(error)
super().__init__(
supports_word_timestamps=True, reconnect_on_error=reconnect_on_error, **kwargs
)
class InterruptibleWordTTSService(WebsocketWordTTSService):
"""Websocket-based TTS service with word timestamps that handles interruptions.
class InterruptibleWordTTSService(InterruptibleTTSService):
"""Deprecated. Use InterruptibleTTSService with supports_word_timestamps=True instead.
For TTS services that support word timestamps but can't correlate generated
audio with requested text. Handles interruptions by reconnecting when needed.
.. deprecated:: 0.0.104
Word timestamp functionality has been moved to TTSService. Pass
``supports_word_timestamps=True`` to InterruptibleTTSService instead.
"""
def __init__(self, **kwargs):
"""Initialize the Interruptible Word TTS service.
Args:
**kwargs: Additional arguments passed to the parent WebsocketWordTTSService.
**kwargs: Additional arguments passed to the parent InterruptibleTTSService.
"""
super().__init__(**kwargs)
# Indicates if the bot is speaking. If the bot is not speaking we don't
# need to reconnect when the user speaks. If the bot is speaking and the
# user interrupts we need to reconnect.
self._bot_speaking = False
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
if self._bot_speaking:
await self._disconnect()
await self._connect()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with bot speaking state tracking.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction)
if isinstance(frame, BotStartedSpeakingFrame):
self._bot_speaking = True
elif isinstance(frame, BotStoppedSpeakingFrame):
self._bot_speaking = False
super().__init__(supports_word_timestamps=True, **kwargs)
class AudioContextTTSService(WebsocketTTSService):
@@ -1042,14 +1161,25 @@ class AudioContextTTSService(WebsocketTTSService):
audio from context ID "A" will be played first.
"""
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
_CONTEXT_KEEPALIVE = object()
def __init__(
self,
*,
reuse_context_id_within_turn: bool = True,
reconnect_on_error: bool = True,
**kwargs,
):
"""Initialize the Audio Context TTS service.
Args:
reuse_context_id_within_turn: Whether the service should reuse context IDs within the same turn.
reconnect_on_error: Whether to automatically reconnect on websocket errors.
**kwargs: Additional arguments passed to the parent WebsocketTTSService.
"""
super().__init__(reconnect_on_error=reconnect_on_error, **kwargs)
self._reuse_context_id_within_turn = reuse_context_id_within_turn
self._context_id = None
self._contexts: Dict[str, asyncio.Queue] = {}
self._audio_context_task = None
@@ -1059,6 +1189,10 @@ class AudioContextTTSService(WebsocketTTSService):
Args:
context_id: Unique identifier for the audio context.
"""
# Set the context ID if not already set
if not self._context_id:
self._context_id = context_id
await self._contexts_queue.put(context_id)
self._contexts[context_id] = asyncio.Queue()
logger.trace(f"{self} created audio context {context_id}")
@@ -1091,6 +1225,32 @@ class AudioContextTTSService(WebsocketTTSService):
else:
logger.warning(f"{self} unable to remove context {context_id}")
def has_active_audio_context(self) -> bool:
"""Check if there is an active audio context.
Returns:
True if an active audio context exists, False otherwise.
"""
return self._context_id is not None and self.audio_context_available(self._context_id)
def get_active_audio_context_id(self) -> Optional[str]:
"""Get the active audio context ID.
Returns:
The active context ID, or None if no context is active.
"""
return self._context_id
async def remove_active_audio_context(self):
"""Remove the active audio context."""
if self._context_id:
await self.remove_audio_context(self._context_id)
self.reset_active_audio_context()
def reset_active_audio_context(self):
"""Reset the active audio context."""
self._context_id = None
def audio_context_available(self, context_id: str) -> bool:
"""Check whether the given audio context is registered.
@@ -1102,6 +1262,26 @@ class AudioContextTTSService(WebsocketTTSService):
"""
return context_id in self._contexts
def create_context_id(self) -> str:
"""Generate or reuse a context ID based on concurrent TTS support.
If _reuse_context_id_within_turn is False and a context already exists,
the existing context ID is returned. Otherwise, a new unique context
ID is generated.
Returns:
A context ID string for the TTS request.
"""
if self._reuse_context_id_within_turn and self._context_id:
self._refresh_active_audio_context()
return self._context_id
return super().create_context_id()
def _refresh_active_audio_context(self):
"""Signal that the audio context is still in use, resetting the timeout."""
if self.has_active_audio_context():
self._contexts[self._context_id].put_nowait(AudioContextTTSService._CONTEXT_KEEPALIVE)
async def start(self, frame: StartFrame):
"""Start the audio context TTS service.
@@ -1137,6 +1317,8 @@ class AudioContextTTSService(WebsocketTTSService):
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
await self._stop_audio_context_task()
await self.on_audio_context_interrupted(context_id=self._context_id)
self.reset_active_audio_context()
self._create_audio_context_task()
def _create_audio_context_task(self):
@@ -1155,6 +1337,7 @@ class AudioContextTTSService(WebsocketTTSService):
running = True
while running:
context_id = await self._contexts_queue.get()
self._context_id = context_id
if context_id:
# Process the audio context until the context doesn't have more
@@ -1163,11 +1346,16 @@ class AudioContextTTSService(WebsocketTTSService):
# We just finished processing the context, so we can safely remove it.
del self._contexts[context_id]
await self.on_audio_context_completed(context_id=context_id)
self.reset_active_audio_context()
# Append some silence between sentences.
silence = b"\x00" * self.sample_rate
frame = TTSAudioRawFrame(
audio=silence, sample_rate=self.sample_rate, num_channels=1
audio=silence,
sample_rate=self.sample_rate,
num_channels=1,
context_id=context_id,
)
await self.push_frame(frame)
else:
@@ -1183,6 +1371,10 @@ class AudioContextTTSService(WebsocketTTSService):
while running:
try:
frame = await asyncio.wait_for(queue.get(), timeout=AUDIO_CONTEXT_TIMEOUT)
if frame is AudioContextTTSService._CONTEXT_KEEPALIVE:
# Context is still in use, reset the timeout.
continue
if frame:
await self.push_frame(frame)
running = frame is not None
@@ -1191,16 +1383,42 @@ class AudioContextTTSService(WebsocketTTSService):
logger.trace(f"{self} time out on audio context {context_id}")
break
async def on_audio_context_interrupted(self, context_id: str):
"""Called when an audio context is cancelled due to an interruption.
class AudioContextWordTTSService(AudioContextTTSService, WebsocketWordTTSService):
"""Websocket-based TTS service with word timestamps and audio context management.
Override this in a subclass to perform provider-specific cleanup (e.g.
sending a cancel/close message over the WebSocket) when the bot is
interrupted mid-speech. The audio context task has already been stopped
and the active context has **not** yet been reset when this is called,
so ``context_id`` reflects the context that was cut short.
This is a base class for websocket-based TTS services that support word
timestamps and also allow correlating the generated audio with the requested
text through audio contexts.
Args:
context_id: The ID of the audio context that was interrupted, or
``None`` if no context was active at the time.
"""
pass
Combines the audio context management capabilities of AudioContextTTSService
with the word timestamp functionality of WebsocketWordTTSService.
async def on_audio_context_completed(self, context_id: str):
"""Called after an audio context has finished playing all of its audio.
Override this in a subclass to perform provider-specific cleanup (e.g.
sending a close-context message to free server-side resources) once an
audio context has been fully processed. The context entry has already
been removed from the internal context map, and the active context has
**not** yet been reset when this is called.
Args:
context_id: The ID of the audio context that finished processing.
"""
pass
class AudioContextWordTTSService(AudioContextTTSService):
"""Deprecated. Use AudioContextTTSService with supports_word_timestamps=True instead.
.. deprecated:: 0.0.104
Word timestamp functionality has been moved to TTSService. Pass
``supports_word_timestamps=True`` to AudioContextTTSService instead.
"""
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
@@ -1210,5 +1428,6 @@ class AudioContextWordTTSService(AudioContextTTSService, WebsocketWordTTSService
reconnect_on_error: Whether to automatically reconnect on websocket errors.
**kwargs: Additional arguments passed to parent classes.
"""
AudioContextTTSService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
WebsocketWordTTSService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
super().__init__(
supports_word_timestamps=True, reconnect_on_error=reconnect_on_error, **kwargs
)

145
src/pipecat/services/ultravox/llm.py
View File

@@ -15,6 +15,7 @@ import asyncio
import datetime
import json
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, Literal, Optional, Union
import aiohttp
@@ -30,11 +31,11 @@ from pipecat.frames.frames import (
Frame,
InputAudioRawFrame,
InputTextRawFrame,
InterruptionFrame,
LLMContextFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
TranscriptionFrame,
TTSAudioRawFrame,
@@ -42,7 +43,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
TTSTextFrame,
UserAudioRawFrame,
UserStoppedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response import (
@@ -56,6 +57,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
from pipecat.utils.time import time_now_iso8601
try:
@@ -66,6 +68,17 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
@dataclass
class UltravoxRealtimeLLMSettings(LLMSettings):
"""Settings for UltravoxRealtimeLLMService.
Parameters:
output_medium: The output medium for the model ("voice" or "text").
"""
output_medium: str | _NotGiven = field(default=NOT_GIVEN)
class AgentInputParams(BaseModel):
"""Input parameters for Ultravox Realtime generation using a pre-defined Agent.
@@ -78,6 +91,9 @@ class AgentInputParams(BaseModel):
template_context: Context variables to use when instantiating a call with the
agent. Defaults to an empty dict.
metadata: Metadata to attach to the call. Default to an empty dict.
output_medium: The initial output medium for the agent. Use "text" for text
responses or "voice" for audio responses. Defaults to None, which uses the
agent's default.
max_duration: The maximum duration of the call. Defaults to None, which will
use the agent's default maximum duration.
extra: Extra parameters to include in the agent call creation request. Defaults
@@ -89,6 +105,7 @@ class AgentInputParams(BaseModel):
agent_id: uuid.UUID
template_context: Dict[str, Any] = Field(default_factory=dict)
metadata: Dict[str, str] = Field(default_factory=dict)
output_medium: Optional[Literal["text", "voice"]] = None
max_duration: Optional[datetime.timedelta] = Field(
default=None, ge=datetime.timedelta(seconds=10), le=datetime.timedelta(hours=1)
)
@@ -105,6 +122,8 @@ class OneShotInputParams(BaseModel):
model: Model identifier to use. Defaults to "fixie-ai/ultravox".
voice: Voice identifier for speech generation. Defaults to None.
metadata: Metadata to attach to the call. Default to an empty dict.
output_medium: The initial output medium for the agent. Use "text" for text
responses or "voice" for audio responses. Defaults to None (voice).
max_duration: The maximum duration of the call. Defaults to one hour.
extra: Extra parameters to include in the call creation request. Defaults
to an empty dict. See the Ultravox API documentation for valid arguments:
@@ -117,6 +136,7 @@ class OneShotInputParams(BaseModel):
model: Optional[str] = None
voice: Optional[uuid.UUID] = None
metadata: Dict[str, str] = Field(default_factory=dict)
output_medium: Optional[Literal["text", "voice"]] = None
max_duration: datetime.timedelta = Field(
default=datetime.timedelta(hours=1),
ge=datetime.timedelta(seconds=10),
@@ -146,6 +166,8 @@ class UltravoxRealtimeLLMService(LLMService):
by the model and may not always align with its understanding of user input.
"""
_settings: UltravoxRealtimeLLMSettings
def __init__(
self,
*,
@@ -162,7 +184,22 @@ class UltravoxRealtimeLLMService(LLMService):
May only be set with OneShotInputParams.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
super().__init__(
settings=UltravoxRealtimeLLMSettings(
model=None,
temperature=None,
max_tokens=None,
top_p=None,
top_k=None,
frequency_penalty=None,
presence_penalty=None,
seed=None,
filter_incomplete_user_turns=False,
user_turn_completion_config=None,
output_medium=None,
),
**kwargs,
)
self._params = params
if one_shot_selected_tools:
if not isinstance(self._params, OneShotInputParams):
@@ -181,6 +218,14 @@ class UltravoxRealtimeLLMService(LLMService):
self._sample_rate = 48000
self._resampler = create_stream_resampler()
def can_generate_metrics(self) -> bool:
"""Check if the service can generate usage metrics.
Returns:
True if metrics generation is supported.
"""
return True
#
# standard AIService frame handling
#
@@ -208,6 +253,14 @@ class UltravoxRealtimeLLMService(LLMService):
except Exception as e:
await self.push_error("Failed to connect to Ultravox", e, fatal=True)
@staticmethod
def _output_medium_to_api(medium: Optional[Literal["text", "voice"]]) -> Optional[str]:
if medium == "text":
return "MESSAGE_MEDIUM_TEXT"
elif medium == "voice":
return "MESSAGE_MEDIUM_VOICE"
return None
async def _start_agent_call(self, params: AgentInputParams) -> str:
request_body = {
"templateContext": params.template_context,
@@ -218,6 +271,9 @@ class UltravoxRealtimeLLMService(LLMService):
}
},
}
initial_output_medium = self._output_medium_to_api(params.output_medium)
if initial_output_medium:
request_body["initialOutputMedium"] = initial_output_medium
if params.max_duration:
request_body["maxDuration"] = f"{params.max_duration.total_seconds():3f}s"
request_body = request_body | params.extra
@@ -248,7 +304,11 @@ class UltravoxRealtimeLLMService(LLMService):
"inputSampleRate": self._sample_rate,
}
},
} | params.extra
}
initial_output_medium = self._output_medium_to_api(params.output_medium)
if initial_output_medium:
request_body["initialOutputMedium"] = initial_output_medium
request_body = request_body | params.extra
async with aiohttp.ClientSession() as session:
async with session.post(
"https://api.ultravox.ai/api/calls",
@@ -310,6 +370,13 @@ class UltravoxRealtimeLLMService(LLMService):
await self.cancel_task(self._receive_task, timeout=1.0)
self._receive_task = None
async def _update_settings(self, delta: UltravoxRealtimeLLMSettings):
changed = await super()._update_settings(delta)
if "output_medium" in changed:
await self._update_output_medium(self._settings.output_medium)
self._warn_unhandled_updated_settings(changed.keys() - {"output_medium"})
return changed
#
# frame processing
# StartFrame, StopFrame, CancelFrame implemented in base class
@@ -331,21 +398,17 @@ class UltravoxRealtimeLLMService(LLMService):
else LLMContext.from_openai_context(frame.context)
)
await self._handle_context(context)
elif isinstance(frame, LLMUpdateSettingsFrame):
if "output_medium" in frame.settings:
await self._update_output_medium(frame.settings.get("output_medium"))
elif isinstance(frame, InterruptionFrame):
await self.stop_all_metrics()
await self.push_frame(frame, direction)
elif isinstance(frame, InputTextRawFrame):
await self._send_user_text(frame.text)
await self.push_frame(frame, direction)
elif isinstance(frame, InputAudioRawFrame):
await self._send_user_audio(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, UserStoppedSpeakingFrame):
# This may or may not align with Ultravox's end of user speech detection,
# which relies on a more complex endpointing model. In particular it will
# yield a seemingly very slow TTFB in the case of endpointing false
# negatives. It will be close in the majority of cases though.
await self.start_ttfb_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame):
await self._handle_vad_user_stopped_speaking(frame)
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
@@ -366,6 +429,25 @@ class UltravoxRealtimeLLMService(LLMService):
}
await self._send(socket_message)
async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame):
"""Handle VAD user stopped speaking frame.
Calculates the actual speech end time and starts a timeout task to wait
for the final transcription before reporting TTFB.
Args:
frame: The VAD user stopped speaking frame.
"""
# Skip TTFB measurement if stop_secs is not set
if frame.stop_secs == 0.0:
return
# Calculate the actual speech end time (current time minus VAD stop delay).
# This approximates when the last user audio was sent to the Ultravox service,
# which we use to measure against the eventual transcription response.
speech_end_time = frame.timestamp - frame.stop_secs
await self.start_ttfb_metrics(start_time=speech_end_time)
async def _send_user_audio(self, frame: InputAudioRawFrame):
"""Send user audio frame to Ultravox Realtime."""
if not self._socket:
@@ -469,6 +551,7 @@ class UltravoxRealtimeLLMService(LLMService):
if not audio:
return
if not self._bot_responding:
await self.start_processing_metrics()
await self.stop_ttfb_metrics()
await self.push_frame(LLMFullResponseStartFrame())
await self.push_frame(TTSStartedFrame())
@@ -476,6 +559,7 @@ class UltravoxRealtimeLLMService(LLMService):
await self.push_frame(TTSAudioRawFrame(audio, self._sample_rate, 1))
async def _handle_response_end(self):
await self.stop_processing_metrics()
if self._bot_responding == "voice":
await self.push_frame(TTSStoppedFrame())
await self.push_frame(LLMFullResponseEndFrame())
@@ -509,22 +593,29 @@ class UltravoxRealtimeLLMService(LLMService):
async def _handle_agent_transcript(
self, medium: str, text: Optional[str], delta: Optional[str], final: bool
):
if text or delta:
frame = LLMTextFrame(text=text or delta)
frame.skip_tts = medium == "voice"
await self.push_frame(frame)
if medium == "text":
if text:
await self.stop_ttfb_metrics()
await self.push_frame(LLMFullResponseStartFrame())
await self.push_frame(TTSStartedFrame())
await self.push_frame(TTSTextFrame(text=text, aggregated_by=AggregationType.WORD))
self._bot_responding = "text"
elif final:
if medium == "voice":
# In voice mode, audio is handled by _handle_audio(). Here we push
# text transcripts of the audio for downstream consumers.
if (text or delta) and not final:
frame = LLMTextFrame(text=text or delta)
frame.append_to_context = False
await self.push_frame(frame)
if delta:
tts_frame = TTSTextFrame(text=delta, aggregated_by=AggregationType.WORD)
tts_frame.includes_inter_frame_spaces = True
await self.push_frame(tts_frame)
elif medium == "text":
if final:
await self.stop_processing_metrics()
await self.push_frame(LLMFullResponseEndFrame())
self._bot_responding = None
elif delta:
await self.push_frame(TTSTextFrame(text=delta, aggregated_by=AggregationType.WORD))
elif text or delta:
if not self._bot_responding:
await self.start_processing_metrics()
await self.stop_ttfb_metrics()
await self.push_frame(LLMFullResponseStartFrame())
self._bot_responding = "text"
await self.push_frame(LLMTextFrame(text=text or delta))
def create_context_aggregator(
self,

14
src/pipecat/services/vision_service.py
View File

@@ -12,11 +12,12 @@ visual content.
"""
from abc import abstractmethod
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from pipecat.frames.frames import Frame, UserImageRawFrame
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.settings import VisionSettings
class VisionService(AIService):
@@ -27,13 +28,20 @@ class VisionService(AIService):
with the AI service infrastructure for metrics and lifecycle management.
"""
def __init__(self, **kwargs):
def __init__(self, *, settings: Optional[VisionSettings] = None, **kwargs):
"""Initialize the vision service.
Args:
settings: The runtime-updatable settings for the vision service.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs)
super().__init__(
settings=settings
# Here in case subclass doesn't implement more specific settings
# (which hopefully should be rare)
or VisionSettings(),
**kwargs,
)
self._describe_text = None
@abstractmethod

74
src/pipecat/services/whisper/base_stt.py
View File

@@ -10,13 +10,15 @@ This module provides common functionality for services implementing the Whisper
interface, including language mapping, metrics generation, and error handling.
"""
from typing import AsyncGenerator, Optional
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
from openai import AsyncOpenAI
from openai.types.audio import Transcription
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_latency import WHISPER_TTFS_P99
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language, resolve_language
@@ -24,6 +26,22 @@ from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
@dataclass
class BaseWhisperSTTSettings(STTSettings):
"""Settings for Whisper API-based STT services.
Parameters:
base_url: API base URL.
prompt: Optional text to guide the model's style or continue
a previous segment.
temperature: Sampling temperature between 0 and 1.
"""
base_url: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
prompt: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
def language_to_whisper_language(language: Language) -> Optional[str]:
"""Maps pipecat Language enum to Whisper API language codes.
@@ -106,6 +124,8 @@ class BaseWhisperSTTService(SegmentedSTTService):
including metrics generation and error handling.
"""
_settings: BaseWhisperSTTSettings
def __init__(
self,
*,
@@ -135,34 +155,45 @@ class BaseWhisperSTTService(SegmentedSTTService):
Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
super().__init__(ttfs_p99_latency=ttfs_p99_latency, **kwargs)
self.set_model_name(model)
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=BaseWhisperSTTSettings(
model=model,
language=self.language_to_service_language(language or Language.EN),
base_url=base_url,
prompt=prompt,
temperature=temperature,
),
**kwargs,
)
self._client = self._create_client(api_key, base_url)
self._language = self.language_to_service_language(language or Language.EN)
self._language = self._settings.language
self._prompt = prompt
self._temperature = temperature
self._include_prob_metrics = include_prob_metrics
self._settings = {
"base_url": base_url,
"language": self._language,
"prompt": self._prompt,
"temperature": self._temperature,
}
def _create_client(self, api_key: Optional[str], base_url: Optional[str]):
return AsyncOpenAI(api_key=api_key, base_url=base_url)
async def set_model(self, model: str):
"""Set the model name for transcription.
async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
"""Apply a settings delta, syncing instance variables.
Args:
model: The name of the model to use.
Keeps ``_language``, ``_prompt``, and ``_temperature`` in sync with
the settings fields.
"""
self.set_model_name(model)
changed = await super()._update_settings(delta)
if "language" in changed:
self._language = self._settings.language
if "prompt" in changed:
self._prompt = self._settings.prompt
if "temperature" in changed:
self._temperature = self._settings.temperature
return changed
def can_generate_metrics(self) -> bool:
"""Indicates whether this service can generate metrics.
"""Whether this service can generate processing metrics.
Returns:
bool: True, as this service supports metric generation.
@@ -180,15 +211,6 @@ class BaseWhisperSTTService(SegmentedSTTService):
"""
return language_to_whisper_language(language)
async def set_language(self, language: Language):
"""Set the language for transcription.
Args:
language: The Language enum value to use for transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._language = self.language_to_service_language(language)
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None

102
src/pipecat/services/whisper/stt.py
View File

@@ -11,6 +11,7 @@ supporting both Faster Whisper and MLX Whisper backends for efficient inference.
"""
import asyncio
from dataclasses import dataclass, field
from enum import Enum
from typing import AsyncGenerator, Optional
@@ -19,6 +20,7 @@ from loguru import logger
from typing_extensions import TYPE_CHECKING, override
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.time import time_now_iso8601
@@ -172,6 +174,36 @@ def language_to_whisper_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class WhisperSTTSettings(STTSettings):
"""Settings for the local Whisper (Faster Whisper) STT service.
Parameters:
device: Inference device ('cpu', 'cuda', or 'auto').
compute_type: Compute type for inference ('default', 'int8', etc.).
no_speech_prob: Probability threshold for filtering non-speech segments.
"""
device: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
compute_type: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
no_speech_prob: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
@dataclass
class WhisperMLXSTTSettings(STTSettings):
"""Settings for the MLX Whisper STT service.
Parameters:
no_speech_prob: Probability threshold for filtering non-speech segments.
temperature: Sampling temperature (0.0-1.0).
engine: Whisper engine identifier.
"""
no_speech_prob: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class WhisperSTTService(SegmentedSTTService):
"""Class to transcribe audio with a locally-downloaded Whisper model.
@@ -179,6 +211,8 @@ class WhisperSTTService(SegmentedSTTService):
segments. It supports multiple languages and various model sizes.
"""
_settings: WhisperSTTSettings
def __init__(
self,
*,
@@ -199,20 +233,21 @@ class WhisperSTTService(SegmentedSTTService):
language: The default language for transcription.
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
super().__init__(**kwargs)
super().__init__(
settings=WhisperSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
device=device,
compute_type=compute_type,
no_speech_prob=no_speech_prob,
),
**kwargs,
)
self._device: str = device
self._compute_type = compute_type
self.set_model_name(model if isinstance(model, str) else model.value)
self._no_speech_prob = no_speech_prob
self._model: Optional[WhisperModel] = None
self._settings = {
"language": language,
"device": self._device,
"compute_type": self._compute_type,
"no_speech_prob": self._no_speech_prob,
}
self._load()
def can_generate_metrics(self) -> bool:
@@ -234,15 +269,6 @@ class WhisperSTTService(SegmentedSTTService):
"""
return language_to_whisper_language(language)
async def set_language(self, language: Language):
"""Set the language for transcription.
Args:
language: The Language enum value to use for transcription.
"""
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
def _load(self):
"""Loads the Whisper model.
@@ -255,7 +281,7 @@ class WhisperSTTService(SegmentedSTTService):
logger.debug("Loading Whisper model...")
self._model = WhisperModel(
self.model_name, device=self._device, compute_type=self._compute_type
self._settings.model, device=self._device, compute_type=self._compute_type
)
logger.debug("Loaded Whisper model")
except ModuleNotFoundError as e:
@@ -293,9 +319,8 @@ class WhisperSTTService(SegmentedSTTService):
# Divide by 32768 because we have signed 16-bit data.
audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0
whisper_lang = self.language_to_service_language(self._settings["language"])
segments, _ = await asyncio.to_thread(
self._model.transcribe, audio_float, language=whisper_lang
self._model.transcribe, audio_float, language=self._settings.language
)
text: str = ""
for segment in segments:
@@ -305,13 +330,13 @@ class WhisperSTTService(SegmentedSTTService):
await self.stop_processing_metrics()
if text:
await self._handle_transcription(text, True, self._settings["language"])
await self._handle_transcription(text, True, self._settings.language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
)
@@ -322,6 +347,8 @@ class WhisperSTTServiceMLX(WhisperSTTService):
segments. It's optimized for Apple Silicon and supports multiple languages and quantizations.
"""
_settings: WhisperMLXSTTSettings
def __init__(
self,
*,
@@ -341,19 +368,21 @@ class WhisperSTTServiceMLX(WhisperSTTService):
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
# Skip WhisperSTTService.__init__ and call its parent directly
SegmentedSTTService.__init__(self, **kwargs)
SegmentedSTTService.__init__(
self,
settings=WhisperMLXSTTSettings(
model=model if isinstance(model, str) else model.value,
language=language,
no_speech_prob=no_speech_prob,
temperature=temperature,
engine="mlx",
),
**kwargs,
)
self.set_model_name(model if isinstance(model, str) else model.value)
self._no_speech_prob = no_speech_prob
self._temperature = temperature
self._settings = {
"language": language,
"no_speech_prob": self._no_speech_prob,
"temperature": self._temperature,
"engine": "mlx",
}
# No need to call _load() as MLX Whisper loads models on demand
@override
@@ -390,13 +419,12 @@ class WhisperSTTServiceMLX(WhisperSTTService):
# Divide by 32768 because we have signed 16-bit data.
audio_float = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0
whisper_lang = self.language_to_service_language(self._settings["language"])
chunk = await asyncio.to_thread(
mlx_whisper.transcribe,
audio_float,
path_or_hf_repo=self.model_name,
path_or_hf_repo=self._settings.model,
temperature=self._temperature,
language=whisper_lang,
language=self._settings.language,
)
text: str = ""
for segment in chunk.get("segments", []):
@@ -413,13 +441,13 @@ class WhisperSTTServiceMLX(WhisperSTTService):
await self.stop_processing_metrics()
if text:
await self._handle_transcription(text, True, self._settings["language"])
await self._handle_transcription(text, True, self._settings.language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text,
self._user_id,
time_now_iso8601(),
self._settings["language"],
self._settings.language,
)
except Exception as e:

42
src/pipecat/services/xtts/tts.py
View File

@@ -10,7 +10,8 @@ This module provides integration with Coqui XTTS streaming server for
text-to-speech synthesis using local Docker deployment.
"""
from typing import Any, AsyncGenerator, Dict, Optional
from dataclasses import dataclass, field
from typing import AsyncGenerator, Dict, Optional
import aiohttp
from loguru import logger
@@ -24,6 +25,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language, resolve_language
from pipecat.utils.tracing.service_decorators import traced_tts
@@ -68,6 +70,17 @@ def language_to_xtts_language(language: Language) -> Optional[str]:
return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
@dataclass
class XTTSTTSSettings(TTSSettings):
"""Settings for XTTS TTS service.
Parameters:
base_url: Base URL of the XTTS streaming server.
"""
base_url: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class XTTSService(TTSService):
"""Coqui XTTS text-to-speech service.
@@ -76,6 +89,8 @@ class XTTSService(TTSService):
studio speakers configuration.
"""
_settings: XTTSTTSSettings
def __init__(
self,
*,
@@ -96,13 +111,16 @@ class XTTSService(TTSService):
sample_rate: Audio sample rate. If None, uses default.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"language": self.language_to_service_language(language),
"base_url": base_url,
}
self.set_voice(voice_id)
super().__init__(
sample_rate=sample_rate,
settings=XTTSTTSSettings(
model=None,
voice=voice_id,
language=self.language_to_service_language(language),
base_url=base_url,
),
**kwargs,
)
self._studio_speakers: Optional[Dict[str, Any]] = None
self._aiohttp_session = aiohttp_session
@@ -138,7 +156,7 @@ class XTTSService(TTSService):
if self._studio_speakers:
return
async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r:
async with self._aiohttp_session.get(self._settings.base_url + "/studio_speakers") as r:
if r.status != 200:
text = await r.text()
await self.push_error(
@@ -164,13 +182,13 @@ class XTTSService(TTSService):
logger.error(f"{self} no studio speakers available")
return
embeddings = self._studio_speakers[self._voice_id]
embeddings = self._studio_speakers[self._settings.voice]
url = self._settings["base_url"] + "/tts_stream"
url = self._settings.base_url + "/tts_stream"
payload = {
"text": text.replace(".", "").replace("*", ""),
"language": self._settings["language"],
"language": self._settings.language,
"speaker_embedding": embeddings["speaker_embedding"],
"gpt_cond_latent": embeddings["gpt_cond_latent"],
"add_wav_header": False,

5
src/pipecat/transports/base_input.py
View File

@@ -424,6 +424,11 @@ class BaseInputTransport(FrameProcessor):
if self._params.audio_in_filter:
frame.audio = await self._params.audio_in_filter.filter(frame.audio)
# Skip frames with no audio data (e.g. filter is buffering).
if not frame.audio:
self._audio_in_queue.task_done()
continue
###################################################################
# DEPRECATED.
#

24
src/pipecat/transports/base_output.py
View File

@@ -237,6 +237,18 @@ class BaseOutputTransport(FrameProcessor):
else:
await self._write_dtmf_audio(frame)
async def write_transport_frame(self, frame: Frame):
"""Handle a queued frame after preceding audio has been sent.
Override in transport subclasses to handle custom frame types that
flow through the audio queue. Called by the media sender after the
frame has waited for any preceding audio to finish.
Args:
frame: The frame to handle.
"""
pass
def _supports_native_dtmf(self) -> bool:
"""Override in transport implementations that support native DTMF.
@@ -613,6 +625,11 @@ class BaseOutputTransport(FrameProcessor):
downstream_frame.transport_destination = self._destination
upstream_frame = BotStartedSpeakingFrame()
upstream_frame.transport_destination = self._destination
# Setting the siblings id
upstream_frame.broadcast_sibling_id = downstream_frame.id
downstream_frame.broadcast_sibling_id = upstream_frame.id
await self._transport.push_frame(downstream_frame)
await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM)
@@ -635,6 +652,11 @@ class BaseOutputTransport(FrameProcessor):
downstream_frame.transport_destination = self._destination
upstream_frame = BotStoppedSpeakingFrame()
upstream_frame.transport_destination = self._destination
# Setting the siblings id
upstream_frame.broadcast_sibling_id = downstream_frame.id
downstream_frame.broadcast_sibling_id = upstream_frame.id
await self._transport.push_frame(downstream_frame)
await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM)
@@ -681,6 +703,8 @@ class BaseOutputTransport(FrameProcessor):
await self._transport.send_message(frame)
elif isinstance(frame, OutputDTMFFrame):
await self._transport.write_dtmf(frame)
else:
await self._transport.write_transport_frame(frame)
def _next_frame(self) -> AsyncGenerator[Frame, None]:
"""Generate the next frame for audio processing.

103
src/pipecat/transports/daily/transport.py
View File

@@ -15,7 +15,7 @@ import asyncio
import time
from concurrent.futures import CancelledError as FuturesCancelledError
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Any, Awaitable, Callable, Dict, Mapping, Optional, Tuple
import aiohttp
@@ -25,7 +25,7 @@ from pydantic import BaseModel
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
from pipecat.frames.frames import (
CancelFrame,
ControlFrame,
DataFrame,
EndFrame,
Frame,
InputAudioRawFrame,
@@ -183,34 +183,44 @@ class DailyInputTransportMessageUrgentFrame(DailyInputTransportMessageFrame):
@dataclass
class DailyUpdateRemoteParticipantsFrame(ControlFrame):
"""Frame to update remote participants in Daily calls.
class DailySIPTransferFrame(DataFrame):
"""SIP call transfer frame for transport queuing.
.. deprecated:: 0.0.87
`DailyUpdateRemoteParticipantsFrame` is deprecated and will be removed in a future version.
Create your own custom frame and use a custom processor to handle it or use, for example,
`on_after_push_frame` event instead in the output transport.
A SIP call transfer that will be queued. The transfer will happen after any
preceding audio finishes playing, allowing the bot to complete its current
utterance before the transfer occurs.
Parameters:
settings: SIP call transfer settings.
"""
settings: Mapping[str, Any] = field(default_factory=dict)
@dataclass
class DailySIPReferFrame(DataFrame):
"""SIP REFER frame for transport queuing.
A SIP REFER that will be queued. The REFER will happen after any preceding
audio finishes playing, allowing the bot to complete its current utterance
before the REFER occurs.
Parameters:
settings: SIP REFER settings.
"""
settings: Mapping[str, Any] = field(default_factory=dict)
@dataclass
class DailyUpdateRemoteParticipantsFrame(DataFrame):
"""Frame to update remote participants in Daily calls.
Parameters:
remote_participants: See https://reference-python.daily.co/api_reference.html#daily.CallClient.update_remote_participants.
"""
remote_participants: Mapping[str, Any] = None
def __post_init__(self):
super().__post_init__()
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"DailyUpdateRemoteParticipantsFrame is deprecated and will be removed in a future version."
"Instead, create your own custom frame and handle it in the "
'`@transport.output().event_handler("on_after_push_frame")` event handler or a '
"custom processor.",
DeprecationWarning,
stacklevel=2,
)
remote_participants: Mapping[str, Any] = field(default_factory=dict)
class WebRTCVADAnalyzer(VADAnalyzer):
@@ -501,6 +511,7 @@ class DailyTransportClient(EventHandler):
self._event_task = None
self._audio_task = None
self._video_task = None
self._join_message_queue: list = []
# Input and ouput sample rates. They will be initialize on setup().
self._in_sample_rate = 0
@@ -567,7 +578,8 @@ class DailyTransportClient(EventHandler):
error: An error description or None.
"""
if not self._joined:
return "Unable to send messages before joining."
self._join_message_queue.append(frame)
return None
participant_id = None
if isinstance(
@@ -768,6 +780,8 @@ class DailyTransportClient(EventHandler):
await self._callbacks.on_joined(data)
self._joined_event.set()
await self._flush_join_messages()
else:
error_msg = f"Error joining {self._room_url}: {error}"
logger.error(error_msg)
@@ -1541,6 +1555,12 @@ class DailyTransportClient(EventHandler):
await callback(*args)
queue.task_done()
async def _flush_join_messages(self):
"""Send any messages that were queued before join completed."""
for frame in self._join_message_queue:
await self.send_message(frame)
self._join_message_queue.clear()
def _get_event_loop(self) -> asyncio.AbstractEventLoop:
"""Get the event loop from the task manager."""
if not self._task_manager:
@@ -1946,18 +1966,6 @@ class DailyOutputTransport(BaseOutputTransport):
# Leave the room.
await self._client.leave()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process outgoing frames, including transport messages.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, DailyUpdateRemoteParticipantsFrame):
await self._client.update_remote_participants(frame.remote_participants)
async def send_message(
self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
):
@@ -1968,7 +1976,7 @@ class DailyOutputTransport(BaseOutputTransport):
"""
error = await self._client.send_message(frame)
if error:
logger.error(f"Unable to send message: {error}")
await self.push_error(f"Unable to send message: {error}")
async def register_video_destination(self, destination: str):
"""Register a video output destination.
@@ -2011,6 +2019,25 @@ class DailyOutputTransport(BaseOutputTransport):
"""
return await self._client.write_video_frame(frame)
async def write_transport_frame(self, frame: Frame):
"""Handle queued SIP frames after preceding audio has been sent.
Args:
frame: The frame to handle.
"""
if isinstance(frame, DailySIPTransferFrame):
error = await self._client.sip_call_transfer(frame.settings)
if error:
await self.push_error(f"Unable to transfer SIP call: {error}")
elif isinstance(frame, DailySIPReferFrame):
error = await self._client.sip_refer(frame.settings)
if error:
await self.push_error(f"Unable to perform SIP REFER: {error}")
elif isinstance(frame, DailyUpdateRemoteParticipantsFrame):
error = await self._client.update_remote_participants(frame.remote_participants)
if error:
await self.push_error(f"Unable to update remote participants: {error}")
def _supports_native_dtmf(self) -> bool:
"""Daily supports native DTMF via telephone events.

Some files were not shown because too many files have changed in this diff Show More