wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into m-ods/assemblyai-universal-streaming

Browse Source
This commit is contained in:
Martin Schweiger
2025-05-23 17:39:00 +08:00
committed by GitHub
parent 9b8800ac1d a809b710c5
commit 7f568e3e7e
94 changed files with 1592 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/pipecat/adapters/schemas/tools_schema.py
View File

@@ -5,7 +5,7 @@
#
from enum import Enum
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
from pipecat.adapters.schemas.function_schema import FunctionSchema
@@ -18,7 +18,7 @@ class ToolsSchema:
def __init__(
self,
standard_tools: List[FunctionSchema],
custom_tools: Dict[AdapterType, List[Dict[str, Any]]] = None,
custom_tools: Optional[Dict[AdapterType, List[Dict[str, Any]]]] = None,
) -> None:
"""
A schema for tools that includes both standardized function schemas

4
src/pipecat/audio/turn/smart_turn/base_smart_turn.py
View File

@@ -36,10 +36,10 @@ class SmartTurnTimeoutException(Exception):
class BaseSmartTurn(BaseTurnAnalyzer):
def __init__(
self, *, sample_rate: Optional[int] = None, params: SmartTurnParams = SmartTurnParams()
self, *, sample_rate: Optional[int] = None, params: Optional[SmartTurnParams] = None
):
super().__init__(sample_rate=sample_rate)
self._params = params
self._params = params or SmartTurnParams()
# Configuration
self._stop_ms = self._params.stop_secs * 1000 # silence threshold in ms
# Inference state

6
src/pipecat/audio/turn/smart_turn/http_smart_turn.py
View File

@@ -6,7 +6,7 @@
import asyncio
import io
from typing import Any, Dict
from typing import Any, Dict, Optional
import aiohttp
import numpy as np
@@ -21,12 +21,12 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
*,
url: str,
aiohttp_session: aiohttp.ClientSession,
headers: Dict[str, str] = {},
headers: Optional[Dict[str, str]] = None,
**kwargs,
):
super().__init__(**kwargs)
self._url = url
self._headers = headers
self._headers = headers or {}
self._aiohttp_session = aiohttp_session
def _serialize_array(self, audio_array: np.ndarray) -> bytes:

2
src/pipecat/audio/vad/silero.py
View File

@@ -105,7 +105,7 @@ class SileroOnnxModel:
class SileroVADAnalyzer(VADAnalyzer):
def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
super().__init__(sample_rate=sample_rate, params=params)
logger.debug("Loading Silero VAD model...")

4
src/pipecat/audio/vad/vad_analyzer.py
View File

@@ -34,10 +34,10 @@ class VADParams(BaseModel):
class VADAnalyzer(ABC):
def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams):
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._params = params
self._params = params or VADParams()
self._num_channels = 1
self._vad_buffer = b""

1
src/pipecat/frames/frames.py
View File

@@ -288,6 +288,7 @@ class TranscriptionMessage:
role: Literal["user", "assistant"]
content: str
user_id: Optional[str] = None
timestamp: Optional[str] = None

5
src/pipecat/observers/base_observer.py
View File

@@ -4,12 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from abc import ABC, abstractmethod
from abc import abstractmethod
from dataclasses import dataclass
from typing_extensions import TYPE_CHECKING
from pipecat.frames.frames import Frame
from pipecat.utils.base_object import BaseObject
if TYPE_CHECKING:
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -39,7 +40,7 @@ class FramePushed:
timestamp: int
class BaseObserver(ABC):
class BaseObserver(BaseObject):
"""This is the base class for pipeline frame observers. Observers can view
all the frames that go through the pipeline without the need to inject
processors in the pipeline. This can be useful, for example, to implement

3
src/pipecat/observers/loggers/debug_log_observer.py
View File

@@ -74,6 +74,7 @@ class DebugLogObserver(BaseObserver):
Union[Tuple[Type[Frame], ...], Dict[Type[Frame], Optional[Tuple[Type, FrameEndpoint]]]]
] = None,
exclude_fields: Optional[Set[str]] = None,
**kwargs,
):
"""Initialize the debug log observer.
@@ -87,6 +88,8 @@ class DebugLogObserver(BaseObserver):
exclude_fields: Set of field names to exclude from logging. If None, only binary
data fields are excluded.
"""
super().__init__(**kwargs)
# Process frame filters
self.frame_filters = {}

31
src/pipecat/observers/turn_tracking_observer.py
View File

@@ -30,7 +30,7 @@ class TurnTrackingObserver(BaseObserver):
"""
def __init__(self, max_frames=100, turn_end_timeout_secs=2.5, **kwargs):
super().__init__()
super().__init__(**kwargs)
self._turn_count = 0
self._is_turn_active = False
self._is_bot_speaking = False
@@ -154,32 +154,3 @@ class TurnTrackingObserver(BaseObserver):
status = "interrupted" if was_interrupted else "completed"
logger.trace(f"Turn {self._turn_count} {status} after {duration:.2f}s")
await self._call_event_handler("on_turn_ended", self._turn_count, duration, was_interrupted)
def _register_event_handler(self, event_name):
"""Register an event handler."""
if not hasattr(self, "_event_handlers"):
self._event_handlers = {}
if event_name not in self._event_handlers:
self._event_handlers[event_name] = []
async def _call_event_handler(self, event_name, *args, **kwargs):
"""Call registered event handlers."""
if not hasattr(self, "_event_handlers"):
return
if event_name in self._event_handlers:
for handler in self._event_handlers[event_name]:
await handler(self, *args, **kwargs)
def event_handler(self, event_name):
"""Decorator for registering event handlers."""
def decorator(func):
if not hasattr(self, "_event_handlers"):
self._event_handlers = {}
if event_name not in self._event_handlers:
self._event_handlers[event_name] = []
self._event_handlers[event_name].append(func)
return func
return decorator

64
src/pipecat/pipeline/task.py
View File

@@ -144,7 +144,26 @@ class PipelineTask(BaseTask):
`LLMFullResponseEndFrame` are received within `idle_timeout_secs`.
@task.event_handler("on_idle_timeout")
async def on_idle_timeout(task):
async def on_pipeline_idle_timeout(task):
...
There are also events to know if a pipeline has been started, stopped, ended
or cancelled.
@task.event_handler("on_pipeline_started")
async def on_pipeline_started(task, frame: StartFrame):
...
@task.event_handler("on_pipeline_stopped")
async def on_pipeline_stopped(task, frame: StopFrame):
...
@task.event_handler("on_pipeline_ended")
async def on_pipeline_ended(task, frame: EndFrame):
...
@task.event_handler("on_pipeline_cancelled")
async def on_pipeline_cancelled(task, frame: CancelFrame):
...
Args:
@@ -169,9 +188,9 @@ class PipelineTask(BaseTask):
self,
pipeline: BasePipeline,
*,
params: PipelineParams = PipelineParams(),
observers: List[BaseObserver] = [],
clock: BaseClock = SystemClock(),
params: Optional[PipelineParams] = None,
observers: Optional[List[BaseObserver]] = None,
clock: Optional[BaseClock] = None,
task_manager: Optional[BaseTaskManager] = None,
check_dangling_tasks: bool = True,
idle_timeout_secs: Optional[float] = 300,
@@ -186,8 +205,8 @@ class PipelineTask(BaseTask):
):
super().__init__()
self._pipeline = pipeline
self._clock = clock
self._params = params
self._clock = clock or SystemClock()
self._params = params or PipelineParams()
self._check_dangling_tasks = check_dangling_tasks
self._idle_timeout_secs = idle_timeout_secs
self._idle_timeout_frames = idle_timeout_frames
@@ -205,14 +224,17 @@ class PipelineTask(BaseTask):
DeprecationWarning,
)
observers = self._params.observers
observers = observers or []
self._turn_tracking_observer: Optional[TurnTrackingObserver] = None
self._turn_trace_observer: Optional[TurnTraceObserver] = None
if self._enable_turn_tracking:
self._turn_tracking_observer = TurnTrackingObserver()
observers = [self._turn_tracking_observer] + list(observers)
if self._enable_turn_tracking and self._enable_tracing:
observers.append(self._turn_tracking_observer)
if self._enable_tracing and self._turn_tracking_observer:
self._turn_trace_observer = TurnTraceObserver(
self._turn_tracking_observer, conversation_id=self._conversation_id
)
observers = [self._turn_trace_observer] + list(observers)
observers.append(self._turn_trace_observer)
self._finished = False
# This queue receives frames coming from the pipeline upstream.
@@ -264,6 +286,10 @@ class PipelineTask(BaseTask):
self._register_event_handler("on_frame_reached_upstream")
self._register_event_handler("on_frame_reached_downstream")
self._register_event_handler("on_idle_timeout")
self._register_event_handler("on_pipeline_started")
self._register_event_handler("on_pipeline_stopped")
self._register_event_handler("on_pipeline_ended")
self._register_event_handler("on_pipeline_cancelled")
@property
def params(self) -> PipelineParams:
@@ -273,12 +299,18 @@ class PipelineTask(BaseTask):
@property
def turn_tracking_observer(self) -> Optional[TurnTrackingObserver]:
"""Return the turn tracking observer if enabled."""
return getattr(self, "_turn_tracking_observer", None)
return self._turn_tracking_observer
@property
def turn_trace_observer(self) -> Optional[TurnTraceObserver]:
"""Return the turn trace observer if enabled."""
return getattr(self, "_turn_trace_observer", None)
return self._turn_trace_observer
async def add_observer(self, observer: BaseObserver):
await self._observer.add_observer(observer)
async def remove_observer(self, observer: BaseObserver):
await self._observer.remove_observer(observer)
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
self._task_manager.set_event_loop(loop)
@@ -552,8 +584,16 @@ class PipelineTask(BaseTask):
if isinstance(frame, self._reached_downstream_types):
await self._call_event_handler("on_frame_reached_downstream", frame)
if isinstance(frame, (EndFrame, StopFrame)):
if isinstance(frame, StartFrame):
await self._call_event_handler("on_pipeline_started", frame)
elif isinstance(frame, EndFrame):
await self._call_event_handler("on_pipeline_ended", frame)
self._pipeline_end_event.set()
elif isinstance(frame, StopFrame):
await self._call_event_handler("on_pipeline_stopped", frame)
self._pipeline_end_event.set()
elif isinstance(frame, CancelFrame):
await self._call_event_handler("on_pipeline_cancelled", frame)
elif isinstance(frame, HeartbeatFrame):
await self._heartbeat_queue.put(frame)
self._down_queue.task_done()

56
src/pipecat/pipeline/task_observer.py
View File

@@ -6,7 +6,7 @@
import asyncio
import inspect
from typing import List
from typing import Dict, List, Optional
from attr import dataclass
@@ -39,10 +39,32 @@ class TaskObserver(BaseObserver):
"""
def __init__(self, *, observers: List[BaseObserver] = [], task_manager: BaseTaskManager):
self._observers = observers
def __init__(
self,
*,
observers: Optional[List[BaseObserver]] = None,
task_manager: BaseTaskManager,
**kwargs,
):
super().__init__(**kwargs)
self._observers = observers or []
self._task_manager = task_manager
self._proxies: List[Proxy] = []
self._proxies: Dict[BaseObserver, Proxy] = {}
async def add_observer(self, observer: BaseObserver):
proxy = self._create_proxy(observer)
self._proxies[observer] = proxy
self._observers.append(observer)
async def remove_observer(self, observer: BaseObserver):
if observer in self._proxies:
proxy = self._proxies[observer]
# Remove the proxy so it doesn't get called anymore.
del self._proxies[observer]
# Cancel the proxy task right away.
await self._task_manager.cancel_task(proxy.task)
# Remove the observer.
self._observers.remove(observer)
async def start(self):
"""Starts all proxy observer tasks."""
@@ -50,23 +72,27 @@ class TaskObserver(BaseObserver):
async def stop(self):
"""Stops all proxy observer tasks."""
for proxy in self._proxies:
for proxy in self._proxies.values():
await self._task_manager.cancel_task(proxy.task)
async def on_push_frame(self, data: FramePushed):
for proxy in self._proxies:
for proxy in self._proxies.values():
await proxy.queue.put(data)
def _create_proxies(self, observers) -> List[Proxy]:
proxies = []
def _create_proxy(self, observer: BaseObserver) -> Proxy:
queue = asyncio.Queue()
task = self._task_manager.create_task(
self._proxy_task_handler(queue, observer),
f"TaskObserver::{observer}::_proxy_task_handler",
)
proxy = Proxy(queue=queue, task=task, observer=observer)
return proxy
def _create_proxies(self, observers: List[BaseObserver]) -> Dict[BaseObserver, Proxy]:
proxies = {}
for observer in observers:
queue = asyncio.Queue()
task = self._task_manager.create_task(
self._proxy_task_handler(queue, observer),
f"TaskObserver::{observer.__class__.__name__}::_proxy_task_handler",
)
proxy = Proxy(queue=queue, task=task, observer=observer)
proxies.append(proxy)
proxy = self._create_proxy(observer)
proxies[observer] = proxy
return proxies
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):

18
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -7,7 +7,7 @@
import asyncio
from abc import abstractmethod
from dataclasses import dataclass
from typing import Dict, List, Literal, Set
from typing import Dict, List, Literal, Optional, Set
from loguru import logger
@@ -243,11 +243,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
self,
context: OpenAILLMContext,
*,
params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
params: Optional[LLMUserAggregatorParams] = None,
**kwargs,
):
super().__init__(context=context, role="user", **kwargs)
self._params = params
self._params = params or LLMUserAggregatorParams()
if "aggregation_timeout" in kwargs:
import warnings
@@ -446,11 +446,11 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
self,
context: OpenAILLMContext,
*,
params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
params: Optional[LLMAssistantAggregatorParams] = None,
**kwargs,
):
super().__init__(context=context, role="assistant", **kwargs)
self._params = params
self._params = params or LLMAssistantAggregatorParams()
if "expect_stripped_words" in kwargs:
import warnings
@@ -640,9 +640,9 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
class LLMUserResponseAggregator(LLMUserContextAggregator):
def __init__(
self,
messages: List[dict] = [],
messages: Optional[List[dict]] = None,
*,
params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
params: Optional[LLMUserAggregatorParams] = None,
**kwargs,
):
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)
@@ -662,9 +662,9 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
def __init__(
self,
messages: List[dict] = [],
messages: Optional[List[dict]] = None,
*,
params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
params: Optional[LLMAssistantAggregatorParams] = None,
**kwargs,
):
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)

3
src/pipecat/processors/filters/wake_check_filter.py
View File

@@ -7,6 +7,7 @@
import re
import time
from enum import Enum
from typing import List
from loguru import logger
@@ -31,7 +32,7 @@ class WakeCheckFilter(FrameProcessor):
self.wake_timer = 0.0
self.accumulator = ""
def __init__(self, wake_phrases: list[str], keepalive_timeout: float = 3):
def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3):
super().__init__()
self._participant_states = {}
self._keepalive_timeout = keepalive_timeout

2
src/pipecat/processors/filters/wake_notifier_filter.py
View File

@@ -22,7 +22,7 @@ class WakeNotifierFilter(FrameProcessor):
self,
notifier: BaseNotifier,
*,
types: Tuple[Type[Frame]],
types: Tuple[Type[Frame], ...],
filter: Callable[[Frame], Awaitable[bool]],
**kwargs,
):

12
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -437,10 +437,12 @@ class RTVIObserver(BaseObserver):
params (RTVIObserverParams): Settings to enable/disable specific messages.
"""
def __init__(self, rtvi: "RTVIProcessor", *, params: RTVIObserverParams = RTVIObserverParams()):
super().__init__()
def __init__(
self, rtvi: "RTVIProcessor", *, params: Optional[RTVIObserverParams] = None, **kwargs
):
super().__init__(**kwargs)
self._rtvi = rtvi
self._params = params
self._params = params or RTVIObserverParams()
self._bot_transcription = ""
self._frames_seen = set()
rtvi.set_errors_enabled(self._params.errors_enabled)
@@ -632,12 +634,12 @@ class RTVIProcessor(FrameProcessor):
def __init__(
self,
*,
config: RTVIConfig = RTVIConfig(config=[]),
config: Optional[RTVIConfig] = None,
transport: Optional[BaseTransport] = None,
**kwargs,
):
super().__init__(**kwargs)
self._config = config
self._config = config or RTVIConfig(config=[])
self._bot_ready = False
self._client_ready = False

4
src/pipecat/processors/gstreamer/pipeline_source.py
View File

@@ -43,10 +43,10 @@ class GStreamerPipelineSource(FrameProcessor):
audio_channels: int = 1
clock_sync: bool = True
def __init__(self, *, pipeline: str, out_params: OutputParams = OutputParams(), **kwargs):
def __init__(self, *, pipeline: str, out_params: Optional[OutputParams] = None, **kwargs):
super().__init__(**kwargs)
self._out_params = out_params
self._out_params = out_params or GStreamerPipelineSource.OutputParams()
self._sample_rate = 0
Gst.init()

6
src/pipecat/processors/idle_frame_processor.py
View File

@@ -5,7 +5,7 @@
#
import asyncio
from typing import Awaitable, Callable, List
from typing import Awaitable, Callable, List, Optional
from pipecat.frames.frames import Frame, StartFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -22,14 +22,14 @@ class IdleFrameProcessor(FrameProcessor):
*,
callback: Callable[["IdleFrameProcessor"], Awaitable[None]],
timeout: float,
types: List[type] = [],
types: Optional[List[type]] = None,
**kwargs,
):
super().__init__(**kwargs)
self._callback = callback
self._timeout = timeout
self._types = types
self._types = types or []
self._idle_task = None
async def process_frame(self, frame: Frame, direction: FrameDirection):

19
src/pipecat/processors/logger.py
View File

@@ -4,11 +4,17 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Optional
from typing import Optional, Tuple, Type
from loguru import logger
from pipecat.frames.frames import AudioRawFrame, BotSpeakingFrame, Frame, TransportMessageFrame
from pipecat.frames.frames import (
BotSpeakingFrame,
Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
TransportMessageFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
logger = logger.opt(ansi=True)
@@ -19,16 +25,17 @@ class FrameLogger(FrameProcessor):
self,
prefix="Frame",
color: Optional[str] = None,
ignored_frame_types: Optional[list] = [
ignored_frame_types: Tuple[Type[Frame], ...] = (
BotSpeakingFrame,
AudioRawFrame,
InputAudioRawFrame,
OutputAudioRawFrame,
TransportMessageFrame,
],
),
):
super().__init__()
self._prefix = prefix
self._color = color
self._ignored_frame_types = tuple(ignored_frame_types) if ignored_frame_types else None
self._ignored_frame_types = ignored_frame_types
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)

2
src/pipecat/processors/transcript_processor.py
View File

@@ -62,7 +62,7 @@ class UserTranscriptProcessor(BaseTranscriptProcessor):
if isinstance(frame, TranscriptionFrame):
message = TranscriptionMessage(
role="user", content=frame.text, timestamp=frame.timestamp
role="user", user_id=frame.user_id, content=frame.text, timestamp=frame.timestamp
)
await self._emit_update([message])

8
src/pipecat/serializers/telnyx.py
View File

@@ -79,7 +79,7 @@ class TelnyxFrameSerializer(FrameSerializer):
inbound_encoding: str,
call_control_id: Optional[str] = None,
api_key: Optional[str] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
):
"""Initialize the TelnyxFrameSerializer.
@@ -92,11 +92,11 @@ class TelnyxFrameSerializer(FrameSerializer):
params: Configuration parameters.
"""
self._stream_id = stream_id
params.outbound_encoding = outbound_encoding
params.inbound_encoding = inbound_encoding
self._call_control_id = call_control_id
self._api_key = api_key
self._params = params
self._params = params or TelnyxFrameSerializer.InputParams()
self._params.outbound_encoding = outbound_encoding
self._params.inbound_encoding = inbound_encoding
self._telnyx_sample_rate = self._params.telnyx_sample_rate
self._sample_rate = 0 # Pipeline input rate

4
src/pipecat/serializers/twilio.py
View File

@@ -69,7 +69,7 @@ class TwilioFrameSerializer(FrameSerializer):
call_sid: Optional[str] = None,
account_sid: Optional[str] = None,
auth_token: Optional[str] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
):
"""Initialize the TwilioFrameSerializer.
@@ -84,7 +84,7 @@ class TwilioFrameSerializer(FrameSerializer):
self._call_sid = call_sid
self._account_sid = account_sid
self._auth_token = auth_token
self._params = params
self._params = params or TwilioFrameSerializer.InputParams()
self._twilio_sample_rate = self._params.twilio_sample_rate
self._sample_rate = 0 # Pipeline input rate

5
src/pipecat/services/anthropic/llm.py
View File

@@ -90,12 +90,13 @@ class AnthropicLLMService(LLMService):
self,
*,
api_key: str,
model: str = "claude-3-7-sonnet-20250219",
params: InputParams = InputParams(),
model: str = "claude-sonnet-4-20250514",
params: Optional[InputParams] = None,
client=None,
**kwargs,
):
super().__init__(**kwargs)
params = params or AnthropicLLMService.InputParams()
self._client = client or AsyncAnthropic(
api_key=api_key
) # if the client is provided, use it and remove it, otherwise create a new one

2
src/pipecat/services/assemblyai/stt.py
View File

@@ -58,7 +58,7 @@ class AssemblyAISTTService(STTService):
self._connection_params = connection_params
super().__init__(sample_rate=self._connection_params.sample_rate, **kwargs)
self._websocket = None
self._termination_event = asyncio.Event()
self._received_termination = False

6
src/pipecat/services/aws/llm.py
View File

@@ -530,17 +530,19 @@ class AWSBedrockLLMService(LLMService):
def __init__(
self,
*,
model: str,
aws_access_key: Optional[str] = None,
aws_secret_key: Optional[str] = None,
aws_session_token: Optional[str] = None,
aws_region: str = "us-east-1",
model: str,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
client_config: Optional[Config] = None,
**kwargs,
):
super().__init__(**kwargs)
params = params or AWSBedrockLLMService.InputParams()
# Initialize the AWS Bedrock client
if not client_config:
client_config = Config(

4
src/pipecat/services/aws/tts.py
View File

@@ -125,11 +125,13 @@ class AWSPollyTTSService(TTSService):
region: Optional[str] = None,
voice_id: str = "Joanna",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AWSPollyTTSService.InputParams()
self._polly_client = boto3.client(
"polly",
aws_access_key_id=aws_access_key_id,

4
src/pipecat/services/aws_nova_sonic/aws.py
View File

@@ -142,7 +142,7 @@ class AWSNovaSonicLLMService(LLMService):
region: str,
model: str = "amazon.nova-sonic-v1:0",
voice_id: str = "matthew", # matthew, tiffany, amy
params: Params = Params(),
params: Optional[Params] = None,
system_instruction: Optional[str] = None,
tools: Optional[ToolsSchema] = None,
send_transcription_frames: bool = True,
@@ -155,7 +155,7 @@ class AWSNovaSonicLLMService(LLMService):
self._model = model
self._client: Optional[BedrockRuntimeClient] = None
self._voice_id = voice_id
self._params = params
self._params = params or Params()
self._system_instruction = system_instruction
self._tools = tools
self._send_transcription_frames = send_transcription_frames

4
src/pipecat/services/azure/tts.py
View File

@@ -68,11 +68,13 @@ class AzureBaseTTSService(TTSService):
region: str,
voice="en-US-SaraNeural",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AzureBaseTTSService.InputParams()
self._settings = {
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)

98
src/pipecat/services/cartesia/tts.py
View File

@@ -7,10 +7,11 @@
import base64
import json
import uuid
import warnings
from typing import AsyncGenerator, List, Optional, Union
from loguru import logger
from pydantic import BaseModel
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
CancelFrame,
@@ -83,13 +84,13 @@ class CartesiaTTSService(AudioContextWordTTSService):
*,
api_key: str,
voice_id: str,
cartesia_version: str = "2024-06-10",
cartesia_version: str = "2025-04-16",
url: str = "wss://api.cartesia.ai/tts/websocket",
model: str = "sonic-2",
sample_rate: Optional[int] = None,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
text_aggregator: Optional[BaseTextAggregator] = None,
**kwargs,
):
@@ -112,6 +113,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
**kwargs,
)
params = params or CartesiaTTSService.InputParams()
self._api_key = api_key
self._cartesia_version = cartesia_version
self._url = url
@@ -151,10 +154,13 @@ class CartesiaTTSService(AudioContextWordTTSService):
voice_config["mode"] = "id"
voice_config["id"] = self._voice_id
if self._settings["speed"] or self._settings["emotion"]:
if self._settings["emotion"]:
warnings.warn(
"The 'emotion' parameter in __experimental_controls is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
voice_config["__experimental_controls"] = {}
if self._settings["speed"]:
voice_config["__experimental_controls"]["speed"] = self._settings["speed"]
if self._settings["emotion"]:
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
@@ -169,6 +175,10 @@ class CartesiaTTSService(AudioContextWordTTSService):
"add_timestamps": add_timestamps,
"use_original_timestamps": False if self.model_name == "sonic" else True,
}
if self._settings["speed"]:
msg["speed"] = self._settings["speed"]
return json.dumps(msg)
async def start(self, frame: StartFrame):
@@ -309,7 +319,7 @@ class CartesiaHttpTTSService(TTSService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
speed: Optional[Union[str, float]] = ""
emotion: Optional[List[str]] = []
emotion: Optional[List[str]] = Field(default_factory=list)
def __init__(
self,
@@ -318,15 +328,20 @@ class CartesiaHttpTTSService(TTSService):
voice_id: str,
model: str = "sonic-2",
base_url: str = "https://api.cartesia.ai",
cartesia_version: str = "2024-11-13",
sample_rate: Optional[int] = None,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or CartesiaHttpTTSService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._cartesia_version = cartesia_version
self._settings = {
"output_format": {
"container": container,
@@ -342,7 +357,10 @@ class CartesiaHttpTTSService(TTSService):
self.set_voice(voice_id)
self.set_model_name(model)
self._client = AsyncCartesia(api_key=api_key, base_url=base_url)
self._client = AsyncCartesia(
api_key=api_key,
base_url=base_url,
)
def can_generate_metrics(self) -> bool:
return True
@@ -367,37 +385,63 @@ class CartesiaHttpTTSService(TTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
voice_controls = None
if self._settings["speed"] or self._settings["emotion"]:
voice_controls = {}
if self._settings["speed"]:
voice_controls["speed"] = self._settings["speed"]
if self._settings["emotion"]:
voice_controls["emotion"] = self._settings["emotion"]
voice_config = {"mode": "id", "id": self._voice_id}
if self._settings["emotion"]:
warnings.warn(
"The 'emotion' parameter in voice.__experimental_controls is deprecated and will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]}
await self.start_ttfb_metrics()
output = await self._client.tts.sse(
model_id=self._model_name,
transcript=text,
voice_id=self._voice_id,
output_format=self._settings["output_format"],
language=self._settings["language"],
stream=False,
_experimental_voice_controls=voice_controls,
)
payload = {
"model_id": self._model_name,
"transcript": text,
"voice": voice_config,
"output_format": self._settings["output_format"],
"language": self._settings["language"],
}
await self.start_tts_usage_metrics(text)
if self._settings["speed"]:
payload["speed"] = self._settings["speed"]
yield TTSStartedFrame()
session = await self._client._get_session()
headers = {
"Cartesia-Version": self._cartesia_version,
"X-API-Key": self._api_key,
"Content-Type": "application/json",
}
url = f"{self._base_url}/tts/bytes"
async with session.post(url, json=payload, headers=headers) as response:
if response.status != 200:
error_text = await response.text()
logger.error(f"Cartesia API error: {error_text}")
await self.push_error(ErrorFrame(f"Cartesia API error: {error_text}"))
raise Exception(f"Cartesia API returned status {response.status}: {error_text}")
audio_data = await response.read()
await self.start_tts_usage_metrics(text)
frame = TTSAudioRawFrame(
audio=output["audio"], sample_rate=self.sample_rate, num_channels=1
audio=audio_data,
sample_rate=self.sample_rate,
num_channels=1,
)
yield frame
except Exception as e:
logger.error(f"{self} exception: {e}")
await self.push_error(ErrorFrame(f"Error generating TTS: {e}"))
finally:
await self.stop_ttfb_metrics()
yield TTSStoppedFrame()

18
src/pipecat/services/elevenlabs/tts.py
View File

@@ -184,7 +184,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
model: str = "eleven_flash_v2_5",
url: str = "wss://api.elevenlabs.io",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
# Aggregating sentences still gives cleaner-sounding results and fewer
@@ -210,6 +210,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
**kwargs,
)
params = params or ElevenLabsTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = {
@@ -252,14 +254,16 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
async def set_model(self, model: str):
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
# No need to disconnect/reconnect for model changes with multi-context API
await self._disconnect()
await self._connect()
async def _update_settings(self, settings: Mapping[str, Any]):
prev_voice = self._voice_id
await super()._update_settings(settings)
# If voice changes, we don't need to reconnect, just use a new context
if not prev_voice == self._voice_id:
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
await self._disconnect()
await self._connect()
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -386,7 +390,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
msg = json.loads(message)
# Check if this message belongs to the current context
# The default context may return null/None for context_id
received_ctx_id = msg.get("context_id")
received_ctx_id = msg.get("contextId")
if (
self._context_id is not None
and received_ctx_id is not None
@@ -406,7 +410,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
await self.add_word_timestamps(word_times)
self._cumulative_time = word_times[-1][1]
if msg.get("is_final"):
if msg.get("isFinal"):
logger.trace(f"Received final message for context {received_ctx_id}")
# Context has finished
if self._context_id == received_ctx_id:
@@ -512,7 +516,7 @@ class ElevenLabsHttpTTSService(WordTTSService):
model: str = "eleven_flash_v2_5",
base_url: str = "https://api.elevenlabs.io",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(
@@ -523,6 +527,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
**kwargs,
)
params = params or ElevenLabsHttpTTSService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._params = params

4
src/pipecat/services/fal/stt.py
View File

@@ -173,7 +173,7 @@ class FalSTTService(SegmentedSTTService):
*,
api_key: Optional[str] = None,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(
@@ -181,6 +181,8 @@ class FalSTTService(SegmentedSTTService):
**kwargs,
)
params = params or FalSTTService.InputParams()
if api_key:
os.environ["FAL_KEY"] = api_key
elif "FAL_KEY" not in os.environ:

4
src/pipecat/services/fish/tts.py
View File

@@ -52,7 +52,7 @@ class FishAudioTTSService(InterruptibleTTSService):
model: str, # This is the reference_id
output_format: FishAudioOutputFormat = "pcm",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(
@@ -62,6 +62,8 @@ class FishAudioTTSService(InterruptibleTTSService):
**kwargs,
)
params = params or FishAudioTTSService.InputParams()
self._api_key = api_key
self._base_url = "wss://api.fish.audio/v1/tts/live"
self._websocket = None

19
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -335,17 +335,20 @@ class GeminiMultimodalLiveLLMService(LLMService):
*,
api_key: str,
base_url: str = "generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent",
model="models/gemini-2.0-flash-live-001",
model="models/gemini-2.5-flash-preview-native-audio-dialog",
voice_id: str = "Charon",
start_audio_paused: bool = False,
start_video_paused: bool = False,
system_instruction: Optional[str] = None,
tools: Optional[Union[List[dict], ToolsSchema]] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
inference_on_context_initialization: bool = True,
**kwargs,
):
super().__init__(base_url=base_url, **kwargs)
params = params or InputParams()
self._last_sent_time = 0
self._api_key = api_key
self._base_url = base_url
@@ -960,11 +963,17 @@ class GeminiMultimodalLiveLLMService(LLMService):
usage = evt.usageMetadata
# Ensure we have valid integers for all token counts
prompt_tokens = usage.promptTokenCount or 0
completion_tokens = usage.responseTokenCount or 0
total_tokens = usage.totalTokenCount or (prompt_tokens + completion_tokens)
tokens = LLMTokenUsage(
prompt_tokens=usage.promptTokenCount,
completion_tokens=usage.responseTokenCount,
total_tokens=usage.totalTokenCount,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
)
await self.start_llm_usage_metrics(tokens)
def create_context_aggregator(

4
src/pipecat/services/gladia/stt.py
View File

@@ -194,7 +194,7 @@ class GladiaSTTService(STTService):
confidence: float = 0.5,
sample_rate: Optional[int] = None,
model: str = "solaria-1",
params: GladiaInputParams = GladiaInputParams(),
params: Optional[GladiaInputParams] = None,
**kwargs,
):
"""Initialize the Gladia STT service.
@@ -211,6 +211,8 @@ class GladiaSTTService(STTService):
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GladiaInputParams()
# Warn about deprecated language parameter if it's used
if params.language is not None:
warnings.warn(

10
src/pipecat/services/google/image.py
View File

@@ -10,7 +10,7 @@ import os
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from loguru import logger
from PIL import Image
@@ -32,19 +32,19 @@ class GoogleImageGenService(ImageGenService):
class InputParams(BaseModel):
number_of_images: int = Field(default=1, ge=1, le=8)
model: str = Field(default="imagen-3.0-generate-002")
negative_prompt: str = Field(default=None)
negative_prompt: Optional[str] = Field(default=None)
def __init__(
self,
*,
params: InputParams = InputParams(),
api_key: str,
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(**kwargs)
self.set_model_name(params.model)
self._params = params
self._params = params or GoogleImageGenService.InputParams()
self._client = genai.Client(api_key=api_key)
self.set_model_name(self._params.model)
def can_generate_metrics(self) -> bool:
return True

5
src/pipecat/services/google/llm.py
View File

@@ -467,13 +467,16 @@ class GoogleLLMService(LLMService):
*,
api_key: str,
model: str = "gemini-2.0-flash",
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
system_instruction: Optional[str] = None,
tools: Optional[List[Dict[str, Any]]] = None,
tool_config: Optional[Dict[str, Any]] = None,
**kwargs,
):
super().__init__(**kwargs)
params = params or GoogleLLMService.InputParams()
self.set_model_name(model)
self._api_key = api_key
self._system_instruction = system_instruction

3
src/pipecat/services/google/llm_vertex.py
View File

@@ -52,7 +52,7 @@ class GoogleVertexLLMService(OpenAILLMService):
credentials: Optional[str] = None,
credentials_path: Optional[str] = None,
model: str = "google/gemini-2.0-flash-001",
params: InputParams = OpenAILLMService.InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
"""Initializes the VertexLLMService.
@@ -64,6 +64,7 @@ class GoogleVertexLLMService(OpenAILLMService):
params (InputParams): Vertex AI input parameters.
**kwargs: Additional arguments for OpenAILLMService.
"""
params = params or OpenAILLMService.InputParams()
base_url = self._get_base_url(params)
self._api_key = self._get_api_token(credentials, credentials_path)

4
src/pipecat/services/google/stt.py
View File

@@ -412,7 +412,7 @@ class GoogleSTTService(STTService):
credentials_path: Optional[str] = None,
location: str = "global",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
"""Initialize the Google STT service.
@@ -431,6 +431,8 @@ class GoogleSTTService(STTService):
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleSTTService.InputParams()
self._location = location
self._stream = None
self._config = None

4
src/pipecat/services/google/tts.py
View File

@@ -219,11 +219,13 @@ class GoogleTTSService(TTSService):
credentials_path: Optional[str] = None,
voice_id: str = "en-US-Neural2-A",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleTTSService.InputParams()
self._settings = {
"pitch": params.pitch,
"rate": params.rate,

5
src/pipecat/services/groq/tts.py
View File

@@ -34,7 +34,7 @@ class GroqTTSService(TTSService):
*,
api_key: str,
output_format: str = "wav",
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
model_name: str = "playai-tts",
voice_id: str = "Celeste-PlayAI",
sample_rate: Optional[int] = GROQ_SAMPLE_RATE,
@@ -42,12 +42,15 @@ class GroqTTSService(TTSService):
):
if sample_rate != self.GROQ_SAMPLE_RATE:
logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ")
super().__init__(
pause_frame_processing=True,
sample_rate=sample_rate,
**kwargs,
)
params = params or GroqTTSService.InputParams()
self._api_key = api_key
self._model_name = model_name
self._output_format = output_format

6
src/pipecat/services/mcp_service.py
View File

@@ -155,8 +155,7 @@ class MCPClient(BaseObject):
error_msg = f"Error calling mcp tool {function_name}: {str(e)}"
logger.error(error_msg)
response = "Sorry, could not call the mcp tool"
image_url = None
response = ""
if results:
if hasattr(results, "content") and results.content:
for i, content in enumerate(results.content):
@@ -171,7 +170,8 @@ class MCPClient(BaseObject):
else:
logger.error(f"Error getting content from {function_name} results.")
await result_callback(response)
final_response = response if len(response) else "Sorry, could not call the mcp tool"
await result_callback(final_response)
async def _list_tools(self, session, mcp_tool_wrapper, llm):
available_tools = await session.list_tools()

17
src/pipecat/services/mem0/memory.py
View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
from loguru import logger
from pydantic import BaseModel, Field
@@ -49,16 +49,19 @@ class Mem0MemoryService(FrameProcessor):
def __init__(
self,
*,
api_key: str = None,
local_config: Dict[str, Any] = {},
user_id: str = None,
agent_id: str = None,
run_id: str = None,
params: InputParams = InputParams(),
api_key: Optional[str] = None,
local_config: Optional[Dict[str, Any]] = None,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
run_id: Optional[str] = None,
params: Optional[InputParams] = None,
):
# Important: Call the parent class __init__ first
super().__init__()
local_config = local_config or {}
params = params or Mem0MemoryService.InputParams()
if local_config:
self.memory_client = Memory.from_config(local_config)
else:

4
src/pipecat/services/minimax/tts.py
View File

@@ -114,11 +114,13 @@ class MiniMaxHttpTTSService(TTSService):
voice_id: str = "Calm_Woman",
aiohttp_session: aiohttp.ClientSession,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or MiniMaxHttpTTSService.InputParams()
self._api_key = api_key
self._group_id = group_id
self._base_url = f"https://api.minimaxi.chat/v1/t2a_v2?GroupId={group_id}"

8
src/pipecat/services/neuphonic/tts.py
View File

@@ -80,7 +80,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
url: str = "wss://api.neuphonic.com",
sample_rate: Optional[int] = 22050,
encoding: str = "pcm_linear",
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(
@@ -92,6 +92,8 @@ class NeuphonicTTSService(InterruptibleTTSService):
**kwargs,
)
params = params or NeuphonicTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = {
@@ -293,11 +295,13 @@ class NeuphonicHttpTTSService(TTSService):
url: str = "https://api.neuphonic.com",
sample_rate: Optional[int] = 22050,
encoding: str = "pcm_linear",
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NeuphonicHttpTTSService.InputParams()
self._api_key = api_key
self._url = url
self._settings = {

7
src/pipecat/services/openai/base_llm.py
View File

@@ -77,11 +77,14 @@ class BaseOpenAILLMService(LLMService):
base_url=None,
organization=None,
project=None,
default_headers: Mapping[str, str] | None = None,
params: InputParams = InputParams(),
default_headers: Optional[Mapping[str, str]] = None,
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(**kwargs)
params = params or BaseOpenAILLMService.InputParams()
self._settings = {
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,

4
src/pipecat/services/openai/llm.py
View File

@@ -6,7 +6,7 @@
import json
from dataclasses import dataclass
from typing import Any
from typing import Any, Optional
from pipecat.frames.frames import (
FunctionCallCancelFrame,
@@ -41,7 +41,7 @@ class OpenAILLMService(BaseOpenAILLMService):
self,
*,
model: str = "gpt-4.1",
params: BaseOpenAILLMService.InputParams = BaseOpenAILLMService.InputParams(),
params: Optional[BaseOpenAILLMService.InputParams] = None,
**kwargs,
):
super().__init__(model=model, params=params, **kwargs)

8
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -8,6 +8,7 @@ import base64
import json
import time
from dataclasses import dataclass
from typing import Optional
from loguru import logger
@@ -89,17 +90,20 @@ class OpenAIRealtimeBetaLLMService(LLMService):
api_key: str,
model: str = "gpt-4o-realtime-preview-2024-12-17",
base_url: str = "wss://api.openai.com/v1/realtime",
session_properties: events.SessionProperties = events.SessionProperties(),
session_properties: Optional[events.SessionProperties] = None,
start_audio_paused: bool = False,
send_transcription_frames: bool = True,
**kwargs,
):
full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs)
self.api_key = api_key
self.base_url = full_url
self._session_properties: events.SessionProperties = session_properties
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._send_transcription_frames = send_transcription_frames
self._websocket = None

8
src/pipecat/services/playht/tts.py
View File

@@ -110,7 +110,7 @@ class PlayHTTTSService(InterruptibleTTSService):
voice_engine: str = "Play3.0-mini",
sample_rate: Optional[int] = None,
output_format: str = "wav",
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(
@@ -119,6 +119,8 @@ class PlayHTTTSService(InterruptibleTTSService):
**kwargs,
)
params = params or PlayHTTTSService.InputParams()
self._api_key = api_key
self._user_id = user_id
self._websocket_url = None
@@ -328,11 +330,13 @@ class PlayHTHttpTTSService(TTSService):
voice_engine: str = "Play3.0-mini",
protocol: str = "http", # Options: http, ws
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or PlayHTHttpTTSService.InputParams()
self._user_id = user_id
self._api_key = api_key

8
src/pipecat/services/rime/tts.py
View File

@@ -80,7 +80,7 @@ class RimeTTSService(AudioContextWordTTSService):
url: str = "wss://users.rime.ai/ws2",
model: str = "mistv2",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
text_aggregator: Optional[BaseTextAggregator] = None,
**kwargs,
):
@@ -105,6 +105,8 @@ class RimeTTSService(AudioContextWordTTSService):
**kwargs,
)
params = params or RimeTTSService.InputParams()
# Store service configuration
self._api_key = api_key
self._url = url
@@ -364,11 +366,13 @@ class RimeHttpTTSService(TTSService):
aiohttp_session: aiohttp.ClientSession,
model: str = "mistv2",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RimeHttpTTSService.InputParams()
self._api_key = api_key
self._session = aiohttp_session
self._base_url = "https://users.rime.ai/v1/rime-tts"

11
src/pipecat/services/riva/stt.py
View File

@@ -99,10 +99,13 @@ class RivaSTTService(STTService):
"model_name": "parakeet-ctc-1.1b-asr",
},
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RivaSTTService.InputParams()
self._api_key = api_key
self._profanity_filter = False
self._automatic_punctuation = True
@@ -322,11 +325,13 @@ class RivaSegmentedSTTService(SegmentedSTTService):
"model_name": "canary-1b-asr",
},
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RivaSegmentedSTTService.InputParams()
# Set model name
self.set_model_name(model_function_map.get("model_name"))
@@ -533,7 +538,7 @@ class ParakeetSTTService(RivaSTTService):
"model_name": "parakeet-ctc-1.1b-asr",
},
sample_rate: Optional[int] = None,
params: RivaSTTService.InputParams = RivaSTTService.InputParams(), # Use parent class's type
params: Optional[RivaSTTService.InputParams] = None, # Use parent class's type
**kwargs,
):
super().__init__(

16
src/pipecat/services/riva/tts.py
View File

@@ -44,7 +44,7 @@ class RivaTTSService(TTSService):
def __init__(
self,
*,
api_key: str = None,
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
voice_id: str = "Magpie-Multilingual.EN-US.Ray",
sample_rate: Optional[int] = None,
@@ -52,10 +52,13 @@ class RivaTTSService(TTSService):
"function_id": "877104f7-e885-42b9-8de8-f6e4c6303969",
"model_name": "magpie-tts-multilingual",
},
params: InputParams = InputParams(),
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RivaTTSService.InputParams()
self._api_key = api_key
self._voice_id = voice_id
self._language_code = params.language
@@ -136,14 +139,10 @@ class RivaTTSService(TTSService):
class FastPitchTTSService(RivaTTSService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN_US
quality: Optional[int] = 20
def __init__(
self,
*,
api_key: str = None,
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
voice_id: str = "English-US.Female-1",
sample_rate: Optional[int] = None,
@@ -151,11 +150,12 @@ class FastPitchTTSService(RivaTTSService):
"function_id": "0149dedb-2be8-4195-b9a0-e57e0e14f972",
"model_name": "fastpitch-hifigan-tts",
},
params: InputParams = InputParams(),
params: Optional[RivaTTSService.InputParams] = None,
**kwargs,
):
super().__init__(
api_key=api_key,
server=server,
voice_id=voice_id,
sample_rate=sample_rate,
model_function_map=model_function_map,

8
src/pipecat/services/sarvam/__init__.py Normal file
View File

@@ -0,0 +1,8 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from .tts import *

195
src/pipecat/services/sarvam/tts.py Normal file
View File

@@ -0,0 +1,195 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import base64
from typing import AsyncGenerator, Optional
import aiohttp
from loguru import logger
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
ErrorFrame,
Frame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
def language_to_sarvam_language(language: Language) -> Optional[str]:
"""Convert Pipecat Language enum to Sarvam AI language codes."""
LANGUAGE_MAP = {
Language.BN: "bn-IN", # Bengali
Language.EN: "en-IN", # English (India)
Language.GU: "gu-IN", # Gujarati
Language.HI: "hi-IN", # Hindi
Language.KN: "kn-IN", # Kannada
Language.ML: "ml-IN", # Malayalam
Language.MR: "mr-IN", # Marathi
Language.OR: "od-IN", # Odia
Language.PA: "pa-IN", # Punjabi
Language.TA: "ta-IN", # Tamil
Language.TE: "te-IN", # Telugu
}
return LANGUAGE_MAP.get(language)
class SarvamTTSService(TTSService):
"""Text-to-Speech service using Sarvam AI's API.
Converts text to speech using Sarvam AI's TTS models with support for multiple
Indian languages. Provides control over voice characteristics like pitch, pace,
and loudness.
Args:
api_key: Sarvam AI API subscription key.
voice_id: Speaker voice ID (e.g., "anushka", "meera").
model: TTS model to use ("bulbul:v1" or "bulbul:v2").
aiohttp_session: Shared aiohttp session for making requests.
base_url: Sarvam AI API base URL.
sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000).
params: Additional voice and preprocessing parameters.
Example:
```python
tts = SarvamTTSService(
api_key="your-api-key",
voice_id="anushka",
model="bulbul:v2",
aiohttp_session=session,
params=SarvamTTSService.InputParams(
language=Language.HI,
pitch=0.1,
pace=1.2
)
)
```
"""
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
pitch: Optional[float] = Field(default=0.0, ge=-0.75, le=0.75)
pace: Optional[float] = Field(default=1.0, ge=0.3, le=3.0)
loudness: Optional[float] = Field(default=1.0, ge=0.1, le=3.0)
enable_preprocessing: Optional[bool] = False
def __init__(
self,
*,
api_key: str,
voice_id: str = "anushka",
model: str = "bulbul:v2",
aiohttp_session: aiohttp.ClientSession,
base_url: str = "https://api.sarvam.ai",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or SarvamTTSService.InputParams()
self._api_key = api_key
self._base_url = base_url
self._session = aiohttp_session
self._settings = {
"language": self.language_to_service_language(params.language)
if params.language
else "en-IN",
"pitch": params.pitch,
"pace": params.pace,
"loudness": params.loudness,
"enable_preprocessing": params.enable_preprocessing,
}
self.set_model_name(model)
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
return language_to_sarvam_language(language)
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
try:
await self.start_ttfb_metrics()
payload = {
"text": text,
"target_language_code": self._settings["language"],
"speaker": self._voice_id,
"pitch": self._settings["pitch"],
"pace": self._settings["pace"],
"loudness": self._settings["loudness"],
"speech_sample_rate": self.sample_rate,
"enable_preprocessing": self._settings["enable_preprocessing"],
"model": self._model_name,
}
headers = {
"api-subscription-key": self._api_key,
"Content-Type": "application/json",
}
url = f"{self._base_url}/text-to-speech"
yield TTSStartedFrame()
async with self._session.post(url, json=payload, headers=headers) as response:
if response.status != 200:
error_text = await response.text()
logger.error(f"Sarvam API error: {error_text}")
await self.push_error(ErrorFrame(f"Sarvam API error: {error_text}"))
return
response_data = await response.json()
await self.start_tts_usage_metrics(text)
# Decode base64 audio data
if "audios" not in response_data or not response_data["audios"]:
logger.error("No audio data received from Sarvam API")
await self.push_error(ErrorFrame("No audio data received"))
return
# Get the first audio (there should be only one for single text input)
base64_audio = response_data["audios"][0]
audio_data = base64.b64decode(base64_audio)
# Strip WAV header (first 44 bytes) if present
if audio_data.startswith(b"RIFF"):
logger.debug("Stripping WAV header from Sarvam audio data")
audio_data = audio_data[44:]
frame = TTSAudioRawFrame(
audio=audio_data,
sample_rate=self.sample_rate,
num_channels=1,
)
yield frame
except Exception as e:
logger.error(f"{self} exception: {e}")
await self.push_error(ErrorFrame(f"Error generating TTS: {e}"))
finally:
await self.stop_ttfb_metrics()
yield TTSStoppedFrame()

18
src/pipecat/services/tts_service.py
View File

@@ -64,7 +64,7 @@ class TTSService(AIService):
# Text aggregator to aggregate incoming tokens and decide when to push to the TTS.
text_aggregator: Optional[BaseTextAggregator] = None,
# Text filter executed after text has been aggregated.
text_filters: Sequence[BaseTextFilter] = [],
text_filters: Optional[Sequence[BaseTextFilter]] = None,
text_filter: Optional[BaseTextFilter] = None,
# Audio transport destination of the generated frames.
transport_destination: Optional[str] = None,
@@ -83,7 +83,7 @@ class TTSService(AIService):
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
self._text_filters: Sequence[BaseTextFilter] = text_filters
self._text_filters: Sequence[BaseTextFilter] = text_filters or []
self._transport_destination: Optional[str] = transport_destination
if text_filter:
@@ -157,7 +157,7 @@ class TTSService(AIService):
self.set_voice(value)
elif key == "text_filter":
for filter in self._text_filters:
filter.update_settings(value)
await filter.update_settings(value)
else:
logger.warning(f"Unknown setting for TTS service: {key}")
@@ -183,7 +183,7 @@ class TTSService(AIService):
await self._maybe_pause_frame_processing()
sentence = self._text_aggregator.text
self._text_aggregator.reset()
await self._text_aggregator.reset()
self._processing_text = False
await self._push_tts_frames(sentence)
if isinstance(frame, LLMFullResponseEndFrame):
@@ -234,9 +234,9 @@ class TTSService(AIService):
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
self._processing_text = False
self._text_aggregator.handle_interruption()
await self._text_aggregator.handle_interruption()
for filter in self._text_filters:
filter.handle_interruption()
await filter.handle_interruption()
async def _maybe_pause_frame_processing(self):
if self._processing_text and self._pause_frame_processing:
@@ -251,7 +251,7 @@ class TTSService(AIService):
if not self._aggregate_sentences:
text = frame.text
else:
text = self._text_aggregator.aggregate(frame.text)
text = await self._text_aggregator.aggregate(frame.text)
if text:
await self._push_tts_frames(text)
@@ -274,8 +274,8 @@ class TTSService(AIService):
# Process all filter.
for filter in self._text_filters:
filter.reset_interruption()
text = filter.filter(text)
await filter.reset_interruption()
text = await filter.filter(text)
if text:
await self.process_generator(self.run_tts(text))

9
src/pipecat/tests/utils.py
View File

@@ -38,7 +38,9 @@ class HeartbeatsObserver(BaseObserver):
*,
target: FrameProcessor,
heartbeat_callback: Callable[[FrameProcessor, HeartbeatFrame], Awaitable[None]],
**kwargs,
):
super().__init__(**kwargs)
self._target = target
self._callback = heartbeat_callback
@@ -79,10 +81,13 @@ async def run_test(
expected_down_frames: Optional[Sequence[type]] = None,
expected_up_frames: Optional[Sequence[type]] = None,
ignore_start: bool = True,
observers: List[BaseObserver] = [],
start_metadata: Dict[str, Any] = {},
observers: Optional[List[BaseObserver]] = None,
start_metadata: Optional[Dict[str, Any]] = None,
send_end_frame: bool = True,
) -> Tuple[Sequence[Frame], Sequence[Frame]]:
observers = observers or []
start_metadata = start_metadata or {}
received_up = asyncio.Queue()
received_down = asyncio.Queue()
source = QueuedFrameProcessor(

6
src/pipecat/transports/network/websocket_client.py
View File

@@ -250,11 +250,11 @@ class WebsocketClientTransport(BaseTransport):
def __init__(
self,
uri: str,
params: WebsocketClientParams = WebsocketClientParams(),
params: Optional[WebsocketClientParams] = None,
):
super().__init__()
self._params = params
self._params = params or WebsocketClientParams()
callbacks = WebsocketClientCallbacks(
on_connected=self._on_connected,
@@ -262,7 +262,7 @@ class WebsocketClientTransport(BaseTransport):
on_message=self._on_message,
)
self._session = WebsocketClientSession(uri, params, callbacks, self.name)
self._session = WebsocketClientSession(uri, self._params, callbacks, self.name)
self._input: Optional[WebsocketClientInputTransport] = None
self._output: Optional[WebsocketClientOutputTransport] = None

10
src/pipecat/transports/services/daily.py
View File

@@ -95,7 +95,7 @@ class WebRTCVADAnalyzer(VADAnalyzer):
params: VAD configuration parameters (VADParams).
"""
def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
super().__init__(sample_rate=sample_rate, params=params)
self._webrtc_vad = Daily.create_native_vad(
@@ -1223,7 +1223,7 @@ class DailyTransport(BaseTransport):
room_url: str,
token: Optional[str],
bot_name: str,
params: DailyParams = DailyParams(),
params: Optional[DailyParams] = None,
input_name: Optional[str] = None,
output_name: Optional[str] = None,
):
@@ -1256,9 +1256,11 @@ class DailyTransport(BaseTransport):
on_recording_stopped=self._on_recording_stopped,
on_recording_error=self._on_recording_error,
)
self._params = params
self._params = params or DailyParams()
self._client = DailyTransportClient(room_url, token, bot_name, params, callbacks, self.name)
self._client = DailyTransportClient(
room_url, token, bot_name, self._params, callbacks, self.name
)
self._input: Optional[DailyInputTransport] = None
self._output: Optional[DailyOutputTransport] = None

4
src/pipecat/transports/services/livekit.py
View File

@@ -499,7 +499,7 @@ class LiveKitTransport(BaseTransport):
url: str,
token: str,
room_name: str,
params: LiveKitParams = LiveKitParams(),
params: Optional[LiveKitParams] = None,
input_name: Optional[str] = None,
output_name: Optional[str] = None,
):
@@ -515,7 +515,7 @@ class LiveKitTransport(BaseTransport):
on_data_received=self._on_data_received,
on_first_participant_joined=self._on_first_participant_joined,
)
self._params = params
self._params = params or LiveKitParams()
self._client = LiveKitTransportClient(
url, token, room_name, self._params, callbacks, self.name

5
src/pipecat/utils/base_object.py
View File

@@ -59,6 +59,11 @@ class BaseObject(ABC):
self._event_handlers[event_name] = []
async def _call_event_handler(self, event_name: str, *args, **kwargs):
# If we haven't registered an event handler, we don't need to do
# anything.
if not self._event_handlers.get(event_name):
return
# Create the task.
task = asyncio.create_task(self._run_task(event_name, *args, **kwargs))

6
src/pipecat/utils/text/base_text_aggregator.py
View File

@@ -25,7 +25,7 @@ class BaseTextAggregator(ABC):
pass
@abstractmethod
def aggregate(self, text: str) -> Optional[str]:
async def aggregate(self, text: str) -> Optional[str]:
"""Aggregates the specified text with the currently accumulated text.
This method should be implemented to define how the new text contributes
@@ -43,7 +43,7 @@ class BaseTextAggregator(ABC):
pass
@abstractmethod
def handle_interruption(self):
async def handle_interruption(self):
"""Handles interruptions. When an interruption occurs it is possible
that we might want to discard the aggregated text or do some internal
modifications to the aggregated text.
@@ -52,6 +52,6 @@ class BaseTextAggregator(ABC):
pass
@abstractmethod
def reset(self):
async def reset(self):
"""Clears the internally aggregated text."""
pass

8
src/pipecat/utils/text/base_text_filter.py
View File

@@ -10,17 +10,17 @@ from typing import Any, Mapping
class BaseTextFilter(ABC):
@abstractmethod
def update_settings(self, settings: Mapping[str, Any]):
async def update_settings(self, settings: Mapping[str, Any]):
pass
@abstractmethod
def filter(self, text: str) -> str:
async def filter(self, text: str) -> str:
pass
@abstractmethod
def handle_interruption(self):
async def handle_interruption(self):
pass
@abstractmethod
def reset_interruption(self):
async def reset_interruption(self):
pass

15
src/pipecat/utils/text/markdown_text_filter.py
View File

@@ -26,19 +26,19 @@ class MarkdownTextFilter(BaseTextFilter):
filter_code: Optional[bool] = False
filter_tables: Optional[bool] = False
def __init__(self, params: InputParams = InputParams(), **kwargs):
def __init__(self, params: Optional[InputParams] = None, **kwargs):
super().__init__(**kwargs)
self._settings = params
self._settings = params or MarkdownTextFilter.InputParams()
self._in_code_block = False
self._in_table = False
self._interrupted = False
def update_settings(self, settings: Mapping[str, Any]):
async def update_settings(self, settings: Mapping[str, Any]):
for key, value in settings.items():
if hasattr(self._settings, key):
setattr(self._settings, key, value)
def filter(self, text: str) -> str:
async def filter(self, text: str) -> str:
if self._settings.enable_text_filter:
# Remove newlines and replace with a space only when there's no text before or after
filtered_text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE)
@@ -100,16 +100,19 @@ class MarkdownTextFilter(BaseTextFilter):
# Restore leading and trailing spaces
filtered_text = re.sub("§", " ", filtered_text)
## Make links more readable
filtered_text = re.sub(r"https?://", "", filtered_text)
return filtered_text
else:
return text
def handle_interruption(self):
async def handle_interruption(self):
self._interrupted = True
self._in_code_block = False
self._in_table = False
def reset_interruption(self):
async def reset_interruption(self):
self._interrupted = False
#

16
src/pipecat/utils/text/pattern_pair_aggregator.py
View File

@@ -5,7 +5,7 @@
#
import re
from typing import Callable, Optional, Tuple
from typing import Awaitable, Callable, Optional, Tuple
from loguru import logger
@@ -106,7 +106,7 @@ class PatternPairAggregator(BaseTextAggregator):
return self
def on_pattern_match(
self, pattern_id: str, handler: Callable[[PatternMatch], None]
self, pattern_id: str, handler: Callable[[PatternMatch], Awaitable[None]]
) -> "PatternPairAggregator":
"""Register a handler for when a pattern pair is matched.
@@ -124,7 +124,7 @@ class PatternPairAggregator(BaseTextAggregator):
self._handlers[pattern_id] = handler
return self
def _process_complete_patterns(self, text: str) -> Tuple[str, bool]:
async def _process_complete_patterns(self, text: str) -> Tuple[str, bool]:
"""Process all complete pattern pairs in the text.
Searches for all complete pattern pairs in the text, calls the
@@ -167,7 +167,7 @@ class PatternPairAggregator(BaseTextAggregator):
# Call the appropriate handler if registered
if pattern_id in self._handlers:
try:
self._handlers[pattern_id](pattern_match)
await self._handlers[pattern_id](pattern_match)
except Exception as e:
logger.error(f"Error in pattern handler for {pattern_id}: {e}")
@@ -204,7 +204,7 @@ class PatternPairAggregator(BaseTextAggregator):
return False
def aggregate(self, text: str) -> Optional[str]:
async def aggregate(self, text: str) -> Optional[str]:
"""Aggregate text and process pattern pairs.
This method adds the new text to the buffer, processes any complete pattern
@@ -223,7 +223,7 @@ class PatternPairAggregator(BaseTextAggregator):
self._text += text
# Process any complete patterns in the buffer
processed_text, modified = self._process_complete_patterns(self._text)
processed_text, modified = await self._process_complete_patterns(self._text)
# Only update the buffer if modifications were made
if modified:
@@ -245,7 +245,7 @@ class PatternPairAggregator(BaseTextAggregator):
# No complete sentence found yet
return None
def handle_interruption(self):
async def handle_interruption(self):
"""Handle interruptions by clearing the buffer.
Called when an interruption occurs in the processing pipeline,
@@ -253,7 +253,7 @@ class PatternPairAggregator(BaseTextAggregator):
"""
self._text = ""
def reset(self):
async def reset(self):
"""Clear the internally aggregated text.
Resets the aggregator to its initial state, discarding any

6
src/pipecat/utils/text/simple_text_aggregator.py
View File

@@ -23,7 +23,7 @@ class SimpleTextAggregator(BaseTextAggregator):
def text(self) -> str:
return self._text
def aggregate(self, text: str) -> Optional[str]:
async def aggregate(self, text: str) -> Optional[str]:
result: Optional[str] = None
self._text += text
@@ -35,8 +35,8 @@ class SimpleTextAggregator(BaseTextAggregator):
return result
def handle_interruption(self):
async def handle_interruption(self):
self._text = ""
def reset(self):
async def reset(self):
self._text = ""

6
src/pipecat/utils/text/skip_tags_aggregator.py
View File

@@ -43,7 +43,7 @@ class SkipTagsAggregator(BaseTextAggregator):
"""
return self._text
def aggregate(self, text: str) -> Optional[str]:
async def aggregate(self, text: str) -> Optional[str]:
"""Aggregate text and process pattern pairs.
This method adds the new text to the buffer, processes any complete pattern
@@ -77,7 +77,7 @@ class SkipTagsAggregator(BaseTextAggregator):
# No complete sentence found yet
return None
def handle_interruption(self):
async def handle_interruption(self):
"""Handle interruptions by clearing the buffer.
Called when an interruption occurs in the processing pipeline,
@@ -85,7 +85,7 @@ class SkipTagsAggregator(BaseTextAggregator):
"""
self._text = ""
def reset(self):
async def reset(self):
"""Clear the internally aggregated text.
Resets the aggregator to its initial state, discarding any

71
src/pipecat/utils/tracing/service_attributes.py
View File

@@ -18,6 +18,39 @@ if is_tracing_available():
from opentelemetry.trace import Span
def _get_gen_ai_system_from_service_name(service_name: str) -> str:
"""Extract the standardized gen_ai.system value from a service class name.
Source:
https://opentelemetry.io/docs/specs/semconv/attributes-registry/gen-ai/#gen-ai-system
Uses standard OTel names where possible, with special case mappings for
service names that don't follow the pattern.
"""
SPECIAL_CASE_MAPPINGS = {
# AWS
"AWSBedrockLLMService": "aws.bedrock",
# Azure
"AzureLLMService": "az.ai.openai",
# Google
"GoogleLLMService": "gcp.gemini",
"GoogleLLMOpenAIBetaService": "gcp.gemini",
"GoogleVertexLLMService": "gcp.vertex_ai",
# Others
"GrokLLMService": "xai",
}
if service_name in SPECIAL_CASE_MAPPINGS:
return SPECIAL_CASE_MAPPINGS[service_name]
if service_name.endswith("LLMService"):
provider = service_name[:-10].lower()
else:
provider = service_name.lower()
return provider
def add_tts_span_attributes(
span: "Span",
service_name: str,
@@ -45,17 +78,18 @@ def add_tts_span_attributes(
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("service.name", service_name)
span.set_attribute("model", model)
span.set_attribute("gen_ai.system", service_name.replace("TTSService", "").lower())
span.set_attribute("gen_ai.request.model", model)
span.set_attribute("gen_ai.operation.name", operation_name)
span.set_attribute("gen_ai.output.type", "speech")
span.set_attribute("voice_id", voice_id)
span.set_attribute("operation", operation_name)
# Add optional attributes
if text:
span.set_attribute("text", text)
if character_count is not None:
span.set_attribute("metrics.tts.character_count", character_count)
span.set_attribute("metrics.character_count", character_count)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
@@ -76,6 +110,7 @@ def add_stt_span_attributes(
span: "Span",
service_name: str,
model: str,
operation_name: str = "stt",
transcript: Optional[str] = None,
is_final: Optional[bool] = None,
language: Optional[str] = None,
@@ -90,6 +125,7 @@ def add_stt_span_attributes(
span: The span to add attributes to
service_name: Name of the STT service (e.g., "deepgram")
model: Model name/identifier
operation_name: Name of the operation (default: "stt")
transcript: The transcribed text
is_final: Whether this is a final transcript
language: Detected or configured language
@@ -99,8 +135,9 @@ def add_stt_span_attributes(
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("service.name", service_name)
span.set_attribute("model", model)
span.set_attribute("gen_ai.system", service_name.replace("STTService", "").lower())
span.set_attribute("gen_ai.request.model", model)
span.set_attribute("gen_ai.operation.name", operation_name)
span.set_attribute("vad_enabled", vad_enabled)
# Add optional attributes
@@ -161,13 +198,15 @@ def add_llm_span_attributes(
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("service.name", service_name)
span.set_attribute("model", model)
span.set_attribute("gen_ai.system", _get_gen_ai_system_from_service_name(service_name))
span.set_attribute("gen_ai.request.model", model)
span.set_attribute("gen_ai.operation.name", "chat")
span.set_attribute("gen_ai.output.type", "text")
span.set_attribute("stream", stream)
# Add optional attributes
if messages:
span.set_attribute("messages", messages)
span.set_attribute("input", messages)
if tools:
span.set_attribute("tools", tools)
@@ -188,7 +227,19 @@ def add_llm_span_attributes(
if parameters:
for key, value in parameters.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(f"param.{key}", value)
if key in [
"temperature",
"max_tokens",
"max_completion_tokens",
"top_p",
"top_k",
"frequency_penalty",
"presence_penalty",
"seed",
]:
span.set_attribute(f"gen_ai.request.{key}", value)
else:
span.set_attribute(f"param.{key}", value)
# Add extra parameters if provided
if extra_parameters:

30
src/pipecat/utils/tracing/service_decorators.py
View File

@@ -67,20 +67,6 @@ def _get_parent_service_context(self):
return context_api.get_current()
def _get_service_name(self, service_prefix: str) -> str:
"""Generate a default span name using service type and class name.
Args:
self: The service instance.
service_prefix: The service type (e.g., 'llm', 'stt', 'tts').
Returns:
A default span name string like "type_classname" (e.g. llm_openaillmservice).
"""
service_class_name = self.__class__.__name__.lower()
return f"{service_prefix}_{service_class_name}"
def _add_token_usage_to_span(span, token_usage):
"""Add token usage metrics to a span (internal use only).
@@ -93,13 +79,15 @@ def _add_token_usage_to_span(span, token_usage):
if isinstance(token_usage, dict):
if "prompt_tokens" in token_usage:
span.set_attribute("llm.prompt_tokens", token_usage["prompt_tokens"])
span.set_attribute("gen_ai.usage.input_tokens", token_usage["prompt_tokens"])
if "completion_tokens" in token_usage:
span.set_attribute("llm.completion_tokens", token_usage["completion_tokens"])
span.set_attribute("gen_ai.usage.output_tokens", token_usage["completion_tokens"])
else:
# Handle LLMTokenUsage object
span.set_attribute("llm.prompt_tokens", getattr(token_usage, "prompt_tokens", 0))
span.set_attribute("llm.completion_tokens", getattr(token_usage, "completion_tokens", 0))
span.set_attribute("gen_ai.usage.input_tokens", getattr(token_usage, "prompt_tokens", 0))
span.set_attribute(
"gen_ai.usage.output_tokens", getattr(token_usage, "completion_tokens", 0)
)
def traced_tts(func: Optional[Callable] = None, *, name: Optional[str] = None) -> Callable:
@@ -134,7 +122,7 @@ def traced_tts(func: Optional[Callable] = None, *, name: Optional[str] = None) -
return
service_class_name = self.__class__.__name__
span_name = name or _get_service_name(self, "tts")
span_name = "tts"
# Get parent context
turn_context = get_current_turn_context()
@@ -237,7 +225,7 @@ def traced_stt(func: Optional[Callable] = None, *, name: Optional[str] = None) -
return await f(self, transcript, is_final, language)
service_class_name = self.__class__.__name__
span_name = name or _get_service_name(self, "stt")
span_name = "stt"
# Get the turn context first, then fall back to service context
turn_context = get_current_turn_context()
@@ -313,7 +301,7 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) -
return await f(self, context, *args, **kwargs)
service_class_name = self.__class__.__name__
span_name = name or _get_service_name(self, "llm")
span_name = "llm"
# Get the parent context - turn context if available, otherwise service context
turn_context = get_current_turn_context()

10
src/pipecat/utils/tracing/turn_trace_observer.py
View File

@@ -34,8 +34,10 @@ class TurnTraceObserver(BaseObserver):
conversation span that encapsulates the entire session.
"""
def __init__(self, turn_tracker: TurnTrackingObserver, conversation_id: Optional[str] = None):
super().__init__()
def __init__(
self, turn_tracker: TurnTrackingObserver, conversation_id: Optional[str] = None, **kwargs
):
super().__init__(**kwargs)
self._turn_tracker = turn_tracker
self._current_span: Optional["Span"] = None
self._current_turn_number: int = 0
@@ -82,7 +84,7 @@ class TurnTraceObserver(BaseObserver):
self._conversation_id = conversation_id
# Create a new span for this conversation
self._conversation_span = self._tracer.start_span(f"conversation-{conversation_id}")
self._conversation_span = self._tracer.start_span("conversation")
# Set span attributes
self._conversation_span.set_attribute("conversation.id", conversation_id)
@@ -143,7 +145,7 @@ class TurnTraceObserver(BaseObserver):
parent_context = context_provider.get_current_conversation_context()
# Create a new span for this turn
self._current_span = self._tracer.start_span(f"turn-{turn_number}", context=parent_context)
self._current_span = self._tracer.start_span("turn", context=parent_context)
self._current_turn_number = turn_number
# Set span attributes