wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'pipecat-ai:main' into main

Browse Source
This commit is contained in:
Jin Kim
2025-05-18 16:48:24 +09:00
committed by GitHub
parent cf2f249f8a deb43df0a4
commit a6f7e7fc30
342 changed files with 29696 additions and 9401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/pipecat/adapters/services/anthropic_adapter.py
View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, Dict, List, Union
from typing import Any, Dict, List
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.schemas.function_schema import FunctionSchema

40
src/pipecat/adapters/services/aws_nova_sonic_adapter.py Normal file
View File

@@ -0,0 +1,40 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import json
from typing import Any, Dict, List
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
class AWSNovaSonicLLMAdapter(BaseLLMAdapter):
@staticmethod
def _to_aws_nova_sonic_function_format(function: FunctionSchema) -> Dict[str, Any]:
return {
"toolSpec": {
"name": function.name,
"description": function.description,
"inputSchema": {
"json": json.dumps(
{
"type": "object",
"properties": function.properties,
"required": function.required,
}
)
},
}
}
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Converts function schemas to AWS Nova Sonic function-calling format.
:return: AWS Nova Sonic formatted function call definition.
"""
functions_schema = tools_schema.standard_tools
return [self._to_aws_nova_sonic_function_format(func) for func in functions_schema]

38
src/pipecat/adapters/services/bedrock_adapter.py Normal file
View File

@@ -0,0 +1,38 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, Dict, List
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
class AWSBedrockLLMAdapter(BaseLLMAdapter):
@staticmethod
def _to_bedrock_function_format(function: FunctionSchema) -> Dict[str, Any]:
return {
"toolSpec": {
"name": function.name,
"description": function.description,
"inputSchema": {
"json": {
"type": "object",
"properties": function.properties,
"required": function.required,
},
},
}
}
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Converts function schemas to Bedrock's function-calling format.
:return: Bedrock formatted function call definition.
"""
functions_schema = tools_schema.standard_tools
return [self._to_bedrock_function_format(func) for func in functions_schema]

50
src/pipecat/audio/turn/smart_turn/http_smart_turn.py
View File

@@ -40,7 +40,7 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
async def _send_raw_request(self, data_bytes: bytes) -> Dict[str, Any]:
headers = {"Content-Type": "application/octet-stream"}
headers.update(self._headers)
logger.trace(f"Sending {len(data_bytes)} bytes as raw body to {self._url}...")
try:
timeout = aiohttp.ClientTimeout(total=self._params.stop_secs)
@@ -50,23 +50,30 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
logger.trace("\n--- Response ---")
logger.trace(f"Status Code: {response.status}")
if response.status == 200:
try:
json_data = await response.json()
logger.trace("Response JSON:")
logger.trace(json_data)
return json_data
except aiohttp.ContentTypeError:
# Non-JSON response
text = await response.text()
logger.trace("Response Content (non-JSON):")
logger.trace(text)
raise Exception(f"Non-JSON response: {text}")
else:
# Check if successful
if response.status != 200:
error_text = await response.text()
logger.trace("Response Content (Error):")
logger.trace(error_text)
response.raise_for_status()
if response.status == 500:
logger.warning(f"Smart turn service returned 500 error: {error_text}")
raise Exception(f"Server returned HTTP 500: {error_text}")
else:
response.raise_for_status()
# Process successful response
try:
json_data = await response.json()
logger.trace("Response JSON:")
logger.trace(json_data)
return json_data
except aiohttp.ContentTypeError:
# Non-JSON response
text = await response.text()
logger.trace("Response Content (non-JSON):")
logger.trace(text)
raise Exception(f"Non-JSON response: {text}")
except asyncio.TimeoutError:
logger.error(f"Request timed out after {self._params.stop_secs} seconds")
@@ -76,5 +83,14 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
raise Exception("Failed to send raw request to Daily Smart Turn.")
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
serialized_array = self._serialize_array(audio_array)
return await self._send_raw_request(serialized_array)
try:
serialized_array = self._serialize_array(audio_array)
return await self._send_raw_request(serialized_array)
except Exception as e:
logger.error(f"Smart turn prediction failed: {str(e)}")
# Return an incomplete prediction when a failure occurs
return {
"prediction": 0,
"probability": 0.0,
"metrics": {"inference_time": 0.0, "total_time": 0.0},
}

73
src/pipecat/audio/turn/smart_turn/local_smart_turn.py Normal file
View File

@@ -0,0 +1,73 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, Dict
import numpy as np
from loguru import logger
from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn
try:
import torch
from transformers import AutoFeatureExtractor, Wav2Vec2BertForSequenceClassification
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`."
)
raise Exception(f"Missing module: {e}")
class LocalSmartTurnAnalyzer(BaseSmartTurn):
def __init__(self, *, smart_turn_model_path: str, **kwargs):
super().__init__(**kwargs)
if not smart_turn_model_path:
# Define the path to the pretrained model on Hugging Face
smart_turn_model_path = "pipecat-ai/smart-turn"
logger.debug("Loading Local Smart Turn model...")
# Load the pretrained model for sequence classification
self._turn_model = Wav2Vec2BertForSequenceClassification.from_pretrained(
smart_turn_model_path
)
# Load the corresponding feature extractor for preprocessing audio
self._turn_processor = AutoFeatureExtractor.from_pretrained(smart_turn_model_path)
# Set device to GPU if available, else CPU
self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Move model to selected device and set it to evaluation mode
self._turn_model = self._turn_model.to(self._device)
self._turn_model.eval()
logger.debug("Loaded Local Smart Turn")
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
inputs = self._turn_processor(
audio_array,
sampling_rate=16000,
padding="max_length",
truncation=True,
max_length=800, # Maximum length as specified in training
return_attention_mask=True,
return_tensors="pt",
)
# Move input tensors to the same device as the model
inputs = {k: v.to(self._device) for k, v in inputs.items()}
# Disable gradient calculation for inference
with torch.no_grad():
outputs = self._turn_model(**inputs)
logits = outputs.logits
probabilities = torch.nn.functional.softmax(logits, dim=1)
completion_prob = probabilities[0, 1].item() # Probability of class 1 (Complete)
prediction = 1 if completion_prob > 0.5 else 0
return {
"prediction": prediction,
"probability": completion_prob,
}

63
src/pipecat/frames/frames.py
View File

@@ -7,7 +7,6 @@
from dataclasses import dataclass, field
from enum import Enum
from typing import (
TYPE_CHECKING,
Any,
Awaitable,
Callable,
@@ -20,16 +19,11 @@ from typing import (
)
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.clocks.base_clock import BaseClock
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
from pipecat.utils.asyncio import BaseTaskManager
from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
if TYPE_CHECKING:
from pipecat.observers.base_observer import BaseObserver
class KeypadEntry(str, Enum):
"""DTMF entries."""
@@ -60,12 +54,16 @@ class Frame:
name: str = field(init=False)
pts: Optional[int] = field(init=False)
metadata: Dict[str, Any] = field(init=False)
transport_source: Optional[str] = field(init=False)
transport_destination: Optional[str] = field(init=False)
def __post_init__(self):
self.id: int = obj_id()
self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self.pts: Optional[int] = None
self.metadata: Dict[str, Any] = {}
self.transport_source: Optional[str] = None
self.transport_destination: Optional[str] = None
def __str__(self):
return self.name
@@ -136,8 +134,9 @@ class ImageRawFrame:
@dataclass
class OutputAudioRawFrame(DataFrame, AudioRawFrame):
"""A chunk of audio. Will be played by the output transport if the
transport's microphone has been enabled.
"""A chunk of audio. Will be played by the output transport. If the
transport supports multiple audio destinations (e.g. multiple audio tracks) the
destination name can be specified.
"""
@@ -147,13 +146,14 @@ class OutputAudioRawFrame(DataFrame, AudioRawFrame):
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, size: {len(self.audio)}, frames: {self.num_frames}, sample_rate: {self.sample_rate}, channels: {self.num_channels})"
return f"{self.name}(pts: {pts}, destination: {self.transport_destination}, size: {len(self.audio)}, frames: {self.num_frames}, sample_rate: {self.sample_rate}, channels: {self.num_channels})"
@dataclass
class OutputImageRawFrame(DataFrame, ImageRawFrame):
"""An image that will be shown by the transport if the transport's camera is
enabled.
"""An image that will be shown by the transport. If the transport supports
multiple video destinations (e.g. multiple video tracks) the destination
name can be specified.
"""
@@ -176,7 +176,7 @@ class URLImageRawFrame(OutputImageRawFrame):
"""
url: Optional[str]
url: Optional[str] = None
def __str__(self):
pts = format_pts(self.pts)
@@ -441,14 +441,11 @@ class OutputDTMFFrame(DTMFFrame):
class StartFrame(SystemFrame):
"""This is the first frame that should be pushed down a pipeline."""
clock: BaseClock
task_manager: BaseTaskManager
audio_in_sample_rate: int = 16000
audio_out_sample_rate: int = 24000
allow_interruptions: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
observer: Optional["BaseObserver"] = None
report_only_initial_ttfb: bool = False
@@ -709,14 +706,19 @@ class UserImageRequestFrame(SystemFrame):
context: Optional[Any] = None
function_name: Optional[str] = None
tool_call_id: Optional[str] = None
video_source: Optional[str] = None
def __str__(self):
return f"{self.name}(user: {self.user_id}, function: {self.function_name}, request: {self.tool_call_id})"
return f"{self.name}(user: {self.user_id}, video_source: {self.video_source}, function: {self.function_name}, request: {self.tool_call_id})"
@dataclass
class InputAudioRawFrame(SystemFrame, AudioRawFrame):
"""A chunk of audio usually coming from an input transport."""
"""A chunk of audio usually coming from an input transport. If the transport
supports multiple audio sources (e.g. multiple audio tracks) the source name
will be specified.
"""
def __post_init__(self):
super().__post_init__()
@@ -724,35 +726,50 @@ class InputAudioRawFrame(SystemFrame, AudioRawFrame):
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, size: {len(self.audio)}, frames: {self.num_frames}, sample_rate: {self.sample_rate}, channels: {self.num_channels})"
return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {len(self.audio)}, frames: {self.num_frames}, sample_rate: {self.sample_rate}, channels: {self.num_channels})"
@dataclass
class InputImageRawFrame(SystemFrame, ImageRawFrame):
"""An image usually coming from an input transport."""
"""An image usually coming from an input transport. If the transport
supports multiple video sources (e.g. multiple video tracks) the source name
will be specified.
"""
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, size: {self.size}, format: {self.format})"
return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {self.size}, format: {self.format})"
@dataclass
class UserAudioRawFrame(InputAudioRawFrame):
"""A chunk of audio, usually coming from an input transport, associated to a user."""
user_id: str = ""
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {len(self.audio)}, frames: {self.num_frames}, sample_rate: {self.sample_rate}, channels: {self.num_channels})"
@dataclass
class UserImageRawFrame(InputImageRawFrame):
"""An image associated to a user."""
user_id: str
user_id: str = ""
request: Optional[UserImageRequestFrame] = None
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format}, request: {self.request})"
return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, request: {self.request})"
@dataclass
class VisionImageRawFrame(InputImageRawFrame):
"""An image with an associated text to ask for a description of it."""
text: Optional[str]
text: Optional[str] = None
def __str__(self):
pts = format_pts(self.pts)

56
src/pipecat/observers/base_observer.py
View File

@@ -5,9 +5,38 @@
#
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing_extensions import TYPE_CHECKING
from pipecat.frames.frames import Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
if TYPE_CHECKING:
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@dataclass
class FramePushed:
"""Represents an event where a frame is pushed from one processor to another
within the pipeline.
This data structure is typically used by observers to track the flow of
frames through the pipeline for logging, debugging, or analytics purposes.
Attributes:
source (FrameProcessor): The processor sending the frame.
destination (FrameProcessor): The processor receiving the frame.
frame (Frame): The frame being transferred.
direction (FrameDirection): The direction of the transfer (e.g., downstream or upstream).
timestamp (int): The time when the frame was pushed, based on the pipeline clock.
"""
source: "FrameProcessor"
destination: "FrameProcessor"
frame: Frame
direction: "FrameDirection"
timestamp: int
class BaseObserver(ABC):
@@ -19,26 +48,15 @@ class BaseObserver(ABC):
"""
@abstractmethod
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
"""Abstract method to handle the event when a frame is pushed from one
processor to another.
async def on_push_frame(self, data: FramePushed):
"""Handle the event when a frame is pushed from one processor to another.
This method should be implemented by subclasses to define specific
behavior (e.g., logging, monitoring, debugging) when a frame is
transferred through the pipeline.
Args:
src (FrameProcessor): The source frame processor that is sending the frame.
dst (FrameProcessor): The destination frame processor that will receive the frame.
frame (Frame): The frame being transferred between processors.
direction (FrameDirection): The direction of the frame transfer.
timestamp (int): The timestamp when the frame was pushed (based on the pipeline clock).
This method should be implemented by subclasses to define specific behavior
when a frame is pushed.
data (FramePushed): The event data containing details about the frame transfer.
"""
pass

222
src/pipecat/observers/loggers/debug_log_observer.py Normal file
View File

@@ -0,0 +1,222 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from dataclasses import fields, is_dataclass
from enum import Enum, auto
from typing import Dict, Optional, Set, Tuple, Type, Union
from loguru import logger
from pipecat.frames.frames import Frame
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.frame_processor import FrameDirection
class FrameEndpoint(Enum):
"""Specifies which endpoint (source or destination) to filter on."""
SOURCE = auto()
DESTINATION = auto()
class DebugLogObserver(BaseObserver):
"""Observer that logs frame activity with detailed content to the console.
Automatically extracts and formats data from any frame type, making it useful
for debugging pipeline behavior without needing frame-specific observers.
Args:
frame_types: Optional tuple of frame types to log, or a dict with frame type
filters. If None, logs all frame types.
exclude_fields: Optional set of field names to exclude from logging.
Examples:
Log all frames from all services:
```python
observers = DebugLogObserver()
```
Log specific frame types from any source/destination:
```python
from pipecat.frames.frames import TranscriptionFrame, InterimTranscriptionFrame
observers=[
DebugLogObserver(frame_types=(LLMTextFrame,TranscriptionFrame,)),
],
```
Log frames with specific source/destination filters:
```python
from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
from pipecat.transports.base_output_transport import BaseOutputTransport
from pipecat.services.stt_service import STTService
observers=[
DebugLogObserver(
frame_types={
# Only log StartInterruptionFrame when source is BaseOutputTransport
StartInterruptionFrame: (BaseOutputTransport, FrameEndpoint.SOURCE),
# Only log UserStartedSpeakingFrame when destination is STTService
UserStartedSpeakingFrame: (STTService, FrameEndpoint.DESTINATION),
# Log LLMTextFrame regardless of source or destination type
LLMTextFrame: None,
}
),
],
```
"""
def __init__(
self,
frame_types: Optional[
Union[Tuple[Type[Frame], ...], Dict[Type[Frame], Optional[Tuple[Type, FrameEndpoint]]]]
] = None,
exclude_fields: Optional[Set[str]] = None,
):
"""Initialize the debug log observer.
Args:
frame_types: Tuple of frame types to log, or a dict mapping frame types to
filter configurations. Filter configs can be:
- None to log all instances of the frame type
- A tuple of (service_type, endpoint) to filter on a specific service
and endpoint (SOURCE or DESTINATION)
If None is provided instead of a tuple/dict, log all frames.
exclude_fields: Set of field names to exclude from logging. If None, only binary
data fields are excluded.
"""
# Process frame filters
self.frame_filters = {}
if frame_types is not None:
if isinstance(frame_types, tuple):
# Tuple of frame types - log all instances
self.frame_filters = {frame_type: None for frame_type in frame_types}
else:
# Dict of frame types with filters
self.frame_filters = frame_types
# By default, exclude binary data fields that would clutter logs
self.exclude_fields = (
exclude_fields
if exclude_fields is not None
else {
"audio", # Skip binary audio data
"image", # Skip binary image data
"images", # Skip lists of images
}
)
def _format_value(self, value):
"""Format a value for logging.
Args:
value: The value to format.
Returns:
str: A string representation of the value suitable for logging.
"""
if value is None:
return "None"
elif isinstance(value, str):
return f"{value!r}"
elif isinstance(value, (list, tuple)):
if len(value) == 0:
return "[]"
if isinstance(value[0], dict) and len(value) > 3:
# For message lists, just show count
return f"{len(value)} items"
return str(value)
elif isinstance(value, (bytes, bytearray)):
return f"{len(value)} bytes"
elif hasattr(value, "get_messages_for_logging") and callable(
getattr(value, "get_messages_for_logging")
):
# Special case for OpenAI context
return f"{value.__class__.__name__} with messages: {value.get_messages_for_logging()}"
else:
return str(value)
def _should_log_frame(self, frame, src, dst):
"""Determine if a frame should be logged based on filters.
Args:
frame: The frame being processed
src: The source component
dst: The destination component
Returns:
bool: True if the frame should be logged, False otherwise
"""
# If no filters, log all frames
if not self.frame_filters:
return True
# Check if this frame type is in our filters
for frame_type, filter_config in self.frame_filters.items():
if isinstance(frame, frame_type):
# If filter is None, log all instances of this frame type
if filter_config is None:
return True
# Otherwise, check the specific filter
service_type, endpoint = filter_config
if endpoint == FrameEndpoint.SOURCE:
return isinstance(src, service_type)
elif endpoint == FrameEndpoint.DESTINATION:
return isinstance(dst, service_type)
return False
async def on_push_frame(self, data: FramePushed):
"""Process a frame being pushed into the pipeline.
Logs frame details to the console with all relevant fields and values.
Args:
data: Event data containing the frame, source, destination, direction, and timestamp.
"""
src = data.source
dst = data.destination
frame = data.frame
direction = data.direction
timestamp = data.timestamp
# Check if we should log this frame
if not self._should_log_frame(frame, src, dst):
return
# Format direction arrow
arrow = "→" if direction == FrameDirection.DOWNSTREAM else "←"
time_sec = timestamp / 1_000_000_000
class_name = frame.__class__.__name__
# Build frame representation
frame_details = []
# If dataclass, extract fields
if is_dataclass(frame):
for field in fields(frame):
if field.name in self.exclude_fields:
continue
value = getattr(frame, field.name)
if value is None:
continue
formatted_value = self._format_value(value)
frame_details.append(f"{field.name}: {formatted_value}")
# Format the message
if frame_details:
details = ", ".join(frame_details)
message = f"{class_name} {details} at {time_sec:.2f}s"
else:
message = f"{class_name} at {time_sec:.2f}s"
# Log the message
logger.debug(f"{src} {arrow} {dst}: {message}")

20
src/pipecat/observers/loggers/llm_log_observer.py
View File

@@ -7,7 +7,6 @@
from loguru import logger
from pipecat.frames.frames import (
Frame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
LLMFullResponseEndFrame,
@@ -15,9 +14,9 @@ from pipecat.frames.frames import (
LLMMessagesFrame,
LLMTextFrame,
)
from pipecat.observers.base_observer import BaseObserver
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
@@ -38,14 +37,13 @@ class LLMLogObserver(BaseObserver):
"""
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
async def on_push_frame(self, data: FramePushed):
src = data.source
dst = data.destination
frame = data.frame
direction = data.direction
timestamp = data.timestamp
if not isinstance(src, LLMService) and not isinstance(dst, LLMService):
return

17
src/pipecat/observers/loggers/transcription_log_observer.py
View File

@@ -7,12 +7,10 @@
from loguru import logger
from pipecat.frames.frames import (
Frame,
InterimTranscriptionFrame,
TranscriptionFrame,
)
from pipecat.observers.base_observer import BaseObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.services.stt_service import STTService
@@ -29,14 +27,11 @@ class TranscriptionLogObserver(BaseObserver):
"""
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
async def on_push_frame(self, data: FramePushed):
src = data.source
frame = data.frame
timestamp = data.timestamp
if not isinstance(src, STTService):
return

185
src/pipecat/observers/turn_tracking_observer.py Normal file
View File

@@ -0,0 +1,185 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from collections import deque
from loguru import logger
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
StartFrame,
UserStartedSpeakingFrame,
)
from pipecat.observers.base_observer import BaseObserver, FramePushed
class TurnTrackingObserver(BaseObserver):
"""Observer that tracks conversation turns in a pipeline.
Turn tracking logic:
- The first turn starts immediately when the pipeline starts (StartFrame)
- Subsequent turns start when the user starts speaking
- A turn ends when the bot stops speaking and either:
- The user starts speaking again
- A timeout period elapses with no more bot speech
"""
def __init__(self, max_frames=100, turn_end_timeout_secs=2.5, **kwargs):
super().__init__()
self._turn_count = 0
self._is_turn_active = False
self._is_bot_speaking = False
self._has_bot_spoken = False
self._turn_start_time = 0
self._turn_end_timeout_secs = turn_end_timeout_secs
self._end_turn_timer = None
# Track processed frames to avoid duplicates
self._processed_frames = set()
self._frame_history = deque(maxlen=max_frames)
self._register_event_handler("on_turn_started")
self._register_event_handler("on_turn_ended")
async def on_push_frame(self, data: FramePushed):
"""Process frame events for turn tracking."""
# Skip already processed frames
if data.frame.id in self._processed_frames:
return
self._processed_frames.add(data.frame.id)
self._frame_history.append(data.frame.id)
# If we've exceeded our history size, remove the oldest frame ID
# from the set of processed frames.
if len(self._processed_frames) > len(self._frame_history):
# Rebuild the set from the current deque contents
self._processed_frames = set(self._frame_history)
if isinstance(data.frame, StartFrame):
# Start the first turn immediately when the pipeline starts
if self._turn_count == 0:
await self._start_turn(data)
elif isinstance(data.frame, UserStartedSpeakingFrame):
await self._handle_user_started_speaking(data)
elif isinstance(data.frame, BotStartedSpeakingFrame):
await self._handle_bot_started_speaking(data)
# A BotStoppedSpeakingFrame can arrive after a UserStartedSpeakingFrame following an interruption
# We only want to end the turn if the bot was previously speaking
elif isinstance(data.frame, BotStoppedSpeakingFrame) and self._is_bot_speaking:
await self._handle_bot_stopped_speaking(data)
def _schedule_turn_end(self, data: FramePushed):
"""Schedule turn end with a timeout."""
# Cancel any existing timer
self._cancel_turn_end_timer()
# Create a new timer
loop = asyncio.get_event_loop()
self._end_turn_timer = loop.call_later(
self._turn_end_timeout_secs,
lambda: asyncio.create_task(self._end_turn_after_timeout(data)),
)
def _cancel_turn_end_timer(self):
"""Cancel the turn end timer if it exists."""
if self._end_turn_timer:
self._end_turn_timer.cancel()
self._end_turn_timer = None
async def _end_turn_after_timeout(self, data: FramePushed):
"""End turn after timeout has expired."""
if self._is_turn_active and not self._is_bot_speaking:
logger.trace(f"Turn {self._turn_count} ending due to timeout")
await self._end_turn(data, was_interrupted=False)
self._end_turn_timer = None
async def _handle_user_started_speaking(self, data: FramePushed):
"""Handle user speaking events, including interruptions."""
if self._is_bot_speaking:
# Handle interruption - end current turn and start a new one
self._cancel_turn_end_timer() # Cancel any pending end turn timer
await self._end_turn(data, was_interrupted=True)
self._is_bot_speaking = False # Bot is considered interrupted
await self._start_turn(data)
elif self._is_turn_active and self._has_bot_spoken:
# User started speaking during the turn_end_timeout_secs period after bot speech
self._cancel_turn_end_timer() # Cancel any pending end turn timer
await self._end_turn(data, was_interrupted=False)
await self._start_turn(data)
elif not self._is_turn_active:
# Start a new turn after previous one ended
await self._start_turn(data)
else:
# User is speaking within the same turn (before bot has responded)
logger.trace(f"User is already speaking in Turn {self._turn_count}")
async def _handle_bot_started_speaking(self, data: FramePushed):
"""Handle bot speaking events."""
self._is_bot_speaking = True
self._has_bot_spoken = True
# Cancel any pending turn end timer when bot starts speaking again
self._cancel_turn_end_timer()
async def _handle_bot_stopped_speaking(self, data: FramePushed):
"""Handle bot stopped speaking events."""
self._is_bot_speaking = False
# Schedule turn end with timeout
# This is needed to handle cases where the bot's speech ends and then resumes
# This can happen with HTTP TTS services or function calls
self._schedule_turn_end(data)
async def _start_turn(self, data: FramePushed):
"""Start a new turn."""
self._is_turn_active = True
self._has_bot_spoken = False
self._turn_count += 1
self._turn_start_time = data.timestamp
logger.trace(f"Turn {self._turn_count} started")
await self._call_event_handler("on_turn_started", self._turn_count)
async def _end_turn(self, data: FramePushed, was_interrupted: bool):
"""End the current turn."""
if not self._is_turn_active:
return
duration = (data.timestamp - self._turn_start_time) / 1_000_000_000 # Convert to seconds
self._is_turn_active = False
status = "interrupted" if was_interrupted else "completed"
logger.trace(f"Turn {self._turn_count} {status} after {duration:.2f}s")
await self._call_event_handler("on_turn_ended", self._turn_count, duration, was_interrupted)
def _register_event_handler(self, event_name):
"""Register an event handler."""
if not hasattr(self, "_event_handlers"):
self._event_handlers = {}
if event_name not in self._event_handlers:
self._event_handlers[event_name] = []
async def _call_event_handler(self, event_name, *args, **kwargs):
"""Call registered event handlers."""
if not hasattr(self, "_event_handlers"):
return
if event_name in self._event_handlers:
for handler in self._event_handlers[event_name]:
await handler(self, *args, **kwargs)
def event_handler(self, event_name):
"""Decorator for registering event handlers."""
def decorator(func):
if not hasattr(self, "_event_handlers"):
self._event_handlers = {}
if event_name not in self._event_handlers:
self._event_handlers[event_name] = []
self._event_handlers[event_name].append(func)
return func
return decorator

8
src/pipecat/pipeline/parallel_pipeline.py
View File

@@ -20,7 +20,7 @@ from pipecat.frames.frames import (
)
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
class ParallelPipelineSource(FrameProcessor):
@@ -118,6 +118,12 @@ class ParallelPipeline(BasePipeline):
# Frame processor
#
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await asyncio.gather(*[s.setup(setup) for s in self._sources])
await asyncio.gather(*[p.setup(setup) for p in self._pipelines])
await asyncio.gather(*[s.setup(setup) for s in self._sinks])
async def cleanup(self):
await super().cleanup()
await asyncio.gather(*[s.cleanup() for s in self._sources])

10
src/pipecat/pipeline/pipeline.py
View File

@@ -8,7 +8,7 @@ from typing import Callable, Coroutine, List
from pipecat.frames.frames import Frame
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
class PipelineSource(FrameProcessor):
@@ -70,6 +70,10 @@ class Pipeline(BasePipeline):
# Frame processor
#
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._setup_processors(setup)
async def cleanup(self):
await super().cleanup()
await self._cleanup_processors()
@@ -82,6 +86,10 @@ class Pipeline(BasePipeline):
elif direction == FrameDirection.UPSTREAM:
await self._sink.queue_frame(frame, FrameDirection.UPSTREAM)
async def _setup_processors(self, setup: FrameProcessorSetup):
for p in self._processors:
await p.setup(setup)
async def _cleanup_processors(self):
for p in self._processors:
await p.cleanup()

8
src/pipecat/pipeline/sync_parallel_pipeline.py
View File

@@ -14,7 +14,7 @@ from loguru import logger
from pipecat.frames.frames import ControlFrame, EndFrame, Frame, SystemFrame
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
@dataclass
@@ -103,6 +103,12 @@ class SyncParallelPipeline(BasePipeline):
# Frame processor
#
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await asyncio.gather(*[s["processor"].setup(setup) for s in self._sources])
await asyncio.gather(*[p.setup(setup) for p in self._pipelines])
await asyncio.gather(*[s["processor"].setup(setup) for s in self._sinks])
async def cleanup(self):
await super().cleanup()
await asyncio.gather(*[s["processor"].cleanup() for s in self._sources])

86
src/pipecat/pipeline/task.py
View File

@@ -30,11 +30,14 @@ from pipecat.frames.frames import (
)
from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
from pipecat.observers.base_observer import BaseObserver
from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.base_task import BaseTask
from pipecat.pipeline.task_observer import TaskObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
from pipecat.utils.asyncio import BaseTaskManager, TaskManager
from pipecat.utils.tracing.setup import is_tracing_available
from pipecat.utils.tracing.turn_trace_observer import TurnTraceObserver
HEARTBEAT_SECONDS = 1.0
HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5
@@ -157,6 +160,8 @@ class PipelineTask(BaseTask):
timeout if not received withing `idle_timeout_seconds`.
cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
the idle timeout is reached.
enable_turn_tracking: Whether to enable turn tracking.
enable_turn_tracing: Whether to enable turn tracing.
"""
@@ -175,6 +180,9 @@ class PipelineTask(BaseTask):
LLMFullResponseEndFrame,
),
cancel_on_idle_timeout: bool = True,
enable_turn_tracking: bool = True,
enable_tracing: bool = False,
conversation_id: Optional[str] = None,
):
super().__init__()
self._pipeline = pipeline
@@ -184,6 +192,9 @@ class PipelineTask(BaseTask):
self._idle_timeout_secs = idle_timeout_secs
self._idle_timeout_frames = idle_timeout_frames
self._cancel_on_idle_timeout = cancel_on_idle_timeout
self._enable_turn_tracking = enable_turn_tracking
self._enable_tracing = enable_tracing and is_tracing_available()
self._conversation_id = conversation_id
if self._params.observers:
import warnings
@@ -194,6 +205,14 @@ class PipelineTask(BaseTask):
DeprecationWarning,
)
observers = self._params.observers
if self._enable_turn_tracking:
self._turn_tracking_observer = TurnTrackingObserver()
observers = [self._turn_tracking_observer] + list(observers)
if self._enable_turn_tracking and self._enable_tracing:
self._turn_trace_observer = TurnTraceObserver(
self._turn_tracking_observer, conversation_id=self._conversation_id
)
observers = [self._turn_trace_observer] + list(observers)
self._finished = False
# This queue receives frames coming from the pipeline upstream.
@@ -251,6 +270,16 @@ class PipelineTask(BaseTask):
"""Returns the pipeline parameters of this task."""
return self._params
@property
def turn_tracking_observer(self) -> Optional[TurnTrackingObserver]:
"""Return the turn tracking observer if enabled."""
return getattr(self, "_turn_tracking_observer", None)
@property
def turn_trace_observer(self) -> Optional[TurnTraceObserver]:
"""Return the turn trace observer if enabled."""
return getattr(self, "_turn_trace_observer", None)
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
self._task_manager.set_event_loop(loop)
@@ -286,12 +315,7 @@ class PipelineTask(BaseTask):
async def cancel(self):
"""Stops the running pipeline immediately."""
logger.debug(f"Canceling pipeline task {self}")
# Make sure everything is cleaned up downstream. This is sent
# out-of-band from the main streaming task which is what we want since
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
# Only cancel the push task. Everything else will be cancelled in run().
await self._task_manager.cancel_task(self._process_push_task)
await self._cancel()
async def run(self):
"""Starts and manages the pipeline execution until completion or cancellation."""
@@ -299,8 +323,15 @@ class PipelineTask(BaseTask):
return
cleanup_pipeline = True
try:
# Setup processors.
await self._setup()
# Create all main tasks and wait of the main push task. This is the
# task that pushes frames to the very beginning of our pipeline (our
# controlled PipelineTaskSource processor).
push_task = await self._create_tasks()
await self._task_manager.wait_for_task(push_task)
# We have already cleaned up the pipeline inside the task.
cleanup_pipeline = False
except asyncio.CancelledError:
@@ -309,11 +340,17 @@ class PipelineTask(BaseTask):
# well, because you get a CancelledError in every place you are
# awaiting a task.
pass
await self._cancel_tasks()
await self._cleanup(cleanup_pipeline)
if self._check_dangling_tasks:
self._print_dangling_tasks()
self._finished = True
finally:
# It's possibe that we get an asyncio.CancelledError from the
# outside, if so we need to make sure everything gets cancelled
# properly.
if cleanup_pipeline:
await self._cancel()
await self._cancel_tasks()
await self._cleanup(cleanup_pipeline)
if self._check_dangling_tasks:
self._print_dangling_tasks()
self._finished = True
async def queue_frame(self, frame: Frame):
"""Queue a single frame to be pushed down the pipeline.
@@ -336,6 +373,14 @@ class PipelineTask(BaseTask):
for frame in frames:
await self.queue_frame(frame)
async def _cancel(self):
# Make sure everything is cleaned up downstream. This is sent
# out-of-band from the main streaming task which is what we want since
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
# Only cancel the push task. Everything else will be cancelled in run().
await self._task_manager.cancel_task(self._process_push_task)
async def _create_tasks(self):
self._process_up_task = self._task_manager.create_task(
self._process_up_queue(), f"{self}::_process_up_queue"
@@ -396,10 +441,24 @@ class PipelineTask(BaseTask):
await self._pipeline_end_event.wait()
self._pipeline_end_event.clear()
async def _setup(self):
setup = FrameProcessorSetup(
clock=self._clock,
task_manager=self._task_manager,
observer=self._observer,
)
await self._source.setup(setup)
await self._pipeline.setup(setup)
await self._sink.setup(setup)
async def _cleanup(self, cleanup_pipeline: bool):
# Cleanup base object.
await self.cleanup()
# End conversation tracing if it's active - this will also close any active turn span
if self._enable_tracing and hasattr(self, "_turn_trace_observer"):
self._turn_trace_observer.end_conversation_tracing()
# Cleanup pipeline processors.
await self._source.cleanup()
if cleanup_pipeline:
@@ -418,14 +477,11 @@ class PipelineTask(BaseTask):
self._maybe_start_idle_task()
start_frame = StartFrame(
clock=self._clock,
task_manager=self._task_manager,
allow_interruptions=self._params.allow_interruptions,
audio_in_sample_rate=self._params.audio_in_sample_rate,
audio_out_sample_rate=self._params.audio_out_sample_rate,
enable_metrics=self._params.enable_metrics,
enable_usage_metrics=self._params.enable_usage_metrics,
observer=self._observer,
report_only_initial_ttfb=self._params.report_only_initial_ttfb,
)
start_frame.metadata = self._params.start_metadata

58
src/pipecat/pipeline/task_observer.py
View File

@@ -5,13 +5,12 @@
#
import asyncio
import inspect
from typing import List
from attr import dataclass
from pipecat.frames.frames import Frame
from pipecat.observers.base_observer import BaseObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.utils.asyncio import BaseTaskManager
@@ -27,20 +26,6 @@ class Proxy:
observer: BaseObserver
@dataclass
class ObserverData:
"""This is the data we receive from the main observer and that we put into a
proxy queue for later processing.
"""
src: FrameProcessor
dst: FrameProcessor
frame: Frame
direction: FrameDirection
timestamp: int
class TaskObserver(BaseObserver):
"""This is a pipeline frame observer that is meant to be used as a proxy to
the user provided observers. That is, this is the observer that should be
@@ -68,20 +53,9 @@ class TaskObserver(BaseObserver):
for proxy in self._proxies:
await self._task_manager.cancel_task(proxy.task)
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
async def on_push_frame(self, data: FramePushed):
for proxy in self._proxies:
await proxy.queue.put(
ObserverData(
src=src, dst=dst, frame=frame, direction=direction, timestamp=timestamp
)
)
await proxy.queue.put(data)
def _create_proxies(self, observers) -> List[Proxy]:
proxies = []
@@ -96,8 +70,26 @@ class TaskObserver(BaseObserver):
return proxies
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):
warning_reported = False
while True:
data = await queue.get()
await observer.on_push_frame(
data.src, data.dst, data.frame, data.direction, data.timestamp
)
signature = inspect.signature(observer.on_push_frame)
if len(signature.parameters) > 1:
if not warning_reported:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Observer `on_push_frame(source, destination, frame, direction, timestamp)` is deprecated, us `on_push_frame(data: FramePushed)` instead.",
DeprecationWarning,
)
warning_reported = True
await observer.on_push_frame(
data.src, data.dst, data.frame, data.direction, data.timestamp
)
else:
await observer.on_push_frame(data)
queue.task_done()

0
src/pipecat/processors/audio/vad/__init__.py
View File

97
src/pipecat/processors/audio/vad/silero.py
View File

@@ -1,97 +0,0 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Optional
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams, VADState
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class SileroVAD(FrameProcessor):
def __init__(
self,
*,
sample_rate: Optional[int] = None,
vad_params: VADParams = VADParams(),
audio_passthrough: bool = False,
):
super().__init__()
self._vad_analyzer = SileroVADAnalyzer(sample_rate=sample_rate, params=vad_params)
self._audio_passthrough = audio_passthrough
self._processor_vad_state: VADState = VADState.QUIET
#
# FrameProcessor
#
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate)
if isinstance(frame, AudioRawFrame):
await self._analyze_audio(frame)
if self._audio_passthrough:
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
#
# Handle interruptions
#
async def _handle_interruptions(self, frame: Frame):
if self.interruptions_allowed:
# Make sure we notify about interruptions quickly out-of-band.
if isinstance(frame, UserStartedSpeakingFrame):
logger.debug("User started speaking")
await self._start_interruption()
# Push an out-of-band frame (i.e. not using the ordered push
# frame task) to stop everything, specially at the output
# transport.
await self.push_frame(StartInterruptionFrame())
elif isinstance(frame, UserStoppedSpeakingFrame):
logger.debug("User stopped speaking")
await self._stop_interruption()
await self.push_frame(StopInterruptionFrame())
await self.push_frame(frame)
async def _analyze_audio(self, frame: AudioRawFrame):
# Check VAD and push event if necessary. We just care about changes
# from QUIET to SPEAKING and vice versa.
new_vad_state = self._vad_analyzer.analyze_audio(frame.audio)
if (
new_vad_state != self._processor_vad_state
and new_vad_state != VADState.STARTING
and new_vad_state != VADState.STOPPING
):
new_frame = None
if new_vad_state == VADState.SPEAKING:
new_frame = UserStartedSpeakingFrame()
elif new_vad_state == VADState.QUIET:
new_frame = UserStoppedSpeakingFrame()
if new_frame:
await self._handle_interruptions(new_frame)
self._processor_vad_state = new_vad_state

4
src/pipecat/processors/filters/stt_mute_filter.py
View File

@@ -24,10 +24,12 @@ from pipecat.frames.frames import (
FunctionCallInProgressFrame,
FunctionCallResultFrame,
InputAudioRawFrame,
InterimTranscriptionFrame,
StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
STTMuteFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
@@ -175,6 +177,8 @@ class STTMuteFilter(FrameProcessor):
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
InputAudioRawFrame,
InterimTranscriptionFrame,
TranscriptionFrame,
),
):
# Only pass VAD-related frames when not muted

67
src/pipecat/processors/frame_processor.py
View File

@@ -5,6 +5,7 @@
#
import asyncio
from dataclasses import dataclass
from enum import Enum
from typing import Awaitable, Callable, Coroutine, Optional
@@ -21,6 +22,7 @@ from pipecat.frames.frames import (
SystemFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage, MetricsData
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
from pipecat.utils.asyncio import BaseTaskManager
from pipecat.utils.base_object import BaseObject
@@ -31,6 +33,13 @@ class FrameDirection(Enum):
UPSTREAM = 2
@dataclass
class FrameProcessorSetup:
clock: BaseClock
task_manager: BaseTaskManager
observer: Optional[BaseObserver] = None
class FrameProcessor(BaseObject):
def __init__(
self,
@@ -50,12 +59,17 @@ class FrameProcessor(BaseObject):
# Task Manager
self._task_manager: Optional[BaseTaskManager] = None
# Observer
self._observer: Optional[BaseObserver] = None
# Other properties
self._allow_interruptions = False
self._enable_metrics = False
self._enable_usage_metrics = False
self._report_only_initial_ttfb = False
self._observer = None
# Indicates whether we have received the StartFrame.
self.__started = False
# Cancellation is done through CancelFrame (a system frame). This could
# cause other events being triggered (e.g. closing a transport) which
@@ -166,6 +180,11 @@ class FrameProcessor(BaseObject):
raise Exception(f"{self} TaskManager is still not initialized.")
await self._task_manager.wait_for_task(task, timeout)
async def setup(self, setup: FrameProcessorSetup):
self._clock = setup.clock
self._task_manager = setup.task_manager
self._observer = setup.observer
async def cleanup(self):
await super().cleanup()
await self.__cancel_input_task()
@@ -226,13 +245,6 @@ class FrameProcessor(BaseObject):
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, StartFrame):
self._clock = frame.clock
self._task_manager = frame.task_manager
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
self._observer = frame.observer
await self.__start(frame)
elif isinstance(frame, StartInterruptionFrame):
await self._start_interruption()
@@ -246,7 +258,7 @@ class FrameProcessor(BaseObject):
await self.push_frame(error, FrameDirection.UPSTREAM)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
if not self._check_ready(frame):
if not self._check_started(frame):
return
if isinstance(frame, SystemFrame):
@@ -255,6 +267,11 @@ class FrameProcessor(BaseObject):
await self.__push_queue.put((frame, direction))
async def __start(self, frame: StartFrame):
self.__started = True
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
self.__create_input_task()
self.__create_push_task()
@@ -294,32 +311,38 @@ class FrameProcessor(BaseObject):
timestamp = self._clock.get_time() if self._clock else 0
if direction == FrameDirection.DOWNSTREAM and self._next:
logger.trace(f"Pushing {frame} from {self} to {self._next}")
if self._observer:
await self._observer.on_push_frame(
self, self._next, frame, direction, timestamp
data = FramePushed(
source=self,
destination=self._next,
frame=frame,
direction=direction,
timestamp=timestamp,
)
await self._observer.on_push_frame(data)
await self._next.queue_frame(frame, direction)
elif direction == FrameDirection.UPSTREAM and self._prev:
logger.trace(f"Pushing {frame} upstream from {self} to {self._prev}")
if self._observer:
await self._observer.on_push_frame(
self, self._prev, frame, direction, timestamp
data = FramePushed(
source=self,
destination=self._prev,
frame=frame,
direction=direction,
timestamp=timestamp,
)
await self._observer.on_push_frame(data)
await self._prev.queue_frame(frame, direction)
except Exception as e:
logger.exception(f"Uncaught exception in {self}: {e}")
await self.push_error(ErrorFrame(str(e)))
raise
def _check_ready(self, frame: Frame):
# If we are trying to push a frame but we still have no clock, it means
# we didn't process a StartFrame.
if not self._clock:
logger.error(
f"{self} not properly initialized, missing super().process_frame(frame, direction)?"
)
return False
return True
def _check_started(self, frame: Frame):
if not self.__started:
logger.error(f"{self} Trying to process {frame} but StartFrame not received yet")
return self.__started
def __create_input_task(self):
if not self.__input_frame_task:

124
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -55,12 +55,15 @@ from pipecat.metrics.metrics import (
TTFBMetricsData,
TTSUsageMetricsData,
)
from pipecat.observers.base_observer import BaseObserver
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.llm_service import (
FunctionCallParams, # TODO(aleix): we shouldn't import `services` from `processors`
)
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport
@@ -251,7 +254,7 @@ class RTVIBotReady(BaseModel):
class RTVILLMFunctionCallMessageData(BaseModel):
function_name: str
tool_call_id: str
arguments: Mapping[str, Any]
args: Mapping[str, Any]
class RTVILLMFunctionCallMessage(BaseModel):
@@ -392,6 +395,32 @@ class RTVIServerMessageFrame(SystemFrame):
return f"{self.name}(data: {self.data})"
@dataclass
class RTVIObserverParams:
"""
Parameters for configuring RTVI Observer behavior.
Attributes:
bot_llm_enabled (bool): Indicates if the bot's LLM messages should be sent.
bot_tts_enabled (bool): Indicates if the bot's TTS messages should be sent.
bot_speaking_enabled (bool): Indicates if the bot's started/stopped speaking messages should be sent.
user_llm_enabled (bool): Indicates if the user's LLM input messages should be sent.
user_speaking_enabled (bool): Indicates if the user's started/stopped speaking messages should be sent.
user_transcription_enabled (bool): Indicates if user's transcription messages should be sent.
metrics_enabled (bool): Indicates if metrics messages should be sent.
errors_enabled (bool): Indicates if errors messages should be sent.
"""
bot_llm_enabled: bool = True
bot_tts_enabled: bool = True
bot_speaking_enabled: bool = True
user_llm_enabled: bool = True
user_speaking_enabled: bool = True
user_transcription_enabled: bool = True
metrics_enabled: bool = True
errors_enabled: bool = True
class RTVIObserver(BaseObserver):
"""Pipeline frame observer for RTVI server message handling.
@@ -404,23 +433,19 @@ class RTVIObserver(BaseObserver):
are handled by the RTVIProcessor.
Args:
rtvi (FrameProcessor): The RTVI processor to push frames to.
rtvi (RTVIProcessor): The RTVI processor to push frames to.
params (RTVIObserverParams): Settings to enable/disable specific messages.
"""
def __init__(self, rtvi: FrameProcessor):
def __init__(self, rtvi: "RTVIProcessor", *, params: RTVIObserverParams = RTVIObserverParams()):
super().__init__()
self._rtvi = rtvi
self._params = params
self._bot_transcription = ""
self._frames_seen = set()
rtvi.set_errors_enabled(self._params.errors_enabled)
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
async def on_push_frame(self, data: FramePushed):
"""Process a frame being pushed through the pipeline.
Args:
@@ -430,6 +455,10 @@ class RTVIObserver(BaseObserver):
direction: Direction of frame flow in pipeline
timestamp: Time when frame was pushed
"""
src = data.source
frame = data.frame
direction = data.direction
# If we have already seen this frame, let's skip it.
if frame.id in self._frames_seen:
return
@@ -438,35 +467,41 @@ class RTVIObserver(BaseObserver):
# again the next time we see the frame.
mark_as_seen = True
if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)):
if (
isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame))
and self._params.user_speaking_enabled
):
await self._handle_interruptions(frame)
elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)) and (
direction == FrameDirection.UPSTREAM
elif (
isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame))
and (direction == FrameDirection.UPSTREAM)
and self._params.bot_speaking_enabled
):
await self._handle_bot_speaking(frame)
elif isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)):
elif (
isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame))
and self._params.user_transcription_enabled
):
await self._handle_user_transcriptions(frame)
elif isinstance(frame, OpenAILLMContextFrame):
elif isinstance(frame, OpenAILLMContextFrame) and self._params.user_llm_enabled:
await self._handle_context(frame)
elif isinstance(frame, UserStartedSpeakingFrame):
await self._push_bot_transcription()
elif isinstance(frame, LLMFullResponseStartFrame):
elif isinstance(frame, LLMFullResponseStartFrame) and self._params.bot_llm_enabled:
await self.push_transport_message_urgent(RTVIBotLLMStartedMessage())
elif isinstance(frame, LLMFullResponseEndFrame):
elif isinstance(frame, LLMFullResponseEndFrame) and self._params.bot_llm_enabled:
await self.push_transport_message_urgent(RTVIBotLLMStoppedMessage())
elif isinstance(frame, LLMTextFrame):
elif isinstance(frame, LLMTextFrame) and self._params.bot_llm_enabled:
await self._handle_llm_text_frame(frame)
elif isinstance(frame, TTSStartedFrame):
elif isinstance(frame, TTSStartedFrame) and self._params.bot_tts_enabled:
await self.push_transport_message_urgent(RTVIBotTTSStartedMessage())
elif isinstance(frame, TTSStoppedFrame):
elif isinstance(frame, TTSStoppedFrame) and self._params.bot_tts_enabled:
await self.push_transport_message_urgent(RTVIBotTTSStoppedMessage())
elif isinstance(frame, TTSTextFrame):
elif isinstance(frame, TTSTextFrame) and self._params.bot_tts_enabled:
if isinstance(src, BaseOutputTransport):
message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text))
await self.push_transport_message_urgent(message)
else:
mark_as_seen = False
elif isinstance(frame, MetricsFrame):
elif isinstance(frame, MetricsFrame) and self._params.metrics_enabled:
await self._handle_metrics(frame)
elif isinstance(frame, RTVIServerMessageFrame):
message = RTVIServerMessage(data=frame.data)
@@ -604,11 +639,10 @@ class RTVIProcessor(FrameProcessor):
super().__init__(**kwargs)
self._config = config
self._pipeline: Optional[FrameProcessor] = None
self._bot_ready = False
self._client_ready = False
self._client_ready_id = ""
self._errors_enabled = True
self._registered_actions: Dict[str, RTVIAction] = {}
self._registered_services: Dict[str, RTVIService] = {}
@@ -648,26 +682,23 @@ class RTVIProcessor(FrameProcessor):
await self._update_config(self._config, False)
await self._send_bot_ready()
def set_errors_enabled(self, enabled: bool):
self._errors_enabled = enabled
async def interrupt_bot(self):
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
async def send_error(self, error: str):
message = RTVIError(data=RTVIErrorData(error=error, fatal=False))
await self._push_transport_message(message)
await self._send_error_frame(ErrorFrame(error=error))
async def handle_message(self, message: RTVIMessage):
await self._message_queue.put(message)
async def handle_function_call(
self,
function_name: str,
tool_call_id: str,
arguments: Mapping[str, Any],
):
async def handle_function_call(self, params: FunctionCallParams):
fn = RTVILLMFunctionCallMessageData(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
function_name=params.function_name,
tool_call_id=params.tool_call_id,
args=params.arguments,
)
message = RTVILLMFunctionCallMessage(data=fn)
await self._push_transport_message(message, exclude_none=False)
@@ -721,11 +752,6 @@ class RTVIProcessor(FrameProcessor):
else:
await self.push_frame(frame, direction)
async def cleanup(self):
await super().cleanup()
if self._pipeline:
await self._pipeline.cleanup()
async def _start(self, frame: StartFrame):
if not self._action_task:
self._action_task = self.create_task(self._action_task_handler())
@@ -917,12 +943,14 @@ class RTVIProcessor(FrameProcessor):
await self._push_transport_message(message)
async def _send_error_frame(self, frame: ErrorFrame):
message = RTVIError(data=RTVIErrorData(error=frame.error, fatal=frame.fatal))
await self._push_transport_message(message)
if self._errors_enabled:
message = RTVIError(data=RTVIErrorData(error=frame.error, fatal=frame.fatal))
await self._push_transport_message(message)
async def _send_error_response(self, id: str, error: str):
message = RTVIErrorResponse(id=id, data=RTVIErrorResponseData(error=error))
await self._push_transport_message(message)
if self._errors_enabled:
message = RTVIErrorResponse(id=id, data=RTVIErrorResponseData(error=error))
await self._push_transport_message(message)
def _action_id(self, service: str, action: str) -> str:
return f"{service}:{action}"

25
src/pipecat/processors/metrics/frame_processor_metrics.py
View File

@@ -5,6 +5,7 @@
#
import time
from typing import Optional
from loguru import logger
@@ -23,8 +24,25 @@ class FrameProcessorMetrics:
def __init__(self):
self._start_ttfb_time = 0
self._start_processing_time = 0
self._last_ttfb_time = 0
self._should_report_ttfb = True
@property
def ttfb_ms(self) -> Optional[float]:
"""Get the current TTFB value in seconds.
Returns:
Optional[float]: The TTFB value in seconds, or None if not measured
"""
if self._last_ttfb_time > 0:
return self._last_ttfb_time
# If TTFB is in progress, calculate current value
if self._start_ttfb_time > 0:
return time.time() - self._start_ttfb_time
return None
def _processor_name(self):
return self._core_metrics_data.processor
@@ -40,16 +58,17 @@ class FrameProcessorMetrics:
async def start_ttfb_metrics(self, report_only_initial_ttfb):
if self._should_report_ttfb:
self._start_ttfb_time = time.time()
self._last_ttfb_time = 0
self._should_report_ttfb = not report_only_initial_ttfb
async def stop_ttfb_metrics(self):
if self._start_ttfb_time == 0:
return None
value = time.time() - self._start_ttfb_time
logger.debug(f"{self._processor_name()} TTFB: {value}")
self._last_ttfb_time = time.time() - self._start_ttfb_time
logger.debug(f"{self._processor_name()} TTFB: {self._last_ttfb_time}")
ttfb = TTFBMetricsData(
processor=self._processor_name(), value=value, model=self._model_name()
processor=self._processor_name(), value=self._last_ttfb_time, model=self._model_name()
)
self._start_ttfb_time = 0
return MetricsFrame(data=[ttfb])

64
src/pipecat/processors/transcript_processor.py
View File

@@ -93,49 +93,55 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
"""Aggregates and emits text fragments as a transcript message.
This method uses a heuristic to automatically detect whether text fragments
use pre-spacing (spaces at the beginning of fragments) or not, and applies
the appropriate joining strategy. It handles fragments from different TTS
services with different formatting patterns.
contain embedded spacing (spaces at the beginning or end of fragments) or not,
and applies the appropriate joining strategy. It handles fragments from different
TTS services with different formatting patterns.
Examples:
Pre-spaced fragments (concatenated):
Fragments with embedded spacing (concatenated):
```
TTSTextFrame: ["Hello"]
TTSTextFrame: [" there"]
TTSTextFrame: [" there"] # Leading space
TTSTextFrame: ["!"]
TTSTextFrame: [" How"]
TTSTextFrame: [" How"] # Leading space
TTSTextFrame: ["'s"]
TTSTextFrame: [" it"]
TTSTextFrame: [" going"]
TTSTextFrame: ["?"]
TTSTextFrame: [" it"] # Leading space
```
Result: "Hello there! How's it going?"
Result: "Hello there! How's it"
Word-by-word fragments (joined with spaces):
Fragments with trailing spaces (concatenated):
```
TTSTextFrame: ["Hel"]
TTSTextFrame: ["lo "] # Trailing space
TTSTextFrame: ["to "] # Trailing space
TTSTextFrame: ["you"]
```
Result: "Hello to you"
Word-by-word fragments without spacing (joined with spaces):
```
TTSTextFrame: ["Hello"]
TTSTextFrame: ["there!"]
TTSTextFrame: ["How"]
TTSTextFrame: ["is"]
TTSTextFrame: ["it"]
TTSTextFrame: ["going?"]
TTSTextFrame: ["there"]
TTSTextFrame: ["how"]
TTSTextFrame: ["are"]
TTSTextFrame: ["you"]
```
Result: "Hello there! How is it going?"
Result: "Hello there how are you"
"""
if self._current_text_parts and self._aggregation_start_time:
# Heuristic to detect pre-spaced fragments
uses_prespacing = False
if len(self._current_text_parts) > 1:
# Check if any fragment after the first one starts with whitespace
has_spaced_parts = any(
part and part[0].isspace() for part in self._current_text_parts[1:]
)
if has_spaced_parts:
uses_prespacing = True
has_leading_spaces = any(
part and part[0].isspace() for part in self._current_text_parts[1:]
)
has_trailing_spaces = any(
part and part[-1].isspace() for part in self._current_text_parts[:-1]
)
# Apply appropriate joining method
if uses_prespacing:
# Pre-spaced fragments - just concatenate
# If there are embedded spaces in the fragments, use direct concatenation
contains_spacing_between_fragments = has_leading_spaces or has_trailing_spaces
# Apply corresponding joining method
if contains_spacing_between_fragments:
# Fragments already have spacing - just concatenate
content = "".join(self._current_text_parts)
else:
# Word-by-word fragments - join with spaces

2
src/pipecat/services/ai_services.py
View File

@@ -18,5 +18,5 @@ from .vision_service import *
sys.modules[__name__] = DeprecatedModuleProxy(
globals(),
"ai_services",
"ai_service.[image_service,llm_service,stt_service,tts_service,vision_service]",
"[ai_service,image_service,llm_service,stt_service,tts_service,vision_service]",
)

26
src/pipecat/services/anthropic/llm.py
View File

@@ -46,6 +46,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.utils.tracing.service_decorators import traced_llm
try:
from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven
@@ -147,6 +148,7 @@ class AnthropicLLMService(LLMService):
assistant = AnthropicAssistantContextAggregator(context, params=assistant_params)
return AnthropicContextAggregatorPair(_user=user, _assistant=assistant)
@traced_llm
async def _process_context(self, context: OpenAILLMContext):
# Usage tracking. We track the usage reported by Anthropic in prompt_tokens and
# completion_tokens. We also estimate the completion tokens from output text
@@ -250,14 +252,24 @@ class AnthropicLLMService(LLMService):
if hasattr(event.message.usage, "output_tokens")
else 0
)
if hasattr(event.message.usage, "cache_creation_input_tokens"):
cache_creation_input_tokens += (
event.message.usage.cache_creation_input_tokens
cache_creation_input_tokens += (
event.message.usage.cache_creation_input_tokens
if (
hasattr(event.message.usage, "cache_creation_input_tokens")
and event.message.usage.cache_creation_input_tokens is not None
)
logger.debug(f"Cache creation input tokens: {cache_creation_input_tokens}")
if hasattr(event.message.usage, "cache_read_input_tokens"):
cache_read_input_tokens += event.message.usage.cache_read_input_tokens
logger.debug(f"Cache read input tokens: {cache_read_input_tokens}")
else 0
)
logger.debug(f"Cache creation input tokens: {cache_creation_input_tokens}")
cache_read_input_tokens += (
event.message.usage.cache_read_input_tokens
if (
hasattr(event.message.usage, "cache_read_input_tokens")
and event.message.usage.cache_read_input_tokens is not None
)
else 0
)
logger.debug(f"Cache read input tokens: {cache_read_input_tokens}")
total_input_tokens = (
prompt_tokens + cache_creation_input_tokens + cache_read_input_tokens
)

32
src/pipecat/services/assemblyai/stt.py
View File

@@ -21,6 +21,7 @@ from pipecat.frames.frames import (
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
import assemblyai as aai
@@ -51,6 +52,9 @@ class AssemblyAISTTService(STTService):
"language": language,
}
def can_generate_metrics(self) -> bool:
return True
async def set_language(self, language: Language):
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
@@ -77,18 +81,25 @@ class AssemblyAISTTService(STTService):
:yield: None (transcription frames are pushed via self.push_frame in callbacks)
"""
if self._transcriber:
await self.start_ttfb_metrics()
await self.start_processing_metrics()
self._transcriber.stream(audio)
await self.stop_processing_metrics()
yield None
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
async def _connect(self):
"""Establish a connection to the AssemblyAI real-time transcription service.
This method sets up the necessary callback functions and initializes the
AssemblyAI transcriber.
"""
if self._transcriber:
return
@@ -107,15 +118,18 @@ class AssemblyAISTTService(STTService):
return
timestamp = time_now_iso8601()
is_final = isinstance(transcript, aai.RealtimeFinalTranscript)
language = self._settings["language"]
if isinstance(transcript, aai.RealtimeFinalTranscript):
frame = TranscriptionFrame(
transcript.text, "", timestamp, self._settings["language"]
)
if is_final:
frame = TranscriptionFrame(transcript.text, "", timestamp, language)
else:
frame = InterimTranscriptionFrame(
transcript.text, "", timestamp, self._settings["language"]
)
frame = InterimTranscriptionFrame(transcript.text, "", timestamp, language)
asyncio.run_coroutine_threadsafe(
self._handle_transcription(transcript.text, is_final, language),
self.get_event_loop(),
)
# Schedule the coroutine to run in the main event loop
# This is necessary because this callback runs in a different thread

4
src/pipecat/services/aws/__init__.py
View File

@@ -8,6 +8,8 @@ import sys
from pipecat.services import DeprecatedModuleProxy
from .llm import *
from .stt import *
from .tts import *
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "aws", "aws.tts")
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "aws", "aws.[llm,stt,tts]")

787
src/pipecat/services/aws/llm.py Normal file
View File

@@ -0,0 +1,787 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import base64
import copy
import io
import json
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from loguru import logger
from PIL import Image
from pydantic import BaseModel, Field
from pipecat.adapters.services.bedrock_adapter import AWSBedrockLLMAdapter
from pipecat.frames.frames import (
Frame,
FunctionCallCancelFrame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
UserImageRawFrame,
VisionImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMAssistantContextAggregator,
LLMUserAggregatorParams,
LLMUserContextAggregator,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.utils.tracing.service_decorators import traced_llm
try:
import boto3
import httpx
from botocore.config import Config
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use AWS services, you need to `pip install pipecat-ai[aws]`. Also, remember to set `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, and `AWS_REGION` environment variable."
)
raise Exception(f"Missing module: {e}")
@dataclass
class AWSBedrockContextAggregatorPair:
_user: "AWSBedrockUserContextAggregator"
_assistant: "AWSBedrockAssistantContextAggregator"
def user(self) -> "AWSBedrockUserContextAggregator":
return self._user
def assistant(self) -> "AWSBedrockAssistantContextAggregator":
return self._assistant
class AWSBedrockLLMContext(OpenAILLMContext):
def __init__(
self,
messages: Optional[List[dict]] = None,
tools: Optional[List[dict]] = None,
tool_choice: Optional[dict] = None,
*,
system: Optional[str] = None,
):
super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
self.system = system
@staticmethod
def upgrade_to_bedrock(obj: OpenAILLMContext) -> "AWSBedrockLLMContext":
logger.debug(f"Upgrading to AWS Bedrock: {obj}")
if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSBedrockLLMContext):
obj.__class__ = AWSBedrockLLMContext
obj._restructure_from_openai_messages()
else:
obj._restructure_from_bedrock_messages()
return obj
@classmethod
def from_openai_context(cls, openai_context: OpenAILLMContext):
self = cls(
messages=openai_context.messages,
tools=openai_context.tools,
tool_choice=openai_context.tool_choice,
)
self.set_llm_adapter(openai_context.get_llm_adapter())
self._restructure_from_openai_messages()
return self
@classmethod
def from_messages(cls, messages: List[dict]) -> "AWSBedrockLLMContext":
self = cls(messages=messages)
self._restructure_from_openai_messages()
return self
@classmethod
def from_image_frame(cls, frame: VisionImageRawFrame) -> "AWSBedrockLLMContext":
context = cls()
context.add_image_frame_message(
format=frame.format, size=frame.size, image=frame.image, text=frame.text
)
return context
def set_messages(self, messages: List):
self._messages[:] = messages
self._restructure_from_openai_messages()
# convert a message in AWS Bedrock format into one or more messages in OpenAI format
def to_standard_messages(self, obj):
"""Convert AWS Bedrock message format to standard structured format.
Handles text content and function calls for both user and assistant messages.
Args:
obj: Message in AWS Bedrock format:
{
"role": "user/assistant",
"content": [{"text": str} | {"toolUse": {...}} | {"toolResult": {...}}]
}
Returns:
List of messages in standard format:
[
{
"role": "user/assistant/tool",
"content": [{"type": "text", "text": str}]
}
]
"""
role = obj.get("role")
content = obj.get("content")
if role == "assistant":
if isinstance(content, str):
return [{"role": role, "content": [{"type": "text", "text": content}]}]
elif isinstance(content, list):
text_items = []
tool_items = []
for item in content:
if "text" in item:
text_items.append({"type": "text", "text": item["text"]})
elif "toolUse" in item:
tool_use = item["toolUse"]
tool_items.append(
{
"type": "function",
"id": tool_use["toolUseId"],
"function": {
"name": tool_use["name"],
"arguments": json.dumps(tool_use["input"]),
},
}
)
messages = []
if text_items:
messages.append({"role": role, "content": text_items})
if tool_items:
messages.append({"role": role, "tool_calls": tool_items})
return messages
elif role == "user":
if isinstance(content, str):
return [{"role": role, "content": [{"type": "text", "text": content}]}]
elif isinstance(content, list):
text_items = []
tool_items = []
for item in content:
if "text" in item:
text_items.append({"type": "text", "text": item["text"]})
elif "toolResult" in item:
tool_result = item["toolResult"]
# Extract content from toolResult
result_content = ""
if isinstance(tool_result["content"], list):
for content_item in tool_result["content"]:
if "text" in content_item:
result_content = content_item["text"]
elif "json" in content_item:
result_content = json.dumps(content_item["json"])
else:
result_content = tool_result["content"]
tool_items.append(
{
"role": "tool",
"tool_call_id": tool_result["toolUseId"],
"content": result_content,
}
)
messages = []
if text_items:
messages.append({"role": role, "content": text_items})
messages.extend(tool_items)
return messages
def from_standard_message(self, message):
"""Convert standard format message to AWS Bedrock format.
Handles conversion of text content, tool calls, and tool results.
Empty text content is converted to "(empty)".
Args:
message: Message in standard format:
{
"role": "user/assistant/tool",
"content": str | [{"type": "text", ...}],
"tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}]
}
Returns:
Message in AWS Bedrock format:
{
"role": "user/assistant",
"content": [
{"text": str} |
{"toolUse": {"toolUseId": str, "name": str, "input": dict}} |
{"toolResult": {"toolUseId": str, "content": [...], "status": str}}
]
}
"""
if message["role"] == "tool":
# Try to parse the content as JSON if it looks like JSON
try:
if message["content"].strip().startswith("{") and message[
"content"
].strip().endswith("}"):
content_json = json.loads(message["content"])
tool_result_content = [{"json": content_json}]
else:
tool_result_content = [{"text": message["content"]}]
except:
tool_result_content = [{"text": message["content"]}]
return {
"role": "user",
"content": [
{
"toolResult": {
"toolUseId": message["tool_call_id"],
"content": tool_result_content,
},
},
],
}
if message.get("tool_calls"):
tc = message["tool_calls"]
ret = {"role": "assistant", "content": []}
for tool_call in tc:
function = tool_call["function"]
arguments = json.loads(function["arguments"])
new_tool_use = {
"toolUse": {
"toolUseId": tool_call["id"],
"name": function["name"],
"input": arguments,
}
}
ret["content"].append(new_tool_use)
return ret
# Handle text content
content = message.get("content")
if isinstance(content, str):
if content == "":
return {"role": message["role"], "content": [{"text": "(empty)"}]}
else:
return {"role": message["role"], "content": [{"text": content}]}
elif isinstance(content, list):
new_content = []
for item in content:
if item.get("type", "") == "text":
text_content = item["text"] if item["text"] != "" else "(empty)"
new_content.append({"text": text_content})
return {"role": message["role"], "content": new_content}
return message
def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
):
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
# Image should be the first content block in the message
content = [{"type": "image", "format": "jpeg", "source": {"bytes": encoded_image}}]
if text:
content.append({"text": text})
self.add_message({"role": "user", "content": content})
def add_message(self, message):
try:
if self.messages:
# AWS Bedrock requires that roles alternate. If this message's
# role is the same as the last message, we should add this
# message's content to the last message.
if self.messages[-1]["role"] == message["role"]:
# if the last message has just a content string, convert it to a list
# in the proper format
if isinstance(self.messages[-1]["content"], str):
self.messages[-1]["content"] = [{"text": self.messages[-1]["content"]}]
# if this message has just a content string, convert it to a list
# in the proper format
if isinstance(message["content"], str):
message["content"] = [{"text": message["content"]}]
# append the content of this message to the last message
self.messages[-1]["content"].extend(message["content"])
else:
self.messages.append(message)
else:
self.messages.append(message)
except Exception as e:
logger.error(f"Error adding message: {e}")
def _restructure_from_bedrock_messages(self):
"""Restructure messages in AWS Bedrock format by handling system
messages, merging consecutive messages with the same role, and ensuring
proper content formatting.
"""
# Handle system message if present at the beginning
if self.messages and self.messages[0]["role"] == "system":
if len(self.messages) == 1:
self.messages[0]["role"] = "user"
else:
system_content = self.messages.pop(0)["content"]
if isinstance(system_content, str):
system_content = [{"text": system_content}]
if self.system:
if isinstance(self.system, str):
self.system = [{"text": self.system}]
self.system.extend(system_content)
else:
self.system = system_content
# Ensure content is properly formatted
for msg in self.messages:
if isinstance(msg["content"], str):
msg["content"] = [{"text": msg["content"]}]
elif not msg["content"]:
msg["content"] = [{"text": "(empty)"}]
elif isinstance(msg["content"], list):
for idx, item in enumerate(msg["content"]):
if isinstance(item, dict) and "text" in item and item["text"] == "":
item["text"] = "(empty)"
elif isinstance(item, str) and item == "":
msg["content"][idx] = {"text": "(empty)"}
# Merge consecutive messages with the same role
merged_messages = []
for msg in self.messages:
if merged_messages and merged_messages[-1]["role"] == msg["role"]:
merged_messages[-1]["content"].extend(msg["content"])
else:
merged_messages.append(msg)
self.messages.clear()
self.messages.extend(merged_messages)
def _restructure_from_openai_messages(self):
# first, map across self._messages calling self.from_standard_message(m) to modify messages in place
try:
self._messages[:] = [self.from_standard_message(m) for m in self._messages]
except Exception as e:
logger.error(f"Error mapping messages: {e}")
# See if we should pull the system message out of our context.messages list. (For
# compatibility with Open AI messages format.)
if self.messages and self.messages[0]["role"] == "system":
self.system = self.messages[0]["content"]
self.messages.pop(0)
# Merge consecutive messages with the same role.
i = 0
while i < len(self.messages) - 1:
current_message = self.messages[i]
next_message = self.messages[i + 1]
if current_message["role"] == next_message["role"]:
# Convert content to list of dictionaries if it's a string
if isinstance(current_message["content"], str):
current_message["content"] = [
{"type": "text", "text": current_message["content"]}
]
if isinstance(next_message["content"], str):
next_message["content"] = [{"type": "text", "text": next_message["content"]}]
# Concatenate the content
current_message["content"].extend(next_message["content"])
# Remove the next message from the list
self.messages.pop(i + 1)
else:
i += 1
# Avoid empty content in messages
for message in self.messages:
if isinstance(message["content"], str) and message["content"] == "":
message["content"] = "(empty)"
elif isinstance(message["content"], list) and len(message["content"]) == 0:
message["content"] = [{"type": "text", "text": "(empty)"}]
def get_messages_for_persistent_storage(self):
messages = super().get_messages_for_persistent_storage()
if self.system:
messages.insert(0, {"role": "system", "content": self.system})
return messages
def get_messages_for_logging(self) -> str:
msgs = []
for message in self.messages:
msg = copy.deepcopy(message)
if "content" in msg:
if isinstance(msg["content"], list):
for item in msg["content"]:
if item.get("image"):
item["source"]["bytes"] = "..."
msgs.append(msg)
return json.dumps(msgs)
class AWSBedrockUserContextAggregator(LLMUserContextAggregator):
pass
class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator):
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
# Format tool use according to AWS Bedrock API
self._context.add_message(
{
"role": "assistant",
"content": [
{
"toolUse": {
"toolUseId": frame.tool_call_id,
"name": frame.function_name,
"input": frame.arguments if frame.arguments else {},
}
}
],
}
)
self._context.add_message(
{
"role": "user",
"content": [
{
"toolResult": {
"toolUseId": frame.tool_call_id,
"content": [{"text": "IN_PROGRESS"}],
}
}
],
}
)
async def handle_function_call_result(self, frame: FunctionCallResultFrame):
if frame.result:
result = json.dumps(frame.result)
await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
else:
await self._update_function_call_result(
frame.function_name, frame.tool_call_id, "COMPLETED"
)
async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
await self._update_function_call_result(
frame.function_name, frame.tool_call_id, "CANCELLED"
)
async def _update_function_call_result(
self, function_name: str, tool_call_id: str, result: Any
):
for message in self._context.messages:
if message["role"] == "user":
for content in message["content"]:
if (
isinstance(content, dict)
and content.get("toolResult")
and content["toolResult"]["toolUseId"] == tool_call_id
):
content["toolResult"]["content"] = [{"text": result}]
async def handle_user_image_frame(self, frame: UserImageRawFrame):
await self._update_function_call_result(
frame.request.function_name, frame.request.tool_call_id, "COMPLETED"
)
self._context.add_image_frame_message(
format=frame.format,
size=frame.size,
image=frame.image,
text=frame.request.context,
)
class AWSBedrockLLMService(LLMService):
"""This class implements inference with AWS Bedrock models including Amazon
Nova and Anthropic Claude.
Requires AWS credentials to be configured in the environment or through
boto3 configuration.
"""
# Overriding the default adapter to use the Anthropic one.
adapter_class = AWSBedrockLLMAdapter
class InputParams(BaseModel):
max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1)
temperature: Optional[float] = Field(default_factory=lambda: 0.7, ge=0.0, le=1.0)
top_p: Optional[float] = Field(default_factory=lambda: 0.999, ge=0.0, le=1.0)
stop_sequences: Optional[List[str]] = Field(default_factory=lambda: [])
latency: Optional[str] = Field(default_factory=lambda: "standard")
additional_model_request_fields: Optional[Dict[str, Any]] = Field(default_factory=dict)
def __init__(
self,
*,
aws_access_key: Optional[str] = None,
aws_secret_key: Optional[str] = None,
aws_session_token: Optional[str] = None,
aws_region: str = "us-east-1",
model: str,
params: InputParams = InputParams(),
client_config: Optional[Config] = None,
**kwargs,
):
super().__init__(**kwargs)
# Initialize the AWS Bedrock client
if not client_config:
client_config = Config(
connect_timeout=300, # 5 minutes
read_timeout=300, # 5 minutes
retries={"max_attempts": 3},
)
session = boto3.Session(
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
aws_session_token=aws_session_token,
region_name=aws_region,
)
self._client = session.client(service_name="bedrock-runtime", config=client_config)
self.set_model_name(model)
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
"top_p": params.top_p,
"latency": params.latency,
"additional_model_request_fields": params.additional_model_request_fields
if isinstance(params.additional_model_request_fields, dict)
else {},
}
logger.info(f"Using AWS Bedrock model: {model}")
def can_generate_metrics(self) -> bool:
return True
def create_context_aggregator(
self,
context: OpenAILLMContext,
*,
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> AWSBedrockContextAggregatorPair:
"""Create an instance of AWSBedrockContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
assistant aggregators can be provided.
Args:
context (OpenAILLMContext): The LLM context.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
Returns:
AWSBedrockContextAggregatorPair: A pair of context aggregators, one
for the user and one for the assistant, encapsulated in an
AWSBedrockContextAggregatorPair.
"""
context.set_llm_adapter(self.get_llm_adapter())
if isinstance(context, OpenAILLMContext):
context = AWSBedrockLLMContext.from_openai_context(context)
user = AWSBedrockUserContextAggregator(context, params=user_params)
assistant = AWSBedrockAssistantContextAggregator(context, params=assistant_params)
return AWSBedrockContextAggregatorPair(_user=user, _assistant=assistant)
@traced_llm
async def _process_context(self, context: AWSBedrockLLMContext):
# Usage tracking
prompt_tokens = 0
completion_tokens = 0
completion_tokens_estimate = 0
cache_read_input_tokens = 0
cache_creation_input_tokens = 0
use_completion_tokens_estimate = False
try:
await self.push_frame(LLMFullResponseStartFrame())
await self.start_processing_metrics()
await self.start_ttfb_metrics()
# Set up inference config
inference_config = {
"maxTokens": self._settings["max_tokens"],
"temperature": self._settings["temperature"],
"topP": self._settings["top_p"],
}
# Prepare request parameters
request_params = {
"modelId": self.model_name,
"messages": context.messages,
"inferenceConfig": inference_config,
"additionalModelRequestFields": self._settings["additional_model_request_fields"],
}
# Add system message
request_params["system"] = context.system
# Add tools if present
if context.tools:
tool_config = {"tools": context.tools}
# Add tool_choice if specified
if context.tool_choice:
if context.tool_choice == "auto":
tool_config["toolChoice"] = {"auto": {}}
elif context.tool_choice == "none":
# Skip adding toolChoice for "none"
pass
elif (
isinstance(context.tool_choice, dict) and "function" in context.tool_choice
):
tool_config["toolChoice"] = {
"tool": {"name": context.tool_choice["function"]["name"]}
}
request_params["toolConfig"] = tool_config
# Add performance config if latency is specified
if self._settings["latency"] in ["standard", "optimized"]:
request_params["performanceConfig"] = {"latency": self._settings["latency"]}
logger.debug(f"Calling AWS Bedrock model with: {request_params}")
# Call AWS Bedrock with streaming
response = self._client.converse_stream(**request_params)
await self.stop_ttfb_metrics()
# Process the streaming response
tool_use_block = None
json_accumulator = ""
for event in response["stream"]:
# Handle text content
if "contentBlockDelta" in event:
delta = event["contentBlockDelta"]["delta"]
if "text" in delta:
await self.push_frame(LLMTextFrame(delta["text"]))
completion_tokens_estimate += self._estimate_tokens(delta["text"])
elif "toolUse" in delta and "input" in delta["toolUse"]:
# Handle partial JSON for tool use
json_accumulator += delta["toolUse"]["input"]
completion_tokens_estimate += self._estimate_tokens(
delta["toolUse"]["input"]
)
# Handle tool use start
elif "contentBlockStart" in event:
content_block_start = event["contentBlockStart"]["start"]
if "toolUse" in content_block_start:
tool_use_block = {
"id": content_block_start["toolUse"].get("toolUseId", ""),
"name": content_block_start["toolUse"].get("name", ""),
}
json_accumulator = ""
# Handle message completion with tool use
elif "messageStop" in event and "stopReason" in event["messageStop"]:
if event["messageStop"]["stopReason"] == "tool_use" and tool_use_block:
try:
arguments = json.loads(json_accumulator) if json_accumulator else {}
await self.call_function(
context=context,
tool_call_id=tool_use_block["id"],
function_name=tool_use_block["name"],
arguments=arguments,
)
except json.JSONDecodeError:
logger.error(f"Failed to parse tool arguments: {json_accumulator}")
# Handle usage metrics if available
if "metadata" in event and "usage" in event["metadata"]:
usage = event["metadata"]["usage"]
prompt_tokens += usage.get("inputTokens", 0)
completion_tokens += usage.get("outputTokens", 0)
cache_read_input_tokens += usage.get("cacheReadInputTokens", 0)
cache_creation_input_tokens += usage.get("cacheWriteInputTokens", 0)
except asyncio.CancelledError:
# If we're interrupted, we won't get a complete usage report. So set our flag to use the
# token estimate. The reraise the exception so all the processors running in this task
# also get cancelled.
use_completion_tokens_estimate = True
raise
except httpx.TimeoutException:
await self._call_event_handler("on_completion_timeout")
except Exception as e:
logger.exception(f"{self} exception: {e}")
finally:
await self.stop_processing_metrics()
await self.push_frame(LLMFullResponseEndFrame())
comp_tokens = (
completion_tokens
if not use_completion_tokens_estimate
else completion_tokens_estimate
)
await self._report_usage_metrics(
prompt_tokens=prompt_tokens,
completion_tokens=comp_tokens,
cache_read_input_tokens=cache_read_input_tokens,
cache_creation_input_tokens=cache_creation_input_tokens,
)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
context = None
if isinstance(frame, OpenAILLMContextFrame):
context = AWSBedrockLLMContext.upgrade_to_bedrock(frame.context)
elif isinstance(frame, LLMMessagesFrame):
context = AWSBedrockLLMContext.from_messages(frame.messages)
elif isinstance(frame, VisionImageRawFrame):
# This is only useful in very simple pipelines because it creates
# a new context. Generally we want a context manager to catch
# UserImageRawFrames coming through the pipeline and add them
# to the context.
context = AWSBedrockLLMContext.from_image_frame(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
if context:
await self._process_context(context)
def _estimate_tokens(self, text: str) -> int:
return int(len(re.split(r"[^\w]+", text)) * 1.3)
async def _report_usage_metrics(
self,
prompt_tokens: int,
completion_tokens: int,
cache_read_input_tokens: int,
cache_creation_input_tokens: int,
):
if prompt_tokens or completion_tokens:
tokens = LLMTokenUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
cache_read_input_tokens=cache_read_input_tokens,
cache_creation_input_tokens=cache_creation_input_tokens,
)
await self.start_llm_usage_metrics(tokens)

341
src/pipecat/services/aws/stt.py Normal file
View File

@@ -0,0 +1,341 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import json
import os
import random
import string
from typing import AsyncGenerator, Optional
from loguru import logger
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
TranscriptionFrame,
)
from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use AWS services, you need to `pip install pipecat-ai[aws]`.")
raise Exception(f"Missing module: {e}")
class AWSTranscribeSTTService(STTService):
def __init__(
self,
*,
api_key: Optional[str] = None,
aws_access_key_id: Optional[str] = None,
aws_session_token: Optional[str] = None,
region: Optional[str] = "us-east-1",
sample_rate: int = 16000,
language: Language = Language.EN,
**kwargs,
):
super().__init__(**kwargs)
self._settings = {
"sample_rate": sample_rate,
"language": language,
"media_encoding": "linear16", # AWS expects raw PCM
"number_of_channels": 1,
"show_speaker_label": False,
"enable_channel_identification": False,
}
# Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz
if sample_rate not in [8000, 16000]:
logger.warning(
f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz."
)
self._settings["sample_rate"] = 16000
self._credentials = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
"aws_secret_access_key": api_key or os.getenv("AWS_SECRET_ACCESS_KEY"),
"aws_session_token": aws_session_token or os.getenv("AWS_SESSION_TOKEN"),
"region": region or os.getenv("AWS_REGION", "us-east-1"),
}
self._ws_client = None
self._connection_lock = asyncio.Lock()
self._connecting = False
self._receive_task = None
def get_service_encoding(self, encoding: str) -> str:
"""Convert internal encoding format to AWS Transcribe format."""
encoding_map = {
"linear16": "pcm", # AWS expects "pcm" for 16-bit linear PCM
}
return encoding_map.get(encoding, encoding)
async def start(self, frame: StartFrame):
"""Initialize the connection when the service starts."""
await super().start(frame)
logger.info("Starting AWS Transcribe service...")
retry_count = 0
max_retries = 3
while retry_count < max_retries:
try:
await self._connect()
if self._ws_client and self._ws_client.open:
logger.info("Successfully established WebSocket connection")
return
logger.warning("WebSocket connection not established after connect")
except Exception as e:
logger.error(f"Failed to connect (attempt {retry_count + 1}/{max_retries}): {e}")
retry_count += 1
if retry_count < max_retries:
await asyncio.sleep(1) # Wait before retrying
raise RuntimeError("Failed to establish WebSocket connection after multiple attempts")
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data and send to AWS Transcribe"""
try:
# Ensure WebSocket is connected
if not self._ws_client or not self._ws_client.open:
logger.debug("WebSocket not connected, attempting to reconnect...")
try:
await self._connect()
except Exception as e:
logger.error(f"Failed to reconnect: {e}")
yield ErrorFrame("Failed to reconnect to AWS Transcribe", fatal=False)
return
# Format the audio data according to AWS event stream format
event_message = build_event_message(audio)
# Send the formatted event message
try:
await self._ws_client.send(event_message)
# Start metrics after first chunk sent
await self.start_processing_metrics()
await self.start_ttfb_metrics()
except websockets.exceptions.ConnectionClosed as e:
logger.warning(f"Connection closed while sending: {e}")
await self._disconnect()
# Don't yield error here - we'll retry on next frame
except Exception as e:
logger.error(f"Error sending audio: {e}")
yield ErrorFrame(f"AWS Transcribe error: {str(e)}", fatal=False)
await self._disconnect()
except Exception as e:
logger.error(f"Error in run_stt: {e}")
yield ErrorFrame(f"AWS Transcribe error: {str(e)}", fatal=False)
await self._disconnect()
async def _connect(self):
"""Connect to AWS Transcribe with connection state management."""
if self._ws_client and self._ws_client.open and self._receive_task:
logger.debug(f"{self} Already connected")
return
async with self._connection_lock:
if self._connecting:
logger.debug(f"{self} Connection already in progress")
return
try:
self._connecting = True
logger.debug(f"{self} Starting connection process...")
if self._ws_client:
await self._disconnect()
language_code = self.language_to_service_language(
Language(self._settings["language"])
)
if not language_code:
raise ValueError(f"Unsupported language: {self._settings['language']}")
# Generate random websocket key
websocket_key = "".join(
random.choices(
string.ascii_uppercase + string.ascii_lowercase + string.digits, k=20
)
)
# Add required headers
extra_headers = {
"Origin": "https://localhost",
"Sec-WebSocket-Key": websocket_key,
"Sec-WebSocket-Version": "13",
"Connection": "keep-alive",
}
# Get presigned URL
presigned_url = get_presigned_url(
region=self._credentials["region"],
credentials={
"access_key": self._credentials["aws_access_key_id"],
"secret_key": self._credentials["aws_secret_access_key"],
"session_token": self._credentials["aws_session_token"],
},
language_code=language_code,
media_encoding=self.get_service_encoding(
self._settings["media_encoding"]
), # Convert to AWS format
sample_rate=self._settings["sample_rate"],
number_of_channels=self._settings["number_of_channels"],
enable_partial_results_stabilization=True,
partial_results_stability="high",
show_speaker_label=self._settings["show_speaker_label"],
enable_channel_identification=self._settings["enable_channel_identification"],
)
logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...")
# Connect with the required headers and settings
self._ws_client = await websockets.connect(
presigned_url,
extra_headers=extra_headers,
subprotocols=["mqtt"],
ping_interval=None,
ping_timeout=None,
compression=None,
)
logger.debug(f"{self} WebSocket connected, starting receive task...")
# Start receive task
self._receive_task = self.create_task(self._receive_loop())
logger.info(f"{self} Successfully connected to AWS Transcribe")
except Exception as e:
logger.error(f"{self} Failed to connect to AWS Transcribe: {e}")
await self._disconnect()
raise
finally:
self._connecting = False
async def _disconnect(self):
"""Disconnect from AWS Transcribe."""
if self._receive_task:
await self.cancel_task(self._receive_task)
self._receive_task = None
try:
if self._ws_client and self._ws_client.open:
# Send end-stream message
end_stream = {"message-type": "event", "event": "end"}
await self._ws_client.send(json.dumps(end_stream))
await self._ws_client.close()
except Exception as e:
logger.warning(f"{self} Error closing WebSocket connection: {e}")
finally:
self._ws_client = None
def language_to_service_language(self, language: Language) -> str | None:
"""Convert internal language enum to AWS Transcribe language code."""
language_map = {
Language.EN: "en-US",
Language.ES: "es-US",
Language.FR: "fr-FR",
Language.DE: "de-DE",
Language.IT: "it-IT",
Language.PT: "pt-BR",
Language.JA: "ja-JP",
Language.KO: "ko-KR",
Language.ZH: "zh-CN",
}
return language_map.get(language)
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[str] = None
):
pass
async def _receive_loop(self):
"""Background task to receive and process messages from AWS Transcribe."""
while True:
if not self._ws_client or not self._ws_client.open:
logger.warning(f"{self} WebSocket closed in receive loop")
break
try:
response = await self._ws_client.recv()
headers, payload = decode_event(response)
if headers.get(":message-type") == "event":
# Process transcription results
results = payload.get("Transcript", {}).get("Results", [])
if results:
result = results[0]
alternatives = result.get("Alternatives", [])
if alternatives:
transcript = alternatives[0].get("Transcript", "")
is_final = not result.get("IsPartial", True)
if transcript:
await self.stop_ttfb_metrics()
if is_final:
await self.push_frame(
TranscriptionFrame(
transcript,
"",
time_now_iso8601(),
self._settings["language"],
)
)
await self._handle_transcription(
transcript,
is_final,
self._settings["language"],
)
await self.stop_processing_metrics()
else:
await self.push_frame(
InterimTranscriptionFrame(
transcript,
"",
time_now_iso8601(),
self._settings["language"],
)
)
elif headers.get(":message-type") == "exception":
error_msg = payload.get("Message", "Unknown error")
logger.error(f"{self} Exception from AWS: {error_msg}")
await self.push_frame(
ErrorFrame(f"AWS Transcribe error: {error_msg}", fatal=False)
)
else:
logger.debug(f"{self} Other message type received: {headers}")
logger.debug(f"{self} Payload: {payload}")
except websockets.exceptions.ConnectionClosed as e:
logger.error(
f"{self} WebSocket connection closed in receive loop with code {e.code}: {e.reason}"
)
break
except Exception as e:
logger.error(f"{self} Unexpected error in receive loop: {e}")
break

60
src/pipecat/services/aws/tts.py
View File

@@ -5,6 +5,7 @@
#
import asyncio
import os
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -20,15 +21,14 @@ from pipecat.frames.frames import (
)
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
import boto3
from botocore.exceptions import BotoCoreError, ClientError
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Deepgram, you need to `pip install pipecat-ai[aws]`. Also, set `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, and `AWS_REGION` environment variable."
)
logger.error("In order to use AWS services, you need to `pip install pipecat-ai[aws]`.")
raise Exception(f"Missing module: {e}")
@@ -108,7 +108,7 @@ def language_to_aws_language(language: Language) -> Optional[str]:
return language_map.get(language)
class PollyTTSService(TTSService):
class AWSPollyTTSService(TTSService):
class InputParams(BaseModel):
engine: Optional[str] = None
language: Optional[Language] = Language.EN
@@ -151,6 +151,24 @@ class PollyTTSService(TTSService):
self.set_voice(voice_id)
# Get credentials from environment variables if not provided
self._credentials = {
"aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
"aws_secret_access_key": api_key or os.getenv("AWS_SECRET_ACCESS_KEY"),
"aws_session_token": aws_session_token or os.getenv("AWS_SESSION_TOKEN"),
"region": region or os.getenv("AWS_REGION", "us-east-1"),
}
# Validate that we have the required credentials
if (
not self._credentials["aws_access_key_id"]
or not self._credentials["aws_secret_access_key"]
):
raise ValueError(
"AWS credentials not found. Please provide them either through constructor parameters "
"or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables."
)
def can_generate_metrics(self) -> bool:
return True
@@ -165,18 +183,17 @@ class PollyTTSService(TTSService):
prosody_attrs = []
# Prosody tags are only supported for standard and neural engines
if self._settings["engine"] != "generative":
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["engine"] == "standard":
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
else:
logger.warning("Prosody tags are not supported for generative engine. Ignoring.")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
ssml += text
@@ -187,8 +204,11 @@ class PollyTTSService(TTSService):
ssml += "</speak>"
logger.trace(f"{self} SSML: {ssml}")
return ssml
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
def read_audio_data(**args):
response = self._polly_client.synthesize_speech(**args)
@@ -248,3 +268,17 @@ class PollyTTSService(TTSService):
finally:
yield TTSStoppedFrame()
class PollyTTSService(AWSPollyTTSService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'PollyTTSService' is deprecated, use 'AWSPollyTTSService' instead.",
DeprecationWarning,
)

261
src/pipecat/services/aws/utils.py Normal file
View File

@@ -0,0 +1,261 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import binascii
import datetime
import hashlib
import hmac
import json
import struct
import urllib.parse
from typing import Dict, Optional
def get_presigned_url(
*,
region: str,
credentials: Dict[str, Optional[str]],
language_code: str,
media_encoding: str = "pcm",
sample_rate: int = 16000,
number_of_channels: int = 1,
enable_partial_results_stabilization: bool = True,
partial_results_stability: str = "high",
vocabulary_name: Optional[str] = None,
vocabulary_filter_name: Optional[str] = None,
show_speaker_label: bool = False,
enable_channel_identification: bool = False,
) -> str:
"""Create a presigned URL for AWS Transcribe streaming."""
access_key = credentials.get("access_key")
secret_key = credentials.get("secret_key")
session_token = credentials.get("session_token")
if not access_key or not secret_key:
raise ValueError("AWS credentials are required")
# Initialize the URL generator
url_generator = AWSTranscribePresignedURL(
access_key=access_key, secret_key=secret_key, session_token=session_token, region=region
)
# Get the presigned URL
return url_generator.get_request_url(
sample_rate=sample_rate,
language_code=language_code,
media_encoding=media_encoding,
vocabulary_name=vocabulary_name,
vocabulary_filter_name=vocabulary_filter_name,
show_speaker_label=show_speaker_label,
enable_channel_identification=enable_channel_identification,
number_of_channels=number_of_channels,
enable_partial_results_stabilization=enable_partial_results_stabilization,
partial_results_stability=partial_results_stability,
)
class AWSTranscribePresignedURL:
def __init__(
self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1"
):
self.access_key = access_key
self.secret_key = secret_key
self.session_token = session_token
self.method = "GET"
self.service = "transcribe"
self.region = region
self.endpoint = ""
self.host = ""
self.amz_date = ""
self.datestamp = ""
self.canonical_uri = "/stream-transcription-websocket"
self.canonical_headers = ""
self.signed_headers = "host"
self.algorithm = "AWS4-HMAC-SHA256"
self.credential_scope = ""
self.canonical_querystring = ""
self.payload_hash = ""
self.canonical_request = ""
self.string_to_sign = ""
self.signature = ""
self.request_url = ""
def get_request_url(
self,
sample_rate: int,
language_code: str = "",
media_encoding: str = "pcm",
vocabulary_name: str = "",
vocabulary_filter_name: str = "",
show_speaker_label: bool = False,
enable_channel_identification: bool = False,
number_of_channels: int = 1,
enable_partial_results_stabilization: bool = False,
partial_results_stability: str = "",
) -> str:
self.endpoint = f"wss://transcribestreaming.{self.region}.amazonaws.com:8443"
self.host = f"transcribestreaming.{self.region}.amazonaws.com:8443"
now = datetime.datetime.utcnow()
self.amz_date = now.strftime("%Y%m%dT%H%M%SZ")
self.datestamp = now.strftime("%Y%m%d")
self.canonical_headers = f"host:{self.host}\n"
self.credential_scope = f"{self.datestamp}%2F{self.region}%2F{self.service}%2Faws4_request"
# Create canonical querystring
self.canonical_querystring = "X-Amz-Algorithm=" + self.algorithm
self.canonical_querystring += (
"&X-Amz-Credential=" + self.access_key + "%2F" + self.credential_scope
)
self.canonical_querystring += "&X-Amz-Date=" + self.amz_date
self.canonical_querystring += "&X-Amz-Expires=300"
if self.session_token:
self.canonical_querystring += "&X-Amz-Security-Token=" + urllib.parse.quote(
self.session_token, safe=""
)
self.canonical_querystring += "&X-Amz-SignedHeaders=" + self.signed_headers
if enable_channel_identification:
self.canonical_querystring += "&enable-channel-identification=true"
if enable_partial_results_stabilization:
self.canonical_querystring += "&enable-partial-results-stabilization=true"
if language_code:
self.canonical_querystring += "&language-code=" + language_code
if media_encoding:
self.canonical_querystring += "&media-encoding=" + media_encoding
if number_of_channels > 1:
self.canonical_querystring += "&number-of-channels=" + str(number_of_channels)
if partial_results_stability:
self.canonical_querystring += "&partial-results-stability=" + partial_results_stability
if sample_rate:
self.canonical_querystring += "&sample-rate=" + str(sample_rate)
if show_speaker_label:
self.canonical_querystring += "&show-speaker-label=true"
if vocabulary_filter_name:
self.canonical_querystring += "&vocabulary-filter-name=" + vocabulary_filter_name
if vocabulary_name:
self.canonical_querystring += "&vocabulary-name=" + vocabulary_name
# Create payload hash
self.payload_hash = hashlib.sha256("".encode("utf-8")).hexdigest()
# Create canonical request
self.canonical_request = f"{self.method}\n{self.canonical_uri}\n{self.canonical_querystring}\n{self.canonical_headers}\n{self.signed_headers}\n{self.payload_hash}"
# Create string to sign
credential_scope = f"{self.datestamp}/{self.region}/{self.service}/aws4_request"
string_to_sign = (
f"{self.algorithm}\n{self.amz_date}\n{credential_scope}\n"
+ hashlib.sha256(self.canonical_request.encode("utf-8")).hexdigest()
)
# Calculate signature
k_date = hmac.new(
f"AWS4{self.secret_key}".encode("utf-8"), self.datestamp.encode("utf-8"), hashlib.sha256
).digest()
k_region = hmac.new(k_date, self.region.encode("utf-8"), hashlib.sha256).digest()
k_service = hmac.new(k_region, self.service.encode("utf-8"), hashlib.sha256).digest()
k_signing = hmac.new(k_service, b"aws4_request", hashlib.sha256).digest()
self.signature = hmac.new(
k_signing, string_to_sign.encode("utf-8"), hashlib.sha256
).hexdigest()
# Add signature to query string
self.canonical_querystring += "&X-Amz-Signature=" + self.signature
# Create request URL
self.request_url = self.endpoint + self.canonical_uri + "?" + self.canonical_querystring
return self.request_url
def get_headers(header_name: str, header_value: str) -> bytearray:
"""Build a header following AWS event stream format."""
name = header_name.encode("utf-8")
name_byte_length = bytes([len(name)])
value_type = bytes([7]) # 7 represents a string
value = header_value.encode("utf-8")
value_byte_length = struct.pack(">H", len(value))
# Construct the header
header_list = bytearray()
header_list.extend(name_byte_length)
header_list.extend(name)
header_list.extend(value_type)
header_list.extend(value_byte_length)
header_list.extend(value)
return header_list
def build_event_message(payload: bytes) -> bytes:
"""
Build an event message for AWS Transcribe streaming.
Matches AWS sample: https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py
"""
# Build headers
content_type_header = get_headers(":content-type", "application/octet-stream")
event_type_header = get_headers(":event-type", "AudioEvent")
message_type_header = get_headers(":message-type", "event")
headers = bytearray()
headers.extend(content_type_header)
headers.extend(event_type_header)
headers.extend(message_type_header)
# Calculate total byte length and headers byte length
# 16 accounts for 8 byte prelude, 2x 4 byte CRCs
total_byte_length = struct.pack(">I", len(headers) + len(payload) + 16)
headers_byte_length = struct.pack(">I", len(headers))
# Build the prelude
prelude = bytearray([0] * 8)
prelude[:4] = total_byte_length
prelude[4:] = headers_byte_length
# Calculate checksum for prelude
prelude_crc = struct.pack(">I", binascii.crc32(prelude) & 0xFFFFFFFF)
# Construct the message
message_as_list = bytearray()
message_as_list.extend(prelude)
message_as_list.extend(prelude_crc)
message_as_list.extend(headers)
message_as_list.extend(payload)
# Calculate checksum for message
message = bytes(message_as_list)
message_crc = struct.pack(">I", binascii.crc32(message) & 0xFFFFFFFF)
# Add message checksum
message_as_list.extend(message_crc)
return bytes(message_as_list)
def decode_event(message):
# Extract the prelude, headers, payload and CRC
prelude = message[:8]
total_length, headers_length = struct.unpack(">II", prelude)
prelude_crc = struct.unpack(">I", message[8:12])[0]
headers = message[12 : 12 + headers_length]
payload = message[12 + headers_length : -4]
message_crc = struct.unpack(">I", message[-4:])[0]
# Check the CRCs
assert prelude_crc == binascii.crc32(prelude) & 0xFFFFFFFF, "Prelude CRC check failed"
assert message_crc == binascii.crc32(message[:-4]) & 0xFFFFFFFF, "Message CRC check failed"
# Parse the headers
headers_dict = {}
while headers:
name_len = headers[0]
name = headers[1 : 1 + name_len].decode("utf-8")
value_type = headers[1 + name_len]
value_len = struct.unpack(">H", headers[2 + name_len : 4 + name_len])[0]
value = headers[4 + name_len : 4 + name_len + value_len].decode("utf-8")
headers_dict[name] = value
headers = headers[4 + name_len + value_len :]
return headers_dict, json.loads(payload)

1
src/pipecat/services/aws_nova_sonic/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .aws import AWSNovaSonicLLMService, Params

1037
src/pipecat/services/aws_nova_sonic/aws.py Normal file
View File

File diff suppressed because it is too large Load Diff

227
src/pipecat/services/aws_nova_sonic/context.py Normal file
View File

@@ -0,0 +1,227 @@
#
# Copyright (c) 2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import copy
from dataclasses import dataclass, field
from enum import Enum
from loguru import logger
from pipecat.frames.frames import (
BotStoppedSpeakingFrame,
DataFrame,
Frame,
FunctionCallResultFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesAppendFrame,
LLMMessagesUpdateFrame,
LLMSetToolChoiceFrame,
LLMSetToolsFrame,
StartInterruptionFrame,
TextFrame,
UserImageRawFrame,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame
from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
class Role(Enum):
SYSTEM = "SYSTEM"
USER = "USER"
ASSISTANT = "ASSISTANT"
TOOL = "TOOL"
@dataclass
class AWSNovaSonicConversationHistoryMessage:
role: Role # only USER and ASSISTANT
text: str
@dataclass
class AWSNovaSonicConversationHistory:
system_instruction: str = None
messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list)
class AWSNovaSonicLLMContext(OpenAILLMContext):
def __init__(self, messages=None, tools=None, **kwargs):
super().__init__(messages=messages, tools=tools, **kwargs)
self.__setup_local()
def __setup_local(self, system_instruction: str = ""):
self._assistant_text = ""
self._user_text = ""
self._system_instruction = system_instruction
@staticmethod
def upgrade_to_nova_sonic(
obj: OpenAILLMContext, system_instruction: str
) -> "AWSNovaSonicLLMContext":
if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext):
obj.__class__ = AWSNovaSonicLLMContext
obj.__setup_local(system_instruction)
return obj
# NOTE: this method has the side-effect of updating _system_instruction from messages
def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory:
history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction)
# Bail if there are no messages
if not self.messages:
return history
messages = copy.deepcopy(self.messages)
# If we have a "system" message as our first message, let's pull that out into "instruction"
if messages[0].get("role") == "system":
system = messages.pop(0)
content = system.get("content")
if isinstance(content, str):
history.system_instruction = content
elif isinstance(content, list):
history.system_instruction = content[0].get("text")
if history.system_instruction:
self._system_instruction = history.system_instruction
# Process remaining messages to fill out conversation history.
# Nova Sonic supports "user" and "assistant" messages in history.
for message in messages:
history_message = self.from_standard_message(message)
if history_message:
history.messages.append(history_message)
return history
def get_messages_for_persistent_storage(self):
messages = super().get_messages_for_persistent_storage()
# If we have a system instruction and messages doesn't already contain it, add it
if self._system_instruction and not (messages and messages[0].get("role") == "system"):
messages.insert(0, {"role": "system", "content": self._system_instruction})
return messages
def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage:
role = message.get("role")
if message.get("role") == "user" or message.get("role") == "assistant":
content = message.get("content")
if isinstance(message.get("content"), list):
content = ""
for c in message.get("content"):
if c.get("type") == "text":
content += " " + c.get("text")
else:
logger.error(
f"Unhandled content type in context message: {c.get('type')} - {message}"
)
# There won't be content if this is an assistant tool call entry.
# We're ignoring those since they can't be loaded into AWS Nova Sonic conversation
# history
if content:
return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content)
# NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova
# Sonic conversation history
def buffer_user_text(self, text):
self._user_text += f" {text}" if self._user_text else text
# logger.debug(f"User text buffered: {self._user_text}")
def flush_aggregated_user_text(self) -> str:
if not self._user_text:
return ""
user_text = self._user_text
message = {
"role": "user",
"content": [{"type": "text", "text": user_text}],
}
self._user_text = ""
self.add_message(message)
# logger.debug(f"Context updated (user): {self.get_messages_for_logging()}")
return user_text
def buffer_assistant_text(self, text):
self._assistant_text += text
# logger.debug(f"Assistant text buffered: {self._assistant_text}")
def flush_aggregated_assistant_text(self):
if not self._assistant_text:
return
message = {
"role": "assistant",
"content": [{"type": "text", "text": self._assistant_text}],
}
self._assistant_text = ""
self.add_message(message)
# logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}")
@dataclass
class AWSNovaSonicMessagesUpdateFrame(DataFrame):
context: AWSNovaSonicLLMContext
class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator):
async def process_frame(
self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
):
await super().process_frame(frame, direction)
# Parent does not push LLMMessagesUpdateFrame
if isinstance(frame, LLMMessagesUpdateFrame):
await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context))
class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator):
async def process_frame(self, frame: Frame, direction: FrameDirection):
# HACK: For now, disable the context aggregator by making it just pass through all frames
# that the parent handles (except the function call stuff, which we still need).
# For an explanation of this hack, see
# AWSNovaSonicLLMService._report_assistant_response_text_added.
if isinstance(
frame,
(
StartInterruptionFrame,
LLMFullResponseStartFrame,
LLMFullResponseEndFrame,
TextFrame,
LLMMessagesAppendFrame,
LLMMessagesUpdateFrame,
LLMSetToolsFrame,
LLMSetToolChoiceFrame,
UserImageRawFrame,
BotStoppedSpeakingFrame,
),
):
await self.push_frame(frame, direction)
else:
await super().process_frame(frame, direction)
async def handle_function_call_result(self, frame: FunctionCallResultFrame):
await super().handle_function_call_result(frame)
# The standard function callback code path pushes the FunctionCallResultFrame from the LLM
# itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side
# context. Let's push a special frame to do that.
await self.push_frame(
AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM
)
@dataclass
class AWSNovaSonicContextAggregatorPair:
_user: AWSNovaSonicUserContextAggregator
_assistant: AWSNovaSonicAssistantContextAggregator
def user(self) -> AWSNovaSonicUserContextAggregator:
return self._user
def assistant(self) -> AWSNovaSonicAssistantContextAggregator:
return self._assistant

14
src/pipecat/services/aws_nova_sonic/frames.py Normal file
View File

@@ -0,0 +1,14 @@
#
# Copyright (c) 2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from dataclasses import dataclass
from pipecat.frames.frames import DataFrame, FunctionCallResultFrame
@dataclass
class AWSNovaSonicFunctionCallResultFrame(DataFrame):
result_frame: FunctionCallResultFrame

BIN
src/pipecat/services/aws_nova_sonic/ready.wav Normal file
View File

Binary file not shown.

25
src/pipecat/services/azure/stt.py
View File

@@ -20,6 +20,7 @@ from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
from azure.cognitiveservices.speech import (
@@ -58,12 +59,20 @@ class AzureSTTService(STTService):
self._audio_stream = None
self._speech_recognizer = None
self._settings = {
"region": region,
"language": language_to_azure_language(language),
"sample_rate": sample_rate,
}
def can_generate_metrics(self) -> bool:
return True
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
await self.start_ttfb_metrics()
if self._audio_stream:
self._audio_stream.write(audio)
await self.stop_processing_metrics()
yield None
async def start(self, frame: StartFrame):
@@ -101,7 +110,19 @@ class AzureSTTService(STTService):
if self._audio_stream:
self._audio_stream.close()
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
def _on_handle_recognized(self, event):
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
frame = TranscriptionFrame(event.result.text, "", time_now_iso8601())
language = getattr(event.result, "language", None) or self._settings.get("language")
frame = TranscriptionFrame(event.result.text, "", time_now_iso8601(), language)
asyncio.run_coroutine_threadsafe(
self._handle_transcription(event.result.text, True, language), self.get_event_loop()
)
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())

3
src/pipecat/services/azure/tts.py
View File

@@ -21,6 +21,7 @@ from pipecat.frames.frames import (
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
from azure.cognitiveservices.speech import (
@@ -196,6 +197,7 @@ class AzureTTSService(AzureBaseTTSService):
async def flush_audio(self):
logger.trace(f"{self}: flushing audio")
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
@@ -263,6 +265,7 @@ class AzureHttpTTSService(AzureBaseTTSService):
speech_config=self._speech_config, audio_config=None
)
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")

13
src/pipecat/services/canonical/__init__.py
View File

@@ -1,13 +0,0 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import sys
from pipecat.services import DeprecatedModuleProxy
from .metrics import *
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "canonical", "canonical.metrics")

230
src/pipecat/services/canonical/metrics.py
View File

@@ -1,230 +0,0 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import io
import os
import uuid
import wave
from datetime import datetime
from typing import Dict, List, Optional, Tuple
import aiohttp
from loguru import logger
from pipecat.frames.frames import CancelFrame, EndFrame, Frame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
try:
import aiofiles
import aiofiles.os
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Canonical Metrics, you need to `pip install pipecat-ai[canonical]`. "
+ "Also, set the `CANONICAL_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
# Multipart upload part size in bytes, cannot be smaller than 5MB
PART_SIZE = 1024 * 1024 * 5
class CanonicalMetricsService(AIService):
"""Initialize a CanonicalAudioProcessor instance.
This class uses an AudioBufferProcessor to get the conversation audio and
uploads it to Canonical Voice API for audio processing.
Args:
call_id (str): Your unique identifier for the call. This is used to match the call in the Canonical Voice system to the call in your system.
assistant (str): Identifier for the AI assistant. This can be whatever you want, it's intended for you convenience so you can distinguish
between different assistants and a grouping mechanism for calls.
assistant_speaks_first (bool, optional): Indicates if the assistant speaks first in the conversation. Defaults to True.
output_dir (str, optional): Directory to save temporary audio files. Defaults to "recordings".
Attributes:
call_id (str): Stores the unique call identifier.
assistant (str): Stores the assistant identifier.
assistant_speaks_first (bool): Indicates whether the assistant speaks first.
output_dir (str): Directory path for saving temporary audio files.
The constructor also ensures that the output directory exists.
"""
def __init__(
self,
*,
aiohttp_session: aiohttp.ClientSession,
call_id: str,
assistant: str,
api_key: str,
api_url: str = "https://voiceapp.canonical.chat/api/v1",
assistant_speaks_first: bool = True,
output_dir: str = "recordings",
audio_buffer_processor: Optional[AudioBufferProcessor] = None,
context: Optional[OpenAILLMContext] = None,
**kwargs,
):
super().__init__(**kwargs)
# Validate that at least one of audio_buffer_processor or context is provided
if audio_buffer_processor is None and context is None:
raise ValueError("At least one of audio_buffer_processor or context must be specified")
self._aiohttp_session = aiohttp_session
self._audio_buffer_processor = audio_buffer_processor
self._api_key = api_key
self._api_url = api_url
self._call_id = call_id
self._assistant = assistant
self._assistant_speaks_first = assistant_speaks_first
self._output_dir = output_dir
self._context = context
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._process_completion()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._process_completion()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
async def _process_completion(self):
if self._audio_buffer_processor is not None:
await self._process_audio()
elif self._context is not None:
await self._process_transcript()
async def _process_transcript(self):
params = {
"callId": self._call_id,
"assistant": {"id": self._assistant, "speaksFirst": self._assistant_speaks_first},
"transcript": self._context.messages,
}
response = await self._aiohttp_session.post(
f"{self._api_url}/call",
headers=self._request_headers(),
json=params,
)
if not response.ok:
logger.error(f"Failed to process transcript: {await response.text()}")
async def _process_audio(self):
audio_buffer_processor = self._audio_buffer_processor
if not audio_buffer_processor.has_audio():
return
os.makedirs(self._output_dir, exist_ok=True)
filename = self._get_output_filename()
audio = audio_buffer_processor.merge_audio_buffers()
with io.BytesIO() as buffer:
with wave.open(buffer, "wb") as wf:
wf.setsampwidth(2)
wf.setnchannels(audio_buffer_processor.num_channels)
wf.setframerate(audio_buffer_processor.sample_rate)
wf.writeframes(audio)
async with aiofiles.open(filename, "wb") as file:
await file.write(buffer.getvalue())
try:
await self._multipart_upload(filename)
await aiofiles.os.remove(filename)
except FileNotFoundError:
pass
except Exception as e:
logger.error(f"Failed to upload recording: {e}")
def _get_output_filename(self):
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
return f"{self._output_dir}/{timestamp}-{uuid.uuid4().hex}.wav"
def _request_headers(self):
return {"Content-Type": "application/json", "X-Canonical-Api-Key": self._api_key}
async def _multipart_upload(self, file_path: str):
upload_request, upload_response = await self._request_upload(file_path)
if upload_request is None or upload_response is None:
return
parts = await self._upload_parts(file_path, upload_response)
if parts is None:
return
await self._upload_complete(parts, upload_request, upload_response)
async def _request_upload(self, file_path: str) -> Tuple[Dict, Dict]:
filename = os.path.basename(file_path)
filesize = os.path.getsize(file_path)
numparts = int((filesize + PART_SIZE - 1) / PART_SIZE)
params = {
"filename": filename,
"parts": numparts,
"callId": self._call_id,
"assistant": {"id": self._assistant, "speaksFirst": self._assistant_speaks_first},
}
logger.debug(f"Requesting presigned URLs for {numparts} parts")
response = await self._aiohttp_session.post(
f"{self._api_url}/recording/uploadRequest", headers=self._request_headers(), json=params
)
if not response.ok:
logger.error(f"Failed to get presigned URLs: {await response.text()}")
return None, None
response_json = await response.json()
return params, response_json
async def _upload_parts(self, file_path: str, upload_response: Dict) -> List[Dict]:
urls = upload_response["urls"]
parts = []
try:
async with aiofiles.open(file_path, "rb") as file:
for partnum, upload_url in enumerate(urls, start=1):
data = await file.read(PART_SIZE)
if not data:
break
response = await self._aiohttp_session.put(upload_url, data=data)
if not response.ok:
logger.error(f"Failed to upload part {partnum}: {await response.text()}")
return None
etag = response.headers["ETag"]
parts.append({"partnum": str(partnum), "etag": etag})
except Exception as e:
logger.error(f"Multipart upload aborted, an error occurred: {str(e)}")
return parts
async def _upload_complete(
self, parts: List[Dict], upload_request: Dict, upload_response: Dict
):
params = {
"filename": upload_request["filename"],
"parts": parts,
"slug": upload_response["slug"],
"callId": self._call_id,
"assistant": {"id": self._assistant, "speaksFirst": self._assistant_speaks_first},
}
if self._context is not None:
params["transcript"] = self._context.messages
logger.debug(f"Completing upload for {params['filename']}")
logger.debug(f"Slug: {params['slug']}")
response = await self._aiohttp_session.post(
f"{self._api_url}/recording/uploadComplete",
headers=self._request_headers(),
json=params,
)
if not response.ok:
logger.error(f"Failed to complete upload: {await response.text()}")
return

7
src/pipecat/services/cartesia/tts.py
View File

@@ -28,6 +28,7 @@ from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
from pipecat.utils.tracing.service_decorators import traced_tts
# See .env.example for Cartesia configuration needed
try:
@@ -250,9 +251,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
continue
if msg["type"] == "done":
await self.stop_ttfb_metrics()
await self.add_word_timestamps(
[("TTSStoppedFrame", 0), ("LLMFullResponseEndFrame", 0), ("Reset", 0)]
)
await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)])
await self.remove_audio_context(msg["context_id"])
elif msg["type"] == "timestamps":
await self.add_word_timestamps(
@@ -276,6 +275,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
else:
logger.error(f"{self} error, unknown message type: {msg}")
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
@@ -362,6 +362,7 @@ class CartesiaHttpTTSService(TTSService):
await super().cancel(frame)
await self._client.close()
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")

10
src/pipecat/services/deepgram/stt.py
View File

@@ -22,6 +22,7 @@ from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
from deepgram import (
@@ -187,6 +188,13 @@ class DeepgramSTTService(STTService):
async def _on_utterance_end(self, *args, **kwargs):
await self._call_event_handler("on_utterance_end", *args, **kwargs)
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
async def _on_message(self, *args, **kwargs):
result: LiveResultResponse = kwargs["result"]
if len(result.channel.alternatives) == 0:
@@ -203,8 +211,10 @@ class DeepgramSTTService(STTService):
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), language)
)
await self._handle_transcription(transcript, is_final, language)
await self.stop_processing_metrics()
else:
# For interim transcriptions, just push the frame without tracing
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)
)

25
src/pipecat/services/deepgram/tts.py
View File

@@ -16,6 +16,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
try:
from deepgram import DeepgramClient, DeepgramClientOptions, SpeakOptions
@@ -30,7 +31,7 @@ class DeepgramTTSService(TTSService):
self,
*,
api_key: str,
voice: str = "aura-helios-en",
voice: str = "aura-2-helena-en",
base_url: str = "",
sample_rate: Optional[int] = None,
encoding: str = "linear16",
@@ -49,6 +50,7 @@ class DeepgramTTSService(TTSService):
def can_generate_metrics(self) -> bool:
return True
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
@@ -62,29 +64,18 @@ class DeepgramTTSService(TTSService):
try:
await self.start_ttfb_metrics()
response = await self._deepgram_client.speak.asyncrest.v("1").stream_memory(
response = await self._deepgram_client.speak.asyncrest.v("1").stream_raw(
{"text": text}, options
)
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
# The response.stream_memory is already a BytesIO object
audio_buffer = response.stream_memory
if audio_buffer is None:
raise ValueError("No audio data received from Deepgram")
# Read and yield the audio data in chunks
audio_buffer.seek(0) # Ensure we're at the start of the buffer
chunk_size = 1024 # Use a fixed buffer size
while True:
async for data in response.aiter_bytes():
await self.stop_ttfb_metrics()
chunk = audio_buffer.read(chunk_size)
if not chunk:
break
frame = TTSAudioRawFrame(audio=chunk, sample_rate=self.sample_rate, num_channels=1)
yield frame
if data:
yield TTSAudioRawFrame(audio=data, sample_rate=self.sample_rate, num_channels=1)
yield TTSStoppedFrame()
except Exception as e:

139
src/pipecat/services/elevenlabs/tts.py
View File

@@ -7,11 +7,12 @@
import asyncio
import base64
import json
import uuid
from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union
import aiohttp
from loguru import logger
from pydantic import BaseModel, model_validator
from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
@@ -26,8 +27,12 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleWordTTSService, WordTTSService
from pipecat.services.tts_service import (
AudioContextWordTTSService,
WordTTSService,
)
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
# See .env.example for ElevenLabs configuration needed
try:
@@ -159,26 +164,17 @@ def calculate_word_times(
return word_times
class ElevenLabsTTSService(InterruptibleWordTTSService):
class ElevenLabsTTSService(AudioContextWordTTSService):
class InputParams(BaseModel):
language: Optional[Language] = None
optimize_streaming_latency: Optional[str] = None
stability: Optional[float] = None
similarity_boost: Optional[float] = None
style: Optional[float] = None
use_speaker_boost: Optional[bool] = None
speed: Optional[float] = None
auto_mode: Optional[bool] = True
@model_validator(mode="after")
def validate_voice_settings(self):
stability = self.stability
similarity_boost = self.similarity_boost
if (stability is None) != (similarity_boost is None):
raise ValueError(
"Both 'stability' and 'similarity_boost' must be provided when using voice settings"
)
return self
enable_ssml_parsing: Optional[bool] = None
enable_logging: Optional[bool] = None
def __init__(
self,
@@ -220,13 +216,14 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
"language": self.language_to_service_language(params.language)
if params.language
else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
"speed": params.speed,
"auto_mode": str(params.auto_mode).lower(),
"enable_ssml_parsing": params.enable_ssml_parsing,
"enable_logging": params.enable_logging,
}
self.set_model_name(model)
self.set_voice(voice_id)
@@ -238,6 +235,8 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
self._started = False
self._cumulative_time = 0
# Context management for v1 multi API
self._context_id = None
self._receive_task = None
self._keepalive_task = None
@@ -253,15 +252,13 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
async def set_model(self, model: str):
await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
# No need to disconnect/reconnect for model changes with multi-context API
async def _update_settings(self, settings: Mapping[str, Any]):
prev_voice = self._voice_id
await super()._update_settings(settings)
# If voice changes, we don't need to reconnect, just use a new context
if not prev_voice == self._voice_id:
await self._disconnect()
await self._connect()
logger.info(f"Switching TTS voice to: [{self._voice_id}]")
async def start(self, frame: StartFrame):
@@ -278,8 +275,8 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
await self._disconnect()
async def flush_audio(self):
if self._websocket:
msg = {"text": " ", "flush": True}
if self._websocket and self._context_id:
msg = {"context_id": self._context_id, "flush": True}
await self._websocket.send(json.dumps(msg))
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
@@ -287,7 +284,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
self._started = False
if isinstance(frame, TTSStoppedFrame):
await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)])
await self.add_word_timestamps([("Reset", 0)])
async def _connect(self):
await self._connect_websocket()
@@ -319,10 +316,13 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
voice_id = self._voice_id
model = self.model_name
output_format = self._output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
url = f"{self._url}/v1/text-to-speech/{voice_id}/multi-stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
if self._settings["optimize_streaming_latency"]:
url += f"&optimize_streaming_latency={self._settings['optimize_streaming_latency']}"
if self._settings["enable_ssml_parsing"]:
url += f"&enable_ssml_parsing={self._settings['enable_ssml_parsing']}"
if self._settings["enable_logging"]:
url += f"&enable_logging={self._settings['enable_logging']}"
# Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS
language = self._settings["language"]
@@ -335,16 +335,10 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
)
# Set max websocket message size to 16MB for large audio responses
self._websocket = await websockets.connect(url, max_size=16 * 1024 * 1024)
self._websocket = await websockets.connect(
url, max_size=16 * 1024 * 1024, extra_headers={"xi-api-key": self._api_key}
)
# According to ElevenLabs, we should always start with a single space.
msg: Dict[str, Any] = {
"text": " ",
"xi_api_key": self._api_key,
}
if self._voice_settings:
msg["voice_settings"] = self._voice_settings
await self._websocket.send(json.dumps(msg))
except Exception as e:
logger.error(f"{self} initialization error: {e}")
self._websocket = None
@@ -356,12 +350,15 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
if self._websocket:
logger.debug("Disconnecting from ElevenLabs")
await self._websocket.send(json.dumps({"text": ""}))
# Close all contexts and the socket
if self._context_id:
await self._websocket.send(json.dumps({"close_socket": True}))
await self._websocket.close()
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
finally:
self._started = False
self._context_id = None
self._websocket = None
def _get_websocket(self):
@@ -369,9 +366,35 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
# Close the current context when interrupted without closing the websocket
if self._context_id and self._websocket:
logger.trace(f"Closing context {self._context_id} due to interruption")
try:
await self._websocket.send(
json.dumps({"context_id": self._context_id, "close_context": True})
)
except Exception as e:
logger.error(f"Error closing context on interruption: {e}")
self._context_id = None
self._started = False
async def _receive_messages(self):
async for message in self._get_websocket():
msg = json.loads(message)
# Check if this message belongs to the current context
# The default context may return null/None for context_id
received_ctx_id = msg.get("context_id")
if (
self._context_id is not None
and received_ctx_id is not None
and received_ctx_id != self._context_id
):
logger.trace(f"Ignoring message from different context: {received_ctx_id}")
continue
if msg.get("audio"):
await self.stop_ttfb_metrics()
self.start_word_timestamps()
@@ -383,21 +406,47 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
await self.add_word_timestamps(word_times)
self._cumulative_time = word_times[-1][1]
if msg.get("is_final"):
logger.trace(f"Received final message for context {received_ctx_id}")
# Context has finished
if self._context_id == received_ctx_id:
self._context_id = None
self._started = False
async def _keepalive_task_handler(self):
while True:
await asyncio.sleep(10)
try:
await self._send_text("")
# Send an empty message to keep the connection alive
if self._websocket and self._websocket.open:
await self._websocket.send(json.dumps({}))
except websockets.ConnectionClosed as e:
logger.warning(f"{self} keepalive error: {e}")
break
async def _send_text(self, text: str):
if self._websocket:
msg = {"text": text + " "}
await self._websocket.send(json.dumps(msg))
if not self._context_id:
# First message for a new context - need a space to initialize
msg = {"text": " ", "context_id": str(uuid.uuid4())}
# Add voice settings only in first message for a context
if self._voice_settings:
msg["voice_settings"] = self._voice_settings
await self._websocket.send(json.dumps(msg))
self._context_id = msg["context_id"]
logger.trace(f"Created new context {self._context_id}")
# Now send the actual text content
msg = {"text": text, "context_id": self._context_id}
await self._websocket.send(json.dumps(msg))
else:
# Continuing with an existing context
msg = {"text": text, "context_id": self._context_id}
await self._websocket.send(json.dumps(msg))
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
@@ -406,6 +455,13 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
await self._connect()
try:
# Close previous context if there was one
if self._context_id and not self._started:
await self._websocket.send(
json.dumps({"context_id": self._context_id, "close_context": True})
)
self._context_id = None
if not self._started:
await self.start_ttfb_metrics()
yield TTSStartedFrame()
@@ -417,8 +473,8 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
except Exception as e:
logger.error(f"{self} error sending message: {e}")
yield TTSStoppedFrame()
await self._disconnect()
await self._connect()
self._started = False
self._context_id = None
return
yield None
except Exception as e:
@@ -526,7 +582,7 @@ class ElevenLabsHttpTTSService(WordTTSService):
self._reset_state()
if isinstance(frame, TTSStoppedFrame):
await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)])
await self.add_word_timestamps([("Reset", 0)])
elif isinstance(frame, LLMFullResponseEndFrame):
# End of turn - reset previous text
@@ -591,6 +647,7 @@ class ElevenLabsHttpTTSService(WordTTSService):
return word_times
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using ElevenLabs streaming API with timestamps.

13
src/pipecat/services/fal/stt.py
View File

@@ -14,6 +14,7 @@ from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
import fal_client
@@ -211,6 +212,14 @@ class FalSTTService(SegmentedSTTService):
await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]")
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[str] = None
):
"""Handle a transcription result with tracing."""
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Transcribes an audio segment using Fal's Wizper API.
@@ -225,6 +234,9 @@ class FalSTTService(SegmentedSTTService):
Only non-empty transcriptions are yielded.
"""
try:
await self.start_processing_metrics()
await self.start_ttfb_metrics()
# Send to Fal directly (audio is already in WAV format from base class)
data_uri = fal_client.encode(audio, "audio/x-wav")
response = await self._fal_client.run(
@@ -235,6 +247,7 @@ class FalSTTService(SegmentedSTTService):
if response and "text" in response:
text = response["text"].strip()
if text: # Only yield non-empty text
await self._handle_transcription(text, True, self._settings["language"])
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text, "", time_now_iso8601(), Language(self._settings["language"])

2
src/pipecat/services/fish/tts.py
View File

@@ -24,6 +24,7 @@ from pipecat.frames.frames import (
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
import ormsgpack
@@ -186,6 +187,7 @@ class FishAudioTTSService(InterruptibleTTSService):
except Exception as e:
logger.error(f"Error processing message: {e}")
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating Fish TTS: [{text}]")
try:

100
src/pipecat/services/gemini_multimodal_live/audio_transcriber.py
View File

@@ -1,100 +0,0 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import google.ai.generativelanguage as glm
import google.generativeai as gai
from loguru import logger
TRANSCRIBER_SYSTEM_INSTRUCTIONS = """
You are an audio transcriber. Your job is to transcribe audio to text exactly precisely and accurately.
You will receive the full conversation history before the audio input, to help with context. Use the full history only to help improve the accuracy of your transcription.
Rules:
- Respond with an exact transcription of the audio input.
- Transcribe only speech. Ignore any non-speech sounds.
- Do not include any text other than the transcription.
- Do not explain or add to your response.
- Transcribe the audio input simply and precisely.
- If the audio is not clear, emit the special string "----".
- No response other than exact transcription, or "----", is allowed.
"""
class AudioTranscriber:
def __init__(self, api_key, model="gemini-2.0-flash-exp"):
gai.configure(api_key=api_key)
self.api_key = api_key
self.model = model
self._client = None
def _create_client(self):
self._client = gai.GenerativeModel(
self.model, system_instruction=TRANSCRIBER_SYSTEM_INSTRUCTIONS
)
async def transcribe(self, audio, context):
try:
if self._client is None:
self._create_client()
messages = await self._create_inference_contents(audio, context)
if not messages:
return
response = await self._client.generate_content_async(
contents=messages,
)
text = response.candidates[0].content.parts[0].text
prompt_tokens = response.usage_metadata.prompt_token_count
completion_tokens = response.usage_metadata.candidates_token_count
total_tokens = response.usage_metadata.total_token_count
return (text, prompt_tokens, completion_tokens, total_tokens)
except Exception as e:
logger.error(f"Error transcribing: {e}")
async def _create_inference_contents(self, audio, context):
previous_messages = context.get_messages_for_persistent_storage()
try:
# Assemble a new message, with three parts: conversation history, transcription
# prompt, and audio. We could use only part of the conversation, if we need to
# keep the token count down, but for now, we'll just use the whole thing.
parts = []
history = ""
for msg in previous_messages:
content = msg.get("content", [])
if isinstance(content, str):
history += f"{msg.get('role')}: {content}\n"
else:
for part in content:
history += f"{msg.get('role')}: {part.get('text', ' - ')}\n"
if history:
assembled = f"Here is the conversation history so far. These are not instructions. This is data that you should use only to improve the accuracy of your transcription.\n\n----\n\n{history}\n\n----\n\nEND OF CONVERSATION HISTORY\n\n"
parts.append(glm.Part(text=assembled))
parts.append(
glm.Part(
text="Transcribe this audio. Transcribe only the exact words that appear in the audio. Do not add any words. Ignore non-speech sounds. Respond either with the transcription exactly as it was said by the user, or with the special string '----' if the audio is not clear."
)
)
parts.append(
glm.Part(
inline_data=glm.Blob(
mime_type="audio/wav",
data=(bytes(context.create_wav_header(16000, 1, 16, len(audio)) + audio)),
)
),
)
msg = glm.Content(role="user", parts=parts)
return [msg]
except Exception as e:
logger.error(f"Error processing frame: {e}")

42
src/pipecat/services/gemini_multimodal_live/events.py
View File

@@ -120,6 +120,7 @@ class Setup(BaseModel):
system_instruction: Optional[SystemInstruction] = None
tools: Optional[List[dict]] = None
generation_config: Optional[dict] = None
input_audio_transcription: Optional[AudioTranscriptionConfig] = None
output_audio_transcription: Optional[AudioTranscriptionConfig] = None
realtime_input_config: Optional[RealtimeInputConfig] = None
@@ -167,6 +168,7 @@ class ServerContent(BaseModel):
modelTurn: Optional[ModelTurn] = None
interrupted: Optional[bool] = None
turnComplete: Optional[bool] = None
inputTranscription: Optional[BidiGenerateContentTranscription] = None
outputTranscription: Optional[BidiGenerateContentTranscription] = None
@@ -180,10 +182,43 @@ class ToolCall(BaseModel):
functionCalls: List[FunctionCall]
class Modality(str, Enum):
"""Modality types in token counts."""
UNSPECIFIED = "MODALITY_UNSPECIFIED"
TEXT = "TEXT"
IMAGE = "IMAGE"
AUDIO = "AUDIO"
VIDEO = "VIDEO"
class ModalityTokenCount(BaseModel):
"""Token count for a specific modality."""
modality: Modality
tokenCount: int
class UsageMetadata(BaseModel):
"""Usage metadata about the response."""
promptTokenCount: Optional[int] = None
cachedContentTokenCount: Optional[int] = None
responseTokenCount: Optional[int] = None
toolUsePromptTokenCount: Optional[int] = None
thoughtsTokenCount: Optional[int] = None
totalTokenCount: Optional[int] = None
promptTokensDetails: Optional[List[ModalityTokenCount]] = None
cacheTokensDetails: Optional[List[ModalityTokenCount]] = None
responseTokensDetails: Optional[List[ModalityTokenCount]] = None
toolUsePromptTokensDetails: Optional[List[ModalityTokenCount]] = None
class ServerEvent(BaseModel):
setupComplete: Optional[SetupComplete] = None
serverContent: Optional[ServerContent] = None
toolCall: Optional[ToolCall] = None
usageMetadata: Optional[UsageMetadata] = None
def parse_server_event(str):
@@ -193,3 +228,10 @@ def parse_server_event(str):
except Exception as e:
print(f"Error parsing server event: {e}")
return None
class ContextWindowCompressionConfig(BaseModel):
"""Configuration for context window compression."""
sliding_window: Optional[bool] = Field(default=True)
trigger_tokens: Optional[int] = Field(default=None)

202
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -4,7 +4,6 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import base64
import json
import time
@@ -59,10 +58,10 @@ from pipecat.services.openai.llm import (
OpenAIUserContextAggregator,
)
from pipecat.transcriptions.language import Language
from pipecat.utils.string import match_endofsentence
from pipecat.utils.time import time_now_iso8601
from . import events
from .audio_transcriber import AudioTranscriber
try:
import websockets
@@ -223,6 +222,14 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator):
class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator):
# The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output,
# but the GeminiMultimodalLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We
# need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames
# are process. This ensures that the context gets only one set of messages.
async def process_frame(self, frame: Frame, direction: FrameDirection):
if not isinstance(frame, LLMTextFrame):
await super().process_frame(frame, direction)
async def handle_user_image_frame(self, frame: UserImageRawFrame):
# We don't want to store any images in the context. Revisit this later
# when the API evolves.
@@ -265,6 +272,15 @@ class GeminiVADParams(BaseModel):
silence_duration_ms: Optional[int] = Field(default=None)
class ContextWindowCompressionParams(BaseModel):
"""Parameters for context window compression."""
enabled: bool = Field(default=False)
trigger_tokens: Optional[int] = Field(
default=None
) # None = use default (80% of context window)
class InputParams(BaseModel):
frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0)
max_tokens: Optional[int] = Field(default=4096, ge=1)
@@ -280,10 +296,37 @@ class InputParams(BaseModel):
default=GeminiMediaResolution.UNSPECIFIED
)
vad: Optional[GeminiVADParams] = Field(default=None)
context_window_compression: Optional[ContextWindowCompressionParams] = Field(default=None)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
class GeminiMultimodalLiveLLMService(LLMService):
"""Provides access to Google's Gemini Multimodal Live API.
This service enables real-time conversations with Gemini, supporting both
text and audio modalities. It handles voice transcription, streaming audio
responses, and tool usage.
Args:
api_key (str): Google AI API key
base_url (str, optional): API endpoint base URL. Defaults to
"generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent".
model (str, optional): Model identifier to use. Defaults to
"models/gemini-2.0-flash-live-001".
voice_id (str, optional): TTS voice identifier. Defaults to "Charon".
start_audio_paused (bool, optional): Whether to start with audio input paused.
Defaults to False.
start_video_paused (bool, optional): Whether to start with video input paused.
Defaults to False.
system_instruction (str, optional): System prompt for the model. Defaults to None.
tools (Union[List[dict], ToolsSchema], optional): Tools/functions available to the model.
Defaults to None.
params (InputParams, optional): Configuration parameters for the model.
Defaults to InputParams().
inference_on_context_initialization (bool, optional): Whether to generate a response
when context is first set. Defaults to True.
"""
# Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter
@@ -298,7 +341,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
start_video_paused: bool = False,
system_instruction: Optional[str] = None,
tools: Optional[Union[List[dict], ToolsSchema]] = None,
transcribe_user_audio: bool = False,
params: InputParams = InputParams(),
inference_on_context_initialization: bool = True,
**kwargs,
@@ -321,18 +363,16 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._context = None
self._websocket = None
self._receive_task = None
self._transcribe_audio_task = None
self._transcribe_audio_queue = asyncio.Queue()
self._disconnecting = False
self._api_session_ready = False
self._run_llm_when_api_session_ready = False
self._transcriber = AudioTranscriber(api_key)
self._transcribe_user_audio = transcribe_user_audio
self._user_is_speaking = False
self._bot_is_speaking = False
self._user_audio_buffer = bytearray()
self._user_transcription_buffer = ""
self._last_transcription_sent = ""
self._bot_audio_buffer = bytearray()
self._bot_text_buffer = ""
@@ -355,6 +395,9 @@ class GeminiMultimodalLiveLLMService(LLMService):
"language": self._language_code,
"media_resolution": params.media_resolution,
"vad": params.vad,
"context_window_compression": params.context_window_compression.model_dump()
if params.context_window_compression
else {},
"extra": params.extra if isinstance(params.extra, dict) else {},
}
@@ -414,7 +457,9 @@ class GeminiMultimodalLiveLLMService(LLMService):
#
async def _handle_interruption(self):
pass
self._bot_is_speaking = False
await self.push_frame(TTSStoppedFrame())
await self.push_frame(LLMFullResponseEndFrame())
async def _handle_user_started_speaking(self, frame):
self._user_is_speaking = True
@@ -422,7 +467,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
async def _handle_user_stopped_speaking(self, frame):
self._user_is_speaking = False
audio = self._user_audio_buffer
self._user_audio_buffer = bytearray()
if self._needs_turn_complete_message:
self._needs_turn_complete_message = False
@@ -430,34 +474,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
{"clientContent": {"turnComplete": True}}
)
await self.send_client_event(evt)
if self._transcribe_user_audio and self._context:
await self._transcribe_audio_queue.put(audio)
async def _handle_transcribe_user_audio(self, audio, context):
text = await self._transcribe_audio(audio, context)
if not text:
return
logger.debug(f"[Transcription:user] {text}")
context.add_message({"role": "user", "content": [{"type": "text", "text": text}]})
await self.push_frame(
TranscriptionFrame(text=text, user_id="user", timestamp=time_now_iso8601())
)
async def _transcribe_audio(self, audio, context):
(text, prompt_tokens, completion_tokens, total_tokens) = await self._transcriber.transcribe(
audio, context
)
if not text:
return ""
# The only usage metrics we have right now are for the transcriber LLM. The Live API is free.
await self.start_llm_usage_metrics(
LLMTokenUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
)
)
return text
#
# frame processing
@@ -535,7 +551,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
uri = f"wss://{self._base_url}?key={self._api_key}"
self._websocket = await websockets.connect(uri=uri)
self._receive_task = self.create_task(self._receive_task_handler())
self._transcribe_audio_task = self.create_task(self._transcribe_audio_handler())
# Create the basic configuration
config_data = {
@@ -557,10 +572,26 @@ class GeminiMultimodalLiveLLMService(LLMService):
},
"media_resolution": self._settings["media_resolution"].value,
},
"input_audio_transcription": {},
"output_audio_transcription": {},
}
}
# Add context window compression if enabled
if self._settings.get("context_window_compression", {}).get("enabled", False):
compression_config = {}
# Add sliding window (always true if compression is enabled)
compression_config["sliding_window"] = {}
# Add trigger_tokens if specified
trigger_tokens = self._settings.get("context_window_compression", {}).get(
"trigger_tokens"
)
if trigger_tokens is not None:
compression_config["trigger_tokens"] = trigger_tokens
config_data["setup"]["context_window_compression"] = compression_config
# Add VAD configuration if provided
if self._settings.get("vad"):
vad_config = {}
@@ -624,9 +655,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
if self._receive_task:
await self.cancel_task(self._receive_task, timeout=1.0)
self._receive_task = None
if self._transcribe_audio_task:
await self.cancel_task(self._transcribe_audio_task)
self._transcribe_audio_task = None
self._disconnecting = False
except Exception as e:
logger.error(f"{self} error disconnecting: {e}")
@@ -661,8 +689,11 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._handle_evt_setup_complete(evt)
elif evt.serverContent and evt.serverContent.modelTurn:
await self._handle_evt_model_turn(evt)
elif evt.serverContent and evt.serverContent.turnComplete:
elif evt.serverContent and evt.serverContent.turnComplete and evt.usageMetadata:
await self._handle_evt_turn_complete(evt)
await self._handle_evt_usage_metadata(evt)
elif evt.serverContent and evt.serverContent.inputTranscription:
await self._handle_evt_input_transcription(evt)
elif evt.serverContent and evt.serverContent.outputTranscription:
await self._handle_evt_output_transcription(evt)
elif evt.toolCall:
@@ -674,11 +705,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
else:
pass
async def _transcribe_audio_handler(self):
while True:
audio = await self._transcribe_audio_queue.get()
await self._handle_transcribe_user_audio(audio, self._context)
#
#
#
@@ -811,6 +837,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
if not part:
return
# part.text is added when `modalities` is set to TEXT; otherwise, it's None
text = part.text
if text:
if not self._bot_text_buffer:
@@ -833,6 +860,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
if not self._bot_is_speaking:
self._bot_is_speaking = True
await self.push_frame(TTSStartedFrame())
await self.push_frame(LLMFullResponseStartFrame())
self._bot_audio_buffer.extend(audio)
frame = TTSAudioRawFrame(
@@ -861,21 +889,83 @@ class GeminiMultimodalLiveLLMService(LLMService):
text = self._bot_text_buffer
self._bot_text_buffer = ""
if text:
await self.push_frame(LLMFullResponseEndFrame())
# Only push the TTSStoppedFrame the bot is outputting audio
# when text is found, modalities is set to TEXT and no audio
# is produced.
if not text:
await self.push_frame(TTSStoppedFrame())
await self.push_frame(TTSStoppedFrame())
await self.push_frame(LLMFullResponseEndFrame())
async def _handle_evt_input_transcription(self, evt):
"""Handle the input transcription event.
Gemini Live sends user transcriptions in either single words or multi-word
phrases. As a result, we have to aggregate the input transcription. This handler
aggregates into sentences, splitting on the end of sentence markers.
"""
if not evt.serverContent.inputTranscription:
return
text = evt.serverContent.inputTranscription.text
if not text:
return
# Strip leading space from sentence starts if buffer is empty
if text.startswith(" ") and not self._user_transcription_buffer:
text = text.lstrip()
# Accumulate text in the buffer
self._user_transcription_buffer += text
# Check for complete sentences
while True:
eos_end_marker = match_endofsentence(self._user_transcription_buffer)
if not eos_end_marker:
break
# Extract the complete sentence
complete_sentence = self._user_transcription_buffer[:eos_end_marker]
# Keep the remainder for the next chunk
self._user_transcription_buffer = self._user_transcription_buffer[eos_end_marker:]
# Send a TranscriptionFrame with the complete sentence
logger.debug(f"[Transcription:user] [{complete_sentence}]")
await self.push_frame(
TranscriptionFrame(
text=complete_sentence, user_id="", timestamp=time_now_iso8601()
),
FrameDirection.UPSTREAM,
)
async def _handle_evt_output_transcription(self, evt):
if not evt.serverContent.outputTranscription:
return
# This is the output transcription text when modalities is set to AUDIO.
# In this case, we push LLMTextFrame and TTSTextFrame to be handled by the
# downstream assistant context aggregator.
text = evt.serverContent.outputTranscription.text
if text:
await self.push_frame(LLMFullResponseStartFrame())
await self.push_frame(LLMTextFrame(text=text))
await self.push_frame(TTSTextFrame(text=text))
await self.push_frame(LLMFullResponseEndFrame())
if not text:
return
await self.push_frame(LLMTextFrame(text=text))
await self.push_frame(TTSTextFrame(text=text))
async def _handle_evt_usage_metadata(self, evt):
if not evt.usageMetadata:
return
usage = evt.usageMetadata
tokens = LLMTokenUsage(
prompt_tokens=usage.promptTokenCount,
completion_tokens=usage.responseTokenCount,
total_tokens=usage.totalTokenCount,
)
await self.start_llm_usage_metrics(tokens)
def create_context_aggregator(
self,
@@ -906,6 +996,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
GeminiMultimodalLiveContext.upgrade(context)
user = GeminiMultimodalLiveUserContextAggregator(context, params=user_params)
assistant_params.expect_stripped_words = True
assistant_params.expect_stripped_words = False
assistant = GeminiMultimodalLiveAssistantContextAggregator(context, params=assistant_params)
return GeminiMultimodalLiveContextAggregatorPair(_user=user, _assistant=assistant)

25
src/pipecat/services/gladia/stt.py
View File

@@ -26,6 +26,7 @@ from pipecat.services.gladia.config import GladiaInputParams
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
import websockets
@@ -227,6 +228,10 @@ class GladiaSTTService(STTService):
self._websocket = None
self._receive_task = None
self._keepalive_task = None
self._settings = {}
def can_generate_metrics(self) -> bool:
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to Gladia's language code."""
@@ -278,6 +283,9 @@ class GladiaSTTService(STTService):
if self._params.messages_config:
settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True)
# Store settings for tracing
self._settings = settings
return settings
async def start(self, frame: StartFrame):
@@ -328,9 +336,9 @@ class GladiaSTTService(STTService):
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Run speech-to-text on audio data."""
await self.start_ttfb_metrics()
await self.start_processing_metrics()
await self._send_audio(audio)
await self.stop_processing_metrics()
yield None
async def _setup_gladia(self, settings: Dict[str, Any]):
@@ -351,6 +359,13 @@ class GladiaSTTService(STTService):
f"Failed to initialize Gladia session: {response.status} - {error_text}"
)
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[str] = None
):
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
async def _send_audio(self, audio: bytes):
data = base64.b64encode(audio).decode("utf-8")
message = {"type": "audio_chunk", "data": {"chunk": data}}
@@ -387,11 +402,17 @@ class GladiaSTTService(STTService):
confidence = utterance.get("confidence", 0)
language = utterance["language"]
transcript = utterance["text"]
is_final = content["data"]["is_final"]
if confidence >= self._confidence:
if content["data"]["is_final"]:
if is_final:
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), language)
)
await self._handle_transcription(
transcript=transcript,
is_final=is_final,
language=language,
)
else:
await self.push_frame(
InterimTranscriptionFrame(

284
src/pipecat/services/google/llm.py
View File

@@ -47,15 +47,22 @@ from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.utils.tracing.service_decorators import traced_llm
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
try:
import google.ai.generativelanguage as glm
import google.generativeai as gai
from google import genai
from google.api_core.exceptions import DeadlineExceeded
from google.generativeai.types import GenerationConfig
from google.genai.types import (
Blob,
Content,
FunctionCall,
FunctionResponse,
GenerateContentConfig,
Part,
)
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
@@ -65,9 +72,7 @@ except ModuleNotFoundError as e:
class GoogleUserContextAggregator(OpenAIUserContextAggregator):
async def push_aggregation(self):
if len(self._aggregation) > 0:
self._context.add_message(
glm.Content(role="user", parts=[glm.Part(text=self._aggregation)])
)
self._context.add_message(Content(role="user", parts=[Part(text=self._aggregation)]))
# Reset the aggregation. Reset it before pushing it down, otherwise
# if the tasks gets cancelled we won't be able to clear things up.
@@ -83,15 +88,15 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator):
class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
async def handle_aggregation(self, aggregation: str):
self._context.add_message(glm.Content(role="model", parts=[glm.Part(text=aggregation)]))
self._context.add_message(Content(role="model", parts=[Part(text=aggregation)]))
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
self._context.add_message(
glm.Content(
Content(
role="model",
parts=[
glm.Part(
function_call=glm.FunctionCall(
Part(
function_call=FunctionCall(
id=frame.tool_call_id, name=frame.function_name, args=frame.arguments
)
)
@@ -99,11 +104,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
)
)
self._context.add_message(
glm.Content(
Content(
role="user",
parts=[
glm.Part(
function_response=glm.FunctionResponse(
Part(
function_response=FunctionResponse(
id=frame.tool_call_id,
name=frame.function_name,
response={"response": "IN_PROGRESS"},
@@ -187,7 +192,7 @@ class GoogleLLMContext(OpenAILLMContext):
# Convert each message individually
converted_messages = []
for msg in messages:
if isinstance(msg, glm.Content):
if isinstance(msg, Content):
# Already in Gemini format
converted_messages.append(msg)
else:
@@ -202,7 +207,7 @@ class GoogleLLMContext(OpenAILLMContext):
def get_messages_for_logging(self):
msgs = []
for message in self.messages:
obj = glm.Content.to_dict(message)
obj = message.to_json_dict()
try:
if "parts" in obj:
for part in obj["parts"]:
@@ -221,10 +226,10 @@ class GoogleLLMContext(OpenAILLMContext):
parts = []
if text:
parts.append(glm.Part(text=text))
parts.append(glm.Part(inline_data=glm.Blob(mime_type="image/jpeg", data=buffer.getvalue())))
parts.append(Part(text=text))
parts.append(Part(inline_data=Blob(mime_type="image/jpeg", data=buffer.getvalue())))
self.add_message(glm.Content(role="user", parts=parts))
self.add_message(Content(role="user", parts=parts))
def add_audio_frames_message(
self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows"
@@ -239,10 +244,10 @@ class GoogleLLMContext(OpenAILLMContext):
data = b"".join(frame.audio for frame in audio_frames)
# NOTE(aleix): According to the docs only text or inline_data should be needed.
# (see https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)
parts.append(glm.Part(text=text))
parts.append(Part(text=text))
parts.append(
glm.Part(
inline_data=glm.Blob(
Part(
inline_data=Blob(
mime_type="audio/wav",
data=(
bytes(
@@ -252,7 +257,7 @@ class GoogleLLMContext(OpenAILLMContext):
)
),
)
self.add_message(glm.Content(role="user", parts=parts))
self.add_message(Content(role="user", parts=parts))
# message = {"mime_type": "audio/mp3", "data": bytes(data + create_wav_header(sample_rate, num_channels, 16, len(data)))}
# self.add_message(message)
@@ -271,7 +276,7 @@ class GoogleLLMContext(OpenAILLMContext):
}
Returns:
glm.Content object with:
Content object with:
- role: "user" or "model" (converted from "assistant")
- parts: List[Part] containing text, inline_data, or function calls
Returns None for system messages.
@@ -288,8 +293,8 @@ class GoogleLLMContext(OpenAILLMContext):
if message.get("tool_calls"):
for tc in message["tool_calls"]:
parts.append(
glm.Part(
function_call=glm.FunctionCall(
Part(
function_call=FunctionCall(
name=tc["function"]["name"],
args=json.loads(tc["function"]["arguments"]),
)
@@ -298,30 +303,30 @@ class GoogleLLMContext(OpenAILLMContext):
elif role == "tool":
role = "model"
parts.append(
glm.Part(
function_response=glm.FunctionResponse(
Part(
function_response=FunctionResponse(
name="tool_call_result", # seems to work to hard-code the same name every time
response=json.loads(message["content"]),
)
)
)
elif isinstance(content, str):
parts.append(glm.Part(text=content))
parts.append(Part(text=content))
elif isinstance(content, list):
for c in content:
if c["type"] == "text":
parts.append(glm.Part(text=c["text"]))
parts.append(Part(text=c["text"]))
elif c["type"] == "image_url":
parts.append(
glm.Part(
inline_data=glm.Blob(
Part(
inline_data=Blob(
mime_type="image/jpeg",
data=base64.b64decode(c["image_url"]["url"].split(",")[1]),
)
)
)
message = glm.Content(role=role, parts=parts)
message = Content(role=role, parts=parts)
return message
def to_standard_messages(self, obj) -> list:
@@ -409,7 +414,7 @@ class GoogleLLMContext(OpenAILLMContext):
# Process each message, preserving Google-formatted messages and converting others
for message in self._messages:
if isinstance(message, glm.Content):
if isinstance(message, Content):
# Keep existing Google-formatted messages (e.g., function calls/responses)
converted_messages.append(message)
continue
@@ -433,9 +438,7 @@ class GoogleLLMContext(OpenAILLMContext):
# Add system message back as a user message if we only have function messages
if self.system_message and not has_regular_messages:
self._messages.append(
glm.Content(role="user", parts=[glm.Part(text=self.system_message)])
)
self._messages.append(Content(role="user", parts=[Part(text=self.system_message)]))
# Remove any empty messages
self._messages = [m for m in self._messages if m.parts]
@@ -463,7 +466,7 @@ class GoogleLLMService(LLMService):
self,
*,
api_key: str,
model: str = "gemini-2.0-flash-001",
model: str = "gemini-2.0-flash",
params: InputParams = InputParams(),
system_instruction: Optional[str] = None,
tools: Optional[List[Dict[str, Any]]] = None,
@@ -471,10 +474,10 @@ class GoogleLLMService(LLMService):
**kwargs,
):
super().__init__(**kwargs)
gai.configure(api_key=api_key)
self.set_model_name(model)
self._api_key = api_key
self._system_instruction = system_instruction
self._create_client()
self._create_client(api_key)
self._settings = {
"max_tokens": params.max_tokens,
"temperature": params.temperature,
@@ -488,11 +491,10 @@ class GoogleLLMService(LLMService):
def can_generate_metrics(self) -> bool:
return True
def _create_client(self):
self._client = gai.GenerativeModel(
self._model_name, system_instruction=self._system_instruction
)
def _create_client(self, api_key: str):
self._client = genai.Client(api_key=api_key)
@traced_llm
async def _process_context(self, context: OpenAILLMContext):
await self.push_frame(LLMFullResponseStartFrame())
@@ -513,23 +515,7 @@ class GoogleLLMService(LLMService):
if context.system_message and self._system_instruction != context.system_message:
logger.debug(f"System instruction changed: {context.system_message}")
self._system_instruction = context.system_message
self._create_client()
# Filter out None values and create GenerationConfig
generation_params = {
k: v
for k, v in {
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"top_k": self._settings["top_k"],
"max_output_tokens": self._settings["max_tokens"],
}.items()
if v is not None
}
generation_config = GenerationConfig(**generation_params) if generation_params else None
await self.start_ttfb_metrics()
tools = []
if context.tools:
tools = context.tools
@@ -538,112 +524,104 @@ class GoogleLLMService(LLMService):
tool_config = None
if self._tool_config:
tool_config = self._tool_config
response = await self._client.generate_content_async(
# Filter out None values and create GenerationContentConfig
generation_params = {
k: v
for k, v in {
"system_instruction": self._system_instruction,
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
"top_k": self._settings["top_k"],
"max_output_tokens": self._settings["max_tokens"],
"tools": tools,
"tool_config": tool_config,
}.items()
if v is not None
}
generation_config = (
GenerateContentConfig(**generation_params) if generation_params else None
)
await self.start_ttfb_metrics()
response = await self._client.aio.models.generate_content_stream(
model=self._model_name,
contents=messages,
tools=tools,
stream=True,
generation_config=generation_config,
tool_config=tool_config,
config=generation_config,
)
await self.stop_ttfb_metrics()
if response.usage_metadata:
# Use only the prompt token count from the response object
prompt_tokens = response.usage_metadata.prompt_token_count
total_tokens = prompt_tokens
async for chunk in response:
if chunk.usage_metadata:
# Use only the completion_tokens from the chunks. Prompt tokens are already counted and
# are repeated here.
completion_tokens += chunk.usage_metadata.candidates_token_count
total_tokens += chunk.usage_metadata.candidates_token_count
try:
for c in chunk.parts:
if c.text:
search_result += c.text
await self.push_frame(LLMTextFrame(c.text))
elif c.function_call:
logger.debug(f"Function call: {c.function_call}")
args = type(c.function_call).to_dict(c.function_call).get("args", {})
await self.call_function(
context=context,
tool_call_id=str(uuid.uuid4()),
function_name=c.function_call.name,
arguments=args,
)
# Handle grounding metadata
# It seems only the last chunk that we receive may contain this information
# If the response doesn't include groundingMetadata, this means the response wasn't grounded.
if chunk.candidates:
for candidate in chunk.candidates:
# logger.debug(f"candidate received: {candidate}")
# Extract grounding metadata
grounding_metadata = (
{
"rendered_content": getattr(
getattr(candidate, "grounding_metadata", None),
"search_entry_point",
None,
).rendered_content
if hasattr(
getattr(candidate, "grounding_metadata", None),
"search_entry_point",
)
else None,
"origins": [
{
"site_uri": getattr(grounding_chunk.web, "uri", None),
"site_title": getattr(
grounding_chunk.web, "title", None
),
"results": [
{
"text": getattr(
grounding_support.segment, "text", ""
),
"confidence": getattr(
grounding_support, "confidence_scores", None
),
}
for grounding_support in getattr(
getattr(candidate, "grounding_metadata", None),
"grounding_supports",
[],
)
if index
in getattr(
grounding_support, "grounding_chunk_indices", []
)
],
}
for index, grounding_chunk in enumerate(
getattr(
getattr(candidate, "grounding_metadata", None),
"grounding_chunks",
[],
)
)
],
}
if getattr(candidate, "grounding_metadata", None)
else None
)
except Exception as e:
# Google LLMs seem to flag safety issues a lot!
if chunk.candidates[0].finish_reason == 3:
logger.debug(
f"LLM refused to generate content for safety reasons - {messages}."
)
else:
logger.exception(f"{self} error: {e}")
prompt_tokens += chunk.usage_metadata.prompt_token_count or 0
completion_tokens += chunk.usage_metadata.candidates_token_count or 0
total_tokens += chunk.usage_metadata.total_token_count or 0
if not chunk.candidates:
continue
for candidate in chunk.candidates:
if candidate.content and candidate.content.parts:
for part in candidate.content.parts:
if not part.thought and part.text:
search_result += part.text
await self.push_frame(LLMTextFrame(part.text))
elif part.function_call:
function_call = part.function_call
id = function_call.id or str(uuid.uuid4())
logger.debug(f"Function call: {function_call.name}:{id}")
await self.call_function(
context=context,
tool_call_id=id,
function_name=function_call.name,
arguments=function_call.args or {},
)
if (
candidate.grounding_metadata
and candidate.grounding_metadata.grounding_chunks
):
m = candidate.grounding_metadata
rendered_content = (
m.search_entry_point.rendered_content if m.search_entry_point else None
)
origins = [
{
"site_uri": grounding_chunk.web.uri
if grounding_chunk.web
else None,
"site_title": grounding_chunk.web.title
if grounding_chunk.web
else None,
"results": [
{
"text": grounding_support.segment.text
if grounding_support.segment
else "",
"confidence": grounding_support.confidence_scores,
}
for grounding_support in (
m.grounding_supports if m.grounding_supports else []
)
if grounding_support.grounding_chunk_indices
and index in grounding_support.grounding_chunk_indices
],
}
for index, grounding_chunk in enumerate(
m.grounding_chunks if m.grounding_chunks else []
)
]
grounding_metadata = {
"rendered_content": rendered_content,
"origins": origins,
}
except DeadlineExceeded:
await self._call_event_handler("on_completion_timeout")
except Exception as e:
logger.exception(f"{self} exception: {e}")
finally:
if grounding_metadata is not None and isinstance(grounding_metadata, dict):
if grounding_metadata and isinstance(grounding_metadata, dict):
llm_search_frame = LLMSearchResponseFrame(
search_result=search_result,
origins=grounding_metadata["origins"],

18
src/pipecat/services/google/rtvi.py
View File

@@ -9,8 +9,9 @@ from typing import List, Literal, Optional
from pydantic import BaseModel
from pipecat.frames.frames import Frame
from pipecat.observers.base_observer import FramePushed
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frameworks.rtvi import RTVIObserver
from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor
from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame
@@ -27,18 +28,13 @@ class RTVIBotLLMSearchResponseMessage(BaseModel):
class GoogleRTVIObserver(RTVIObserver):
def __init__(self, rtvi: FrameProcessor):
def __init__(self, rtvi: RTVIProcessor):
super().__init__(rtvi)
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
await super().on_push_frame(src, dst, frame, direction, timestamp)
async def on_push_frame(self, data: FramePushed):
await super().on_push_frame(data)
frame = data.frame
if isinstance(frame, LLMSearchResponseFrame):
await self._handle_llm_search_response_frame(frame)

20
src/pipecat/services/google/stt.py
View File

@@ -9,6 +9,8 @@ import json
import os
import time
from pipecat.utils.tracing.service_decorators import traced_stt
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
@@ -496,6 +498,9 @@ class GoogleSTTService(STTService):
"enable_voice_activity_events": params.enable_voice_activity_events,
}
def can_generate_metrics(self) -> bool:
return True
def language_to_service_language(self, language: Language | List[Language]) -> str | List[str]:
"""Convert Language enum(s) to Google STT language code(s).
@@ -773,9 +778,17 @@ class GoogleSTTService(STTService):
"""Process an audio chunk for STT transcription."""
if self._streaming_task:
# Queue the audio data
await self.start_ttfb_metrics()
await self.start_processing_metrics()
await self._request_queue.put(audio)
yield None
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[str] = None
):
pass
async def _process_responses(self, streaming_recognize):
"""Process streaming recognition responses."""
try:
@@ -803,8 +816,15 @@ class GoogleSTTService(STTService):
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), primary_language)
)
await self.stop_processing_metrics()
await self._handle_transcription(
transcript,
is_final=True,
language=primary_language,
)
else:
self._last_transcript_was_final = False
await self.stop_ttfb_metrics()
await self.push_frame(
InterimTranscriptionFrame(
transcript, "", time_now_iso8601(), primary_language

3
src/pipecat/services/google/tts.py
View File

@@ -8,6 +8,8 @@ import asyncio
import json
import os
from pipecat.utils.tracing.service_decorators import traced_tts
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
@@ -318,6 +320,7 @@ class GoogleTTSService(TTSService):
return ssml
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")

12
src/pipecat/services/groq/tts.py
View File

@@ -12,6 +12,7 @@ from pydantic import BaseModel
from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
from groq import AsyncGroq
@@ -25,7 +26,6 @@ class GroqTTSService(TTSService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0
seed: Optional[int] = None
GROQ_SAMPLE_RATE = 48000 # Groq TTS only supports 48kHz sample rate
@@ -54,11 +54,21 @@ class GroqTTSService(TTSService):
self._voice_id = voice_id
self._params = params
self._settings = {
"model": model_name,
"voice_id": voice_id,
"output_format": output_format,
"language": str(params.language) if params.language else "en",
"speed": params.speed,
"sample_rate": sample_rate,
}
self._client = AsyncGroq(api_key=self._api_key)
def can_generate_metrics(self) -> bool:
return True
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
measuring_ttfb = True

2
src/pipecat/services/llm_service.py
View File

@@ -190,6 +190,7 @@ class LLMService(AIService):
function_name: Optional[str] = None,
tool_call_id: Optional[str] = None,
text_content: Optional[str] = None,
video_source: Optional[str] = None,
):
await self.push_frame(
UserImageRequestFrame(
@@ -197,6 +198,7 @@ class LLMService(AIService):
function_name=function_name,
tool_call_id=tool_call_id,
context=text_content,
video_source=video_source,
),
FrameDirection.UPSTREAM,
)

19
src/pipecat/services/lmnt/tts.py
View File

@@ -23,6 +23,7 @@ from pipecat.frames.frames import (
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
# See .env.example for LMNT configuration needed
try:
@@ -39,8 +40,20 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
Language.EN: "en",
Language.ES: "es",
Language.FR: "fr",
Language.HI: "hi",
Language.ID: "id",
Language.IT: "it",
Language.JA: "ja",
Language.KO: "ko",
Language.NL: "nl",
Language.PL: "pl",
Language.PT: "pt",
Language.RU: "ru",
Language.SV: "sv",
Language.TH: "th",
Language.TR: "tr",
Language.UK: "uk",
Language.VI: "vi",
Language.ZH: "zh",
}
@@ -65,6 +78,7 @@ class LmntTTSService(InterruptibleTTSService):
voice_id: str,
sample_rate: Optional[int] = None,
language: Language = Language.EN,
model: str = "aurora",
**kwargs,
):
super().__init__(
@@ -75,7 +89,8 @@ class LmntTTSService(InterruptibleTTSService):
)
self._api_key = api_key
self._voice_id = voice_id
self.set_voice(voice_id)
self.set_model_name(model)
self._settings = {
"language": self.language_to_service_language(language),
"format": "raw", # Use raw format for direct PCM data
@@ -134,6 +149,7 @@ class LmntTTSService(InterruptibleTTSService):
"format": self._settings["format"],
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"model": self.model_name,
}
# Connect to LMNT's websocket directly
@@ -198,6 +214,7 @@ class LmntTTSService(InterruptibleTTSService):
except json.JSONDecodeError:
logger.error(f"Invalid JSON message: {message}")
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text."""
logger.debug(f"{self}: Generating TTS [{text}]")

55
src/pipecat/services/mem0/memory.py
View File

@@ -17,7 +17,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
try:
from mem0 import MemoryClient # noqa: F401
from mem0 import Memory, MemoryClient # noqa: F401
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
@@ -49,7 +49,8 @@ class Mem0MemoryService(FrameProcessor):
def __init__(
self,
*,
api_key: str,
api_key: str = None,
local_config: Dict[str, Any] = {},
user_id: str = None,
agent_id: str = None,
run_id: str = None,
@@ -58,7 +59,10 @@ class Mem0MemoryService(FrameProcessor):
# Important: Call the parent class __init__ first
super().__init__()
self.memory_client = MemoryClient(api_key=api_key)
if local_config:
self.memory_client = Memory.from_config(local_config)
else:
self.memory_client = MemoryClient(api_key=api_key)
# At least one of user_id, agent_id, or run_id must be provided
if not any([user_id, agent_id, run_id]):
raise ValueError("At least one of user_id, agent_id, or run_id must be provided")
@@ -91,6 +95,9 @@ class Mem0MemoryService(FrameProcessor):
for id in ["user_id", "agent_id", "run_id"]:
if getattr(self, id):
params[id] = getattr(self, id)
if isinstance(self.memory_client, Memory):
del params["output_format"]
# Note: You can run this in background to avoid blocking the conversation
self.memory_client.add(**params)
except Exception as e:
@@ -107,20 +114,32 @@ class Mem0MemoryService(FrameProcessor):
"""
try:
logger.debug(f"Retrieving memories for query: {query}")
id_pairs = [
("user_id", self.user_id),
("agent_id", self.agent_id),
("run_id", self.run_id),
]
clauses = [{name: value} for name, value in id_pairs if value is not None]
filters = {"AND": clauses} if clauses else {}
results = self.memory_client.search(
query=query,
filters=filters,
version=self.api_version,
top_k=self.search_limit,
threshold=self.search_threshold,
)
if isinstance(self.memory_client, Memory):
params = {
"query": query,
"user_id": self.user_id,
"agent_id": self.agent_id,
"run_id": self.run_id,
"limit": self.search_limit,
}
params = {k: v for k, v in params.items() if v is not None}
results = self.memory_client.search(**params)
else:
id_pairs = [
("user_id", self.user_id),
("agent_id", self.agent_id),
("run_id", self.run_id),
]
clauses = [{name: value} for name, value in id_pairs if value is not None]
filters = {"AND": clauses} if clauses else {}
results = self.memory_client.search(
query=query,
filters=filters,
version=self.api_version,
top_k=self.search_limit,
threshold=self.search_threshold,
output_format="v1.1",
)
logger.debug(f"Retrieved {len(results)} memories from Mem0")
return results
@@ -147,7 +166,7 @@ class Mem0MemoryService(FrameProcessor):
# Format memories as a message
memory_text = self.system_prompt
for i, memory in enumerate(memories, 1):
for i, memory in enumerate(memories["results"], 1):
memory_text += f"{i}. {memory.get('memory', '')}\n\n"
# Add memories as a system message or user message based on configuration

8
src/pipecat/services/minimax/__init__.py Normal file
View File

@@ -0,0 +1,8 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from .tts import *

298
src/pipecat/services/minimax/tts.py Normal file
View File

@@ -0,0 +1,298 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import json
from typing import AsyncGenerator, Optional
import aiohttp
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
ErrorFrame,
Frame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
def language_to_minimax_language(language: Language) -> Optional[str]:
BASE_LANGUAGES = {
Language.AR: "Arabic",
Language.CS: "Czech",
Language.DE: "German",
Language.EL: "Greek",
Language.EN: "English",
Language.ES: "Spanish",
Language.FI: "Finnish",
Language.FR: "French",
Language.HI: "Hindi",
Language.ID: "Indonesian",
Language.IT: "Italian",
Language.JA: "Japanese",
Language.KO: "Korean",
Language.NL: "Dutch",
Language.PL: "Polish",
Language.PT: "Portuguese",
Language.RO: "Romanian",
Language.RU: "Russian",
Language.TH: "Thai",
Language.TR: "Turkish",
Language.UK: "Ukrainian",
Language.VI: "Vietnamese",
Language.YUE: "Chinese,Yue",
Language.ZH: "Chinese",
}
result = BASE_LANGUAGES.get(language)
# If not found in base languages, try to find the base language from a variant
if not result:
# Convert enum value to string and get the base language part (e.g. es-ES -> es)
lang_str = str(language.value)
base_code = lang_str.split("-")[0].lower()
# Find matching language
for code, name in BASE_LANGUAGES.items():
if str(code.value).lower().startswith(base_code):
result = name
break
return result
class MiniMaxHttpTTSService(TTSService):
"""Text-to-speech service using MiniMax's T2A (Text-to-Audio) API.
Platform documentation:
https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643
Args:
api_key: MiniMax API key for authentication.
group_id: MiniMax Group ID to identify project.
model: TTS model name (default: "speech-02-turbo"). Options include
"speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo".
voice_id: Voice identifier (default: "Calm_Woman").
aiohttp_session: aiohttp.ClientSession for API communication.
sample_rate: Output audio sample rate in Hz (default: None, set from pipeline).
params: Additional configuration parameters.
"""
class InputParams(BaseModel):
"""Configuration parameters for MiniMax TTS.
Attributes:
language: Language for TTS generation.
speed: Speech speed (range: 0.5 to 2.0).
volume: Speech volume (range: 0 to 10).
pitch: Pitch adjustment (range: -12 to 12).
emotion: Emotional tone (options: "happy", "sad", "angry", "fearful",
"disgusted", "surprised", "neutral").
english_normalization: Whether to apply English text normalization.
"""
language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0
volume: Optional[float] = 1.0
pitch: Optional[float] = 0
emotion: Optional[str] = None
english_normalization: Optional[bool] = None
def __init__(
self,
*,
api_key: str,
group_id: str,
model: str = "speech-02-turbo",
voice_id: str = "Calm_Woman",
aiohttp_session: aiohttp.ClientSession,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._group_id = group_id
self._base_url = f"https://api.minimaxi.chat/v1/t2a_v2?GroupId={group_id}"
self._session = aiohttp_session
self._model_name = model
self._voice_id = voice_id
# Create voice settings
self._settings = {
"stream": True,
"voice_setting": {
"speed": params.speed,
"vol": params.volume,
"pitch": params.pitch,
},
"audio_setting": {
"bitrate": 128000,
"format": "pcm",
"channel": 1,
},
}
# Set voice and model
self.set_voice(voice_id)
self.set_model_name(model)
# Add language boost if provided
if params.language:
service_lang = self.language_to_service_language(params.language)
if service_lang:
self._settings["language_boost"] = service_lang
# Add optional emotion if provided
if params.emotion:
# Validate emotion is in the supported list
supported_emotions = [
"happy",
"sad",
"angry",
"fearful",
"disgusted",
"surprised",
"neutral",
]
if params.emotion in supported_emotions:
self._settings["voice_setting"]["emotion"] = params.emotion
else:
logger.warning(f"Unsupported emotion: {params.emotion}. Using default.")
# Add english_normalization if provided
if params.english_normalization is not None:
self._settings["english_normalization"] = params.english_normalization
def can_generate_metrics(self) -> bool:
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
return language_to_minimax_language(language)
def set_model_name(self, model: str):
"""Set the TTS model to use"""
self._model_name = model
def set_voice(self, voice: str):
"""Set the voice to use"""
self._voice_id = voice
if "voice_setting" in self._settings:
self._settings["voice_setting"]["voice_id"] = voice
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["audio_setting"]["sample_rate"] = self.sample_rate
logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}")
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
headers = {
"accept": "application/json, text/plain, */*",
"Content-Type": "application/json",
"Authorization": f"Bearer {self._api_key}",
}
# Create payload from settings
payload = self._settings.copy()
payload["model"] = self._model_name
payload["text"] = text
try:
await self.start_ttfb_metrics()
async with self._session.post(
self._base_url, headers=headers, json=payload
) as response:
if response.status != 200:
error_message = f"MiniMax TTS error: HTTP {response.status}"
logger.error(error_message)
yield ErrorFrame(error=error_message)
return
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
# Process the streaming response
buffer = bytearray()
CHUNK_SIZE = 1024
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
if not chunk:
continue
buffer.extend(chunk)
# Find complete data blocks
while b"data:" in buffer:
start = buffer.find(b"data:")
next_start = buffer.find(b"data:", start + 5)
if next_start == -1:
# No next data block found, keep current data for next iteration
if start > 0:
buffer = buffer[start:]
break
# Extract a complete data block
data_block = buffer[start:next_start]
buffer = buffer[next_start:]
try:
data = json.loads(data_block[5:].decode("utf-8"))
# Skip data blocks containing extra_info
if "extra_info" in data:
logger.debug("Received final chunk with extra info")
continue
chunk_data = data.get("data", {})
if not chunk_data:
continue
audio_data = chunk_data.get("audio")
if not audio_data:
continue
# Process audio data in chunks
for i in range(0, len(audio_data), CHUNK_SIZE * 2): # *2 for hex string
# Split hex string
hex_chunk = audio_data[i : i + CHUNK_SIZE * 2]
if not hex_chunk:
continue
try:
# Convert this chunk of data
audio_chunk = bytes.fromhex(hex_chunk)
if audio_chunk:
await self.stop_ttfb_metrics()
yield TTSAudioRawFrame(
audio=audio_chunk,
sample_rate=self._settings["audio_setting"][
"sample_rate"
],
num_channels=self._settings["audio_setting"]["channel"],
)
except ValueError as e:
logger.error(f"Error converting hex to binary: {e}")
continue
except json.JSONDecodeError as e:
logger.error(f"Error decoding JSON: {e}, data: {data_block[:100]}")
continue
except Exception as e:
logger.exception(f"Error generating TTS: {e}")
yield ErrorFrame(error=f"MiniMax TTS error: {str(e)}")
finally:
await self.stop_ttfb_metrics()
yield TTSStoppedFrame()

3
src/pipecat/services/neuphonic/tts.py
View File

@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
import websockets
@@ -239,6 +240,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
logger.debug(f"Sending text to websocket: {msg}")
await self._websocket.send(json.dumps(msg))
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -315,6 +317,7 @@ class NeuphonicHttpTTSService(TTSService):
async def flush_audio(self):
pass
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Neuphonic streaming API.

9
src/pipecat/services/openai/base_llm.py
View File

@@ -35,6 +35,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.utils.tracing.service_decorators import traced_llm
class OpenAIUnhandledFunctionException(Exception):
@@ -176,6 +177,7 @@ class BaseOpenAILLMService(LLMService):
return chunks
@traced_llm
async def _process_context(self, context: OpenAILLMContext):
functions_list = []
arguments_list = []
@@ -238,6 +240,13 @@ class BaseOpenAILLMService(LLMService):
elif chunk.choices[0].delta.content:
await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content))
# When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm
# we need to get LLMTextFrame for the transcript
elif hasattr(chunk.choices[0].delta, "audio") and chunk.choices[0].delta.audio.get(
"transcript"
):
await self.push_frame(LLMTextFrame(chunk.choices[0].delta.audio["transcript"]))
# if we got a function name and arguments, check to see if it's a function with
# a registered handler. If so, run the registered callback, save the result to
# the context, and re-prompt to get a chat answer. If we don't have a registered

2
src/pipecat/services/openai/tts.py
View File

@@ -18,6 +18,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
ValidVoice = Literal[
"alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse"
@@ -94,6 +95,7 @@ class OpenAITTSService(TTSService):
f"Current rate of {self.sample_rate}Hz may cause issues."
)
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
try:

22
src/pipecat/services/openai_realtime_beta/context.py
View File

@@ -12,8 +12,11 @@ from loguru import logger
from pipecat.frames.frames import (
Frame,
FunctionCallResultFrame,
InterimTranscriptionFrame,
LLMMessagesUpdateFrame,
LLMSetToolsFrame,
LLMTextFrame,
TranscriptionFrame,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
@@ -136,15 +139,6 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext):
}
self.add_message(message)
def add_assistant_content_item_as_message(self, item):
message = {"role": "assistant", "content": []}
for content in item.content:
if content.type == "audio":
message["content"].append({"type": "text", "text": content.transcript})
else:
logger.error(f"Unhandled content type in assistant item: {content.type} - {item}")
self.add_message(message)
class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator):
async def process_frame(
@@ -170,6 +164,16 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator):
class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator):
# The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output,
# but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We
# need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames
# are process. This ensures that the context gets only one set of messages.
# OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames,
# so we need to ignore pushing those as well, as they're also TextFrames.
async def process_frame(self, frame: Frame, direction: FrameDirection):
if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)):
await super().process_frame(frame, direction)
async def handle_function_call_result(self, frame: FunctionCallResultFrame):
await super().handle_function_call_result(frame)

16
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -562,13 +562,11 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True))
async def _handle_assistant_output(self, output):
# logger.debug(f"!!! HANDLE Assistant output: {output}")
# We haven't seen intermixed audio and function_call items in the same response. But let's
# try to write logic that handles that, if it does happen.
messages = [item for item in output if item.type == "message"]
# Also, the assistant output is pushed as LLMTextFrame and TTSTextFrame to be handled by
# the assistant context aggregator.
function_calls = [item for item in output if item.type == "function_call"]
for item in messages:
self._context.add_assistant_content_item_as_message(item)
await self._handle_function_call_items(function_calls)
async def _handle_function_call_items(self, items):
@@ -579,15 +577,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
arguments = json.loads(item.arguments)
if self.has_function(function_name):
run_llm = index == total_items - 1
if function_name in self._functions.keys():
await self.call_function(
context=self._context,
tool_call_id=tool_id,
function_name=function_name,
arguments=arguments,
run_llm=run_llm,
)
elif None in self._functions.keys():
if function_name in self._functions.keys() or None in self._functions.keys():
await self.call_function(
context=self._context,
tool_call_id=tool_id,

2
src/pipecat/services/piper/tts.py
View File

@@ -17,6 +17,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts
# This assumes a running TTS service running: https://github.com/rhasspy/piper/blob/master/src/python_run/README_http.md
@@ -54,6 +55,7 @@ class PiperTTSService(TTSService):
def can_generate_metrics(self) -> bool:
return True
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Piper API.

3
src/pipecat/services/playht/tts.py
View File

@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
try:
from pyht.async_client import AsyncClient
@@ -268,6 +269,7 @@ class PlayHTTTSService(InterruptibleTTSService):
except json.JSONDecodeError:
logger.error(f"Invalid JSON message: {message}")
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
@@ -391,6 +393,7 @@ class PlayHTHttpTTSService(TTSService):
def language_to_service_language(self, language: Language) -> Optional[str]:
return language_to_playht_language(language)
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")

22
src/pipecat/services/rime/tts.py
View File

@@ -29,6 +29,7 @@ from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
from pipecat.utils.tracing.service_decorators import traced_tts
try:
import websockets
@@ -68,13 +69,15 @@ class RimeTTSService(AudioContextWordTTSService):
language: Optional[Language] = Language.EN
speed_alpha: Optional[float] = 1.0
reduce_latency: Optional[bool] = False
pause_between_brackets: Optional[bool] = False
phonemize_between_brackets: Optional[bool] = False
def __init__(
self,
*,
api_key: str,
voice_id: str,
url: str = "wss://users-ws.rime.ai/ws2",
url: str = "wss://users.rime.ai/ws2",
model: str = "mistv2",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
@@ -117,6 +120,8 @@ class RimeTTSService(AudioContextWordTTSService):
else "eng",
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": json.dumps(params.pause_between_brackets),
"phonemizeBetweenBrackets": json.dumps(params.phonemize_between_brackets),
}
# State tracking
@@ -304,8 +309,9 @@ class RimeTTSService(AudioContextWordTTSService):
await super().push_frame(frame, direction)
if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
if isinstance(frame, TTSStoppedFrame):
await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)])
await self.add_word_timestamps([("Reset", 0)])
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text.
@@ -381,6 +387,7 @@ class RimeHttpTTSService(TTSService):
def can_generate_metrics(self) -> bool:
return True
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
@@ -396,6 +403,13 @@ class RimeHttpTTSService(TTSService):
payload["modelId"] = self._model_name
payload["samplingRate"] = self.sample_rate
# Arcana does not support PCM audio
if payload["modelId"] == "arcana":
headers["Accept"] = "audio/wav"
need_to_strip_wav_header = True
else:
need_to_strip_wav_header = False
try:
await self.start_ttfb_metrics()
@@ -416,6 +430,10 @@ class RimeHttpTTSService(TTSService):
CHUNK_SIZE = 1024
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
if need_to_strip_wav_header and chunk.startswith(b"RIFF"):
chunk = chunk[44:]
need_to_strip_wav_header = False
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)

374
src/pipecat/services/riva/stt.py
View File

@@ -5,7 +5,7 @@
#
import asyncio
from typing import AsyncGenerator, Optional
from typing import AsyncGenerator, List, Mapping, Optional
from loguru import logger
from pydantic import BaseModel
@@ -13,14 +13,16 @@ from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
TranscriptionFrame,
)
from pipecat.services.stt_service import STTService
from pipecat.services.stt_service import SegmentedSTTService, STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
import riva.client
@@ -31,7 +33,59 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
class ParakeetSTTService(STTService):
def language_to_riva_language(language: Language) -> Optional[str]:
"""Maps Language enum to Riva ASR language codes.
Source:
https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-riva-build-table.html?highlight=fr%20fr
Args:
language: Language enum value.
Returns:
Optional[str]: Riva language code or None if not supported.
"""
language_map = {
# Arabic
Language.AR: "ar-AR",
# English
Language.EN: "en-US", # Default to US
Language.EN_US: "en-US",
Language.EN_GB: "en-GB",
# French
Language.FR: "fr-FR",
Language.FR_FR: "fr-FR",
# German
Language.DE: "de-DE",
Language.DE_DE: "de-DE",
# Hindi
Language.HI: "hi-IN",
Language.HI_IN: "hi-IN",
# Italian
Language.IT: "it-IT",
Language.IT_IT: "it-IT",
# Japanese
Language.JA: "ja-JP",
Language.JA_JP: "ja-JP",
# Korean
Language.KO: "ko-KR",
Language.KO_KR: "ko-KR",
# Portuguese
Language.PT: "pt-BR", # Default to Brazilian
Language.PT_BR: "pt-BR",
# Russian
Language.RU: "ru-RU",
Language.RU_RU: "ru-RU",
# Spanish
Language.ES: "es-ES", # Default to Spain
Language.ES_ES: "es-ES",
Language.ES_US: "es-US", # US Spanish
}
return language_map.get(language)
class RivaSTTService(STTService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN_US
@@ -40,7 +94,10 @@ class ParakeetSTTService(STTService):
*,
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
model_function_map: Mapping[str, str] = {
"function_id": "1598d209-5e27-4d3c-8079-4751568b1081",
"model_name": "parakeet-ctc-1.1b-asr",
},
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
@@ -48,7 +105,7 @@ class ParakeetSTTService(STTService):
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._profanity_filter = False
self._automatic_punctuation = False
self._automatic_punctuation = True
self._no_verbatim_transcripts = False
self._language_code = params.language
self._boosted_lm_words = None
@@ -60,11 +117,21 @@ class ParakeetSTTService(STTService):
self._stop_history_eou = -1
self._stop_threshold_eou = -1.0
self._custom_configuration = ""
self._function_id = model_function_map.get("function_id")
self.set_model_name("parakeet-ctc-1.1b-asr")
self._settings = {
"language": str(params.language),
"profanity_filter": self._profanity_filter,
"automatic_punctuation": self._automatic_punctuation,
"verbatim_transcripts": not self._no_verbatim_transcripts,
"boosted_lm_words": self._boosted_lm_words,
"boosted_lm_score": self._boosted_lm_score,
}
self.set_model_name(model_function_map.get("model_name"))
metadata = [
["function-id", function_id],
["function-id", self._function_id],
["authorization", f"Bearer {api_key}"],
]
auth = riva.client.Auth(None, True, server, metadata)
@@ -79,6 +146,13 @@ class ParakeetSTTService(STTService):
def can_generate_metrics(self) -> bool:
return False
async def set_model(self, model: str):
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -161,6 +235,13 @@ class ParakeetSTTService(STTService):
self._thread_running = False
raise
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
async def _handle_response(self, response):
for result in response.results:
if result and not result.alternatives:
@@ -172,11 +253,18 @@ class ParakeetSTTService(STTService):
if result.is_final:
await self.stop_processing_metrics()
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), None)
TranscriptionFrame(transcript, "", time_now_iso8601(), self._language_code)
)
await self._handle_transcription(
transcript=transcript,
is_final=result.is_final,
language=self._language_code,
)
else:
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601(), None)
InterimTranscriptionFrame(
transcript, "", time_now_iso8601(), self._language_code
)
)
async def _response_task_handler(self):
@@ -185,6 +273,8 @@ class ParakeetSTTService(STTService):
await self._handle_response(response)
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_ttfb_metrics()
await self.start_processing_metrics()
await self._queue.put(audio)
yield None
@@ -196,3 +286,269 @@ class ParakeetSTTService(STTService):
def __iter__(self):
return self
class RivaSegmentedSTTService(SegmentedSTTService):
"""Speech-to-text service using NVIDIA Riva's offline/batch models.
By default, his service uses NVIDIA's Riva Canary ASR API to perform speech-to-text
transcription on audio segments. It inherits from SegmentedSTTService to handle
audio buffering and speech detection.
Args:
api_key: NVIDIA API key for authentication
server: Riva server address (defaults to NVIDIA Cloud Function endpoint)
model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID
sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate
params: Additional configuration parameters for Riva
**kwargs: Additional arguments passed to SegmentedSTTService
"""
class InputParams(BaseModel):
language: Optional[Language] = Language.EN_US
profanity_filter: bool = False
automatic_punctuation: bool = True
verbatim_transcripts: bool = False
boosted_lm_words: Optional[List[str]] = None
boosted_lm_score: float = 4.0
def __init__(
self,
*,
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
model_function_map: Mapping[str, str] = {
"function_id": "ee8dc628-76de-4acc-8595-1836e7e857bd",
"model_name": "canary-1b-asr",
},
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
# Set model name
self.set_model_name(model_function_map.get("model_name"))
# Initialize Riva settings
self._api_key = api_key
self._server = server
self._function_id = model_function_map.get("function_id")
self._model_name = model_function_map.get("model_name")
# Store the language as a Language enum and as a string
self._language_enum = params.language or Language.EN_US
self._language = self.language_to_service_language(self._language_enum) or "en-US"
# Configure transcription parameters
self._profanity_filter = params.profanity_filter
self._automatic_punctuation = params.automatic_punctuation
self._verbatim_transcripts = params.verbatim_transcripts
self._boosted_lm_words = params.boosted_lm_words
self._boosted_lm_score = params.boosted_lm_score
# Voice activity detection thresholds (use Riva defaults)
self._start_history = -1
self._start_threshold = -1.0
self._stop_history = -1
self._stop_threshold = -1.0
self._stop_history_eou = -1
self._stop_threshold_eou = -1.0
self._custom_configuration = ""
# Create Riva client
self._config = None
self._asr_service = None
self._settings = {"language": self._language_enum}
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to Riva's language code."""
return language_to_riva_language(language)
def _initialize_client(self):
"""Initialize the Riva ASR client with authentication metadata."""
if self._asr_service is not None:
return
# Set up authentication metadata for NVIDIA Cloud Functions
metadata = [
["function-id", self._function_id],
["authorization", f"Bearer {self._api_key}"],
]
# Create authenticated client
auth = riva.client.Auth(None, True, self._server, metadata)
self._asr_service = riva.client.ASRService(auth)
logger.info(f"Initialized RivaSegmentedSTTService with model: {self.model_name}")
def _create_recognition_config(self):
"""Create the Riva ASR recognition configuration."""
# Create base configuration
config = riva.client.RecognitionConfig(
language_code=self._language, # Now using the string, not a tuple
max_alternatives=1,
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=self._verbatim_transcripts,
)
# Add word boosting if specified
if self._boosted_lm_words:
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
)
# Add voice activity detection parameters
riva.client.add_endpoint_parameters_to_config(
config,
self._start_history,
self._start_threshold,
self._stop_history,
self._stop_history_eou,
self._stop_threshold,
self._stop_threshold_eou,
)
# Add any custom configuration
if self._custom_configuration:
riva.client.add_custom_configuration_to_config(config, self._custom_configuration)
return config
def can_generate_metrics(self) -> bool:
"""Indicates whether this service can generate processing metrics."""
return True
async def set_model(self, model: str):
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
async def start(self, frame: StartFrame):
"""Initialize the service when the pipeline starts."""
await super().start(frame)
self._initialize_client()
self._config = self._create_recognition_config()
async def set_language(self, language: Language):
"""Set the language for the STT service."""
logger.info(f"Switching STT language to: [{language}]")
self._language_enum = language
self._language = self.language_to_service_language(language) or "en-US"
self._settings["language"] = language
# Update configuration with new language
if self._config:
self._config.language_code = self._language
@traced_stt
async def _handle_transcription(self, transcript: str, language: Optional[Language] = None):
"""Handle a transcription result with tracing."""
pass
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Transcribe an audio segment.
Args:
audio: Raw audio bytes in WAV format (already converted by base class).
Yields:
Frame: TranscriptionFrame containing the transcribed text.
"""
try:
await self.start_processing_metrics()
await self.start_ttfb_metrics()
# Make sure the client is initialized
if self._asr_service is None:
self._initialize_client()
# Make sure the config is created
if self._config is None:
self._config = self._create_recognition_config()
# Type assertion to satisfy the IDE
assert self._asr_service is not None, "ASR service not initialized"
assert self._config is not None, "Recognition config not created"
# Process audio with Riva ASR - explicitly request non-future response
raw_response = self._asr_service.offline_recognize(audio, self._config, future=False)
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
# Process the response - handle different possible return types
try:
# If it's a future-like object, get the result
if hasattr(raw_response, "result"):
response = raw_response.result()
else:
response = raw_response
# Process transcription results
transcription_found = False
# Now we can safely check results
# Type hint for the IDE
results = getattr(response, "results", [])
for result in results:
alternatives = getattr(result, "alternatives", [])
if alternatives:
text = alternatives[0].transcript.strip()
if text:
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text, "", time_now_iso8601(), self._language_enum
)
transcription_found = True
await self._handle_transcription(text, True, self._language_enum)
if not transcription_found:
logger.debug("No transcription results found in Riva response")
except AttributeError as ae:
logger.error(f"Unexpected response structure from Riva: {ae}")
yield ErrorFrame(f"Unexpected Riva response format: {str(ae)}")
except Exception as e:
logger.exception(f"Riva Canary ASR error: {e}")
yield ErrorFrame(f"Riva Canary ASR error: {str(e)}")
class ParakeetSTTService(RivaSTTService):
"""Deprecated: Use RivaSTTService instead."""
def __init__(
self,
*,
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
model_function_map: Mapping[str, str] = {
"function_id": "1598d209-5e27-4d3c-8079-4751568b1081",
"model_name": "parakeet-ctc-1.1b-asr",
},
sample_rate: Optional[int] = None,
params: RivaSTTService.InputParams = RivaSTTService.InputParams(), # Use parent class's type
**kwargs,
):
super().__init__(
api_key=api_key,
server=server,
model_function_map=model_function_map,
sample_rate=sample_rate,
params=params,
**kwargs,
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"`ParakeetSTTService` is deprecated, use `RivaSTTService` instead.",
DeprecationWarning,
)

79
src/pipecat/services/riva/tts.py
View File

@@ -5,7 +5,13 @@
#
import asyncio
from typing import AsyncGenerator, Optional
import os
from typing import AsyncGenerator, Mapping, Optional
from pipecat.utils.tracing.service_decorators import traced_tts
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from loguru import logger
from pydantic import BaseModel
@@ -27,10 +33,10 @@ except ModuleNotFoundError as e:
logger.error("In order to use NVIDIA Riva TTS, you need to `pip install pipecat-ai[riva]`.")
raise Exception(f"Missing module: {e}")
FASTPITCH_TIMEOUT_SECS = 5
RIVA_TTS_TIMEOUT_SECS = 5
class FastPitchTTSService(TTSService):
class RivaTTSService(TTSService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN_US
quality: Optional[int] = 20
@@ -38,11 +44,14 @@ class FastPitchTTSService(TTSService):
def __init__(
self,
*,
api_key: str,
api_key: str = None,
server: str = "grpc.nvcf.nvidia.com:443",
voice_id: str = "English-US.Female-1",
voice_id: str = "Magpie-Multilingual.EN-US.Ray",
sample_rate: Optional[int] = None,
function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972",
model_function_map: Mapping[str, str] = {
"function_id": "877104f7-e885-42b9-8de8-f6e4c6303969",
"model_name": "magpie-tts-multilingual",
},
params: InputParams = InputParams(),
**kwargs,
):
@@ -51,12 +60,13 @@ class FastPitchTTSService(TTSService):
self._voice_id = voice_id
self._language_code = params.language
self._quality = params.quality
self._function_id = model_function_map.get("function_id")
self.set_model_name("fastpitch-hifigan-tts")
self.set_model_name(model_function_map.get("model_name"))
self.set_voice(voice_id)
metadata = [
["function-id", function_id],
["function-id", self._function_id],
["authorization", f"Bearer {api_key}"],
]
auth = riva.client.Auth(None, True, server, metadata)
@@ -68,6 +78,14 @@ class FastPitchTTSService(TTSService):
riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest()
)
async def set_model(self, model: str):
logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
example = {"function_id": "<UUID>", "model_name": "<model_name>"}
logger.warning(
f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
)
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
def read_audio_responses(queue: asyncio.Queue):
def add_response(r):
@@ -79,8 +97,8 @@ class FastPitchTTSService(TTSService):
self._voice_id,
self._language_code,
sample_rate_hz=self.sample_rate,
audio_prompt_file=None,
quality=self._quality,
zero_shot_audio_prompt_file=None,
zero_shot_quality=self._quality,
custom_dictionary={},
)
for r in responses:
@@ -100,7 +118,7 @@ class FastPitchTTSService(TTSService):
await asyncio.to_thread(read_audio_responses, queue)
# Wait for the thread to start.
resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS)
resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS)
while resp:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
@@ -109,9 +127,46 @@ class FastPitchTTSService(TTSService):
num_channels=1,
)
yield frame
resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS)
resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS)
except asyncio.TimeoutError:
logger.error(f"{self} timeout waiting for audio response")
await self.start_tts_usage_metrics(text)
yield TTSStoppedFrame()
class FastPitchTTSService(RivaTTSService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN_US
quality: Optional[int] = 20
def __init__(
self,
*,
api_key: str = None,
server: str = "grpc.nvcf.nvidia.com:443",
voice_id: str = "English-US.Female-1",
sample_rate: Optional[int] = None,
model_function_map: Mapping[str, str] = {
"function_id": "0149dedb-2be8-4195-b9a0-e57e0e14f972",
"model_name": "fastpitch-hifigan-tts",
},
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(
api_key=api_key,
voice_id=voice_id,
sample_rate=sample_rate,
model_function_map=model_function_map,
params=params,
**kwargs,
)
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"`FastPitchTTSService` is deprecated, use `RivaTTSService` instead.",
DeprecationWarning,
)

17
src/pipecat/services/simli/video.py
View File

@@ -64,13 +64,16 @@ class SimliVideoService(FrameProcessor):
async for audio_frame in self._simli_client.getAudioStreamIterator():
resampled_frames = self._pipecat_resampler.resample(audio_frame)
for resampled_frame in resampled_frames:
await self.push_frame(
TTSAudioRawFrame(
audio=resampled_frame.to_ndarray().tobytes(),
sample_rate=self._pipecat_resampler.rate,
num_channels=1,
),
)
audio_array = resampled_frame.to_ndarray()
# Only push frame is there is audio (e.g. not silence)
if audio_array.any():
await self.push_frame(
TTSAudioRawFrame(
audio=audio_array.tobytes(),
sample_rate=self._pipecat_resampler.rate,
num_channels=1,
),
)
async def _consume_and_process_video(self):
await self._pipecat_resampler_event.wait()

35
src/pipecat/services/tts_service.py
View File

@@ -19,6 +19,7 @@ from pipecat.frames.frames import (
Frame,
InterimTranscriptionFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
StartFrame,
StartInterruptionFrame,
TextFrame,
@@ -65,6 +66,8 @@ class TTSService(AIService):
# Text filter executed after text has been aggregated.
text_filters: Sequence[BaseTextFilter] = [],
text_filter: Optional[BaseTextFilter] = None,
# Audio transport destination of the generated frames.
transport_destination: Optional[str] = None,
**kwargs,
):
super().__init__(**kwargs)
@@ -81,6 +84,8 @@ class TTSService(AIService):
self._settings: Dict[str, Any] = {}
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
self._text_filters: Sequence[BaseTextFilter] = text_filters
self._transport_destination: Optional[str] = transport_destination
if text_filter:
import warnings
@@ -206,13 +211,16 @@ class TTSService(AIService):
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
if self._push_silence_after_stop and isinstance(frame, TTSStoppedFrame):
silence_num_bytes = int(self._silence_time_s * self.sample_rate * 2) # 16-bit
await self.push_frame(
TTSAudioRawFrame(
audio=b"\x00" * silence_num_bytes,
sample_rate=self.sample_rate,
num_channels=1,
)
silence_frame = TTSAudioRawFrame(
audio=b"\x00" * silence_num_bytes,
sample_rate=self.sample_rate,
num_channels=1,
)
silence_frame.transport_destination = self._transport_destination
await self.push_frame(silence_frame)
if isinstance(frame, (TTSStartedFrame, TTSStoppedFrame, TTSAudioRawFrame, TTSTextFrame)):
frame.transport_destination = self._transport_destination
await super().push_frame(frame, direction)
@@ -308,6 +316,7 @@ class WordTTSService(TTSService):
self._initial_word_timestamp = -1
self._words_queue = asyncio.Queue()
self._words_task = None
self._llm_response_started: bool = False
def start_word_timestamps(self):
if self._initial_word_timestamp == -1:
@@ -335,11 +344,14 @@ class WordTTSService(TTSService):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
if isinstance(frame, LLMFullResponseStartFrame):
self._llm_response_started = True
elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
await self.flush_audio()
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
self._llm_response_started = False
self.reset_word_timestamps()
def _create_words_task(self):
@@ -354,13 +366,14 @@ class WordTTSService(TTSService):
async def _words_task_handler(self):
last_pts = 0
while True:
frame = None
(word, timestamp) = await self._words_queue.get()
if word == "Reset" and timestamp == 0:
self.reset_word_timestamps()
frame = None
elif word == "LLMFullResponseEndFrame" and timestamp == 0:
frame = LLMFullResponseEndFrame()
frame.pts = last_pts
if self._llm_response_started:
self._llm_response_started = False
frame = LLMFullResponseEndFrame()
frame.pts = last_pts
elif word == "TTSStoppedFrame" and timestamp == 0:
frame = TTSStoppedFrame()
frame.pts = last_pts

2
src/pipecat/services/ultravox/stt.py
View File

@@ -425,7 +425,7 @@ class UltravoxSTTService(AIService):
if "content" in delta:
new_text = delta["content"]
if new_text:
yield LLMTextFrame(text=new_text.strip())
yield LLMTextFrame(text=new_text)
# Stop processing metrics after completion
await self.stop_processing_metrics()

16
src/pipecat/services/whisper/base_stt.py
View File

@@ -14,6 +14,7 @@ from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
def language_to_whisper_language(language: Language) -> Optional[str]:
@@ -126,6 +127,13 @@ class BaseWhisperSTTService(SegmentedSTTService):
self._prompt = prompt
self._temperature = temperature
self._settings = {
"base_url": base_url,
"language": self._language,
"prompt": self._prompt,
"temperature": self._temperature,
}
def _create_client(self, api_key: Optional[str], base_url: Optional[str]):
return AsyncOpenAI(api_key=api_key, base_url=base_url)
@@ -147,6 +155,13 @@ class BaseWhisperSTTService(SegmentedSTTService):
logger.info(f"Switching STT language to: [{language}]")
self._language = language
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
try:
await self.start_processing_metrics()
@@ -160,6 +175,7 @@ class BaseWhisperSTTService(SegmentedSTTService):
text = response.text.strip()
if text:
await self._handle_transcription(text, True, self._language)
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(text, "", time_now_iso8601())
else:

23
src/pipecat/services/whisper/stt.py
View File

@@ -18,6 +18,7 @@ from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
if TYPE_CHECKING:
try:
@@ -291,6 +292,9 @@ class WhisperSTTService(SegmentedSTTService):
self._settings = {
"language": language,
"device": self._device,
"compute_type": self._compute_type,
"no_speech_prob": self._no_speech_prob,
}
self._load()
@@ -343,6 +347,13 @@ class WhisperSTTService(SegmentedSTTService):
logger.error("In order to use Whisper, you need to `pip install pipecat-ai[whisper]`.")
self._model = None
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Transcribes given audio using Whisper.
@@ -381,6 +392,7 @@ class WhisperSTTService(SegmentedSTTService):
await self.stop_processing_metrics()
if text:
await self._handle_transcription(text, True, self._settings["language"])
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(text, "", time_now_iso8601(), self._settings["language"])
@@ -422,6 +434,9 @@ class WhisperSTTServiceMLX(WhisperSTTService):
self._settings = {
"language": language,
"no_speech_prob": self._no_speech_prob,
"temperature": self._temperature,
"engine": "mlx",
}
# No need to call _load() as MLX Whisper loads models on demand
@@ -431,6 +446,13 @@ class WhisperSTTServiceMLX(WhisperSTTService):
"""MLX Whisper loads models on demand, so this is a no-op."""
pass
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
@override
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Transcribes given audio using MLX Whisper.
@@ -479,6 +501,7 @@ class WhisperSTTServiceMLX(WhisperSTTService):
await self.stop_processing_metrics()
if text:
await self._handle_transcription(text, True, self._settings["language"])
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(text, "", time_now_iso8601(), self._settings["language"])

2
src/pipecat/services/xtts/tts.py
View File

@@ -20,6 +20,7 @@ from pipecat.frames.frames import (
)
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.tracing.service_decorators import traced_tts
# The server below can connect to XTTS through a local running docker
#
@@ -117,6 +118,7 @@ class XTTSService(TTSService):
return
self._studio_speakers = await r.json()
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")

18
src/pipecat/tests/utils.py
View File

@@ -6,7 +6,7 @@
import asyncio
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Dict, Optional, Sequence, Tuple
from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple
from pipecat.frames.frames import (
EndFrame,
@@ -15,7 +15,7 @@ from pipecat.frames.frames import (
StartFrame,
SystemFrame,
)
from pipecat.observers.base_observer import BaseObserver
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -42,14 +42,10 @@ class HeartbeatsObserver(BaseObserver):
self._target = target
self._callback = heartbeat_callback
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
async def on_push_frame(self, data: FramePushed):
src = data.source
frame = data.frame
if src == self._target and isinstance(frame, HeartbeatFrame):
await self._callback(self._target, frame)
@@ -83,6 +79,7 @@ async def run_test(
expected_down_frames: Optional[Sequence[type]] = None,
expected_up_frames: Optional[Sequence[type]] = None,
ignore_start: bool = True,
observers: List[BaseObserver] = [],
start_metadata: Dict[str, Any] = {},
send_end_frame: bool = True,
) -> Tuple[Sequence[Frame], Sequence[Frame]]:
@@ -104,6 +101,7 @@ async def run_test(
task = PipelineTask(
pipeline,
params=PipelineParams(start_metadata=start_metadata),
observers=observers,
cancel_on_idle_timeout=False,
)

14
src/pipecat/transports/base_input.py
View File

@@ -79,7 +79,7 @@ class BaseInputTransport(FrameProcessor):
)
self._params.audio_in_passthrough = True
if self._params.camera_in_enabled or self._params.camera_out_enabled:
if self._params.camera_in_enabled:
import warnings
with warnings.catch_warnings():
@@ -122,6 +122,7 @@ class BaseInputTransport(FrameProcessor):
# Configure VAD analyzer.
if self._params.vad_analyzer:
self._params.vad_analyzer.set_sample_rate(self._sample_rate)
# Configure End of turn analyzer.
if self._params.turn_analyzer:
self._params.turn_analyzer.set_sample_rate(self._sample_rate)
@@ -129,10 +130,6 @@ class BaseInputTransport(FrameProcessor):
# Start audio filter.
if self._params.audio_in_filter:
await self._params.audio_in_filter.start(self._sample_rate)
# Create audio input queue and task if needed.
if not self._audio_task and self._params.audio_in_enabled:
self._audio_in_queue = asyncio.Queue()
self._audio_task = self.create_task(self._audio_task_handler())
async def stop(self, frame: EndFrame):
# Cancel and wait for the audio input task to finish.
@@ -149,6 +146,13 @@ class BaseInputTransport(FrameProcessor):
await self.cancel_task(self._audio_task)
self._audio_task = None
async def set_transport_ready(self, frame: StartFrame):
"""To be called when the transport is ready to stream."""
# Create audio input queue and task if needed.
if not self._audio_task and self._params.audio_in_enabled:
self._audio_in_queue = asyncio.Queue()
self._audio_task = self.create_task(self._audio_task_handler())
async def push_audio_frame(self, frame: InputAudioRawFrame):
if self._params.audio_in_enabled:
await self._audio_in_queue.put(frame)

747
src/pipecat/transports/base_output.py
View File

@@ -8,11 +8,12 @@ import asyncio
import itertools
import sys
import time
from typing import AsyncGenerator, List
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional
from loguru import logger
from PIL import Image
from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer
from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
BotSpeakingFrame,
@@ -46,35 +47,28 @@ class BaseOutputTransport(FrameProcessor):
self._params = params
# Task to process incoming frames so we don't block upstream elements.
self._sink_task = None
# Task to process incoming frames using a clock.
self._sink_clock_task = None
# Task to write/send audio and image frames.
self._video_out_task = None
# These are the images that we should send at our desired framerate.
self._video_images = None
# Output sample rate. It will be initialized on StartFrame.
self._sample_rate = 0
self._resampler = create_default_resampler()
# Chunk size that will be written. It will be computed on StartFrame
# We write 10ms*CHUNKS of audio at a time (where CHUNKS is the
# `audio_out_10ms_chunks` parameter). If we receive long audio frames we
# will chunk them. This helps with interruption handling. It will be
# initialized on StartFrame.
self._audio_chunk_size = 0
self._audio_buffer = bytearray()
self._stopped_event = asyncio.Event()
# Indicates if the bot is currently speaking.
self._bot_speaking = False
# We will have one media sender per output frame destination. This allow
# us to send multiple streams at the same time if the transport allows
# it.
self._media_senders: Dict[Any, "BaseOutputTransport.MediaSender"] = {}
@property
def sample_rate(self) -> int:
return self._sample_rate
@property
def audio_chunk_size(self) -> int:
return self._audio_chunk_size
async def start(self, frame: StartFrame):
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
@@ -84,42 +78,65 @@ class BaseOutputTransport(FrameProcessor):
audio_bytes_10ms = int(self._sample_rate / 100) * self._params.audio_out_channels * 2
self._audio_chunk_size = audio_bytes_10ms * self._params.audio_out_10ms_chunks
# Start audio mixer.
if self._params.audio_out_mixer:
await self._params.audio_out_mixer.start(self._sample_rate)
self._create_video_task()
self._create_sink_tasks()
async def stop(self, frame: EndFrame):
# Let the sink tasks process the queue until they reach this EndFrame.
await self._sink_clock_queue.put((sys.maxsize, frame.id, frame))
await self._sink_queue.put(frame)
# At this point we have enqueued an EndFrame and we need to wait for
# that EndFrame to be processed by the sink tasks. We also need to wait
# for these tasks before cancelling the video and audio tasks below
# because they might be still rendering.
if self._sink_task:
await self.wait_for_task(self._sink_task)
if self._sink_clock_task:
await self.wait_for_task(self._sink_clock_task)
# We can now cancel the video task.
await self._cancel_video_task()
for _, sender in self._media_senders.items():
await sender.stop(frame)
async def cancel(self, frame: CancelFrame):
# Since we are cancelling everything it doesn't matter if we cancel sink
# tasks first or not.
await self._cancel_sink_tasks()
await self._cancel_video_task()
for _, sender in self._media_senders.items():
await sender.cancel(frame)
async def set_transport_ready(self, frame: StartFrame):
"""To be called when the transport is ready to stream."""
# Register destinations.
for destination in self._params.audio_out_destinations:
await self.register_audio_destination(destination)
for destination in self._params.video_out_destinations:
await self.register_video_destination(destination)
# Start default media sender.
self._media_senders[None] = BaseOutputTransport.MediaSender(
self,
destination=None,
sample_rate=self.sample_rate,
audio_chunk_size=self.audio_chunk_size,
params=self._params,
)
await self._media_senders[None].start(frame)
# Media senders already send both audio and video, so make sure we only
# have one media server per shared name.
destinations = list(
set(self._params.audio_out_destinations + self._params.video_out_destinations)
)
# Start media senders.
for destination in destinations:
self._media_senders[destination] = BaseOutputTransport.MediaSender(
self,
destination=destination,
sample_rate=self.sample_rate,
audio_chunk_size=self.audio_chunk_size,
params=self._params,
)
await self._media_senders[destination].start(frame)
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
pass
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
async def register_video_destination(self, destination: str):
pass
async def write_raw_audio_frames(self, frames: bytes):
async def register_audio_destination(self, destination: str):
pass
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
pass
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
pass
async def send_audio(self, frame: OutputAudioRawFrame):
@@ -150,7 +167,7 @@ class BaseOutputTransport(FrameProcessor):
await self.push_frame(frame, direction)
elif isinstance(frame, (StartInterruptionFrame, StopInterruptionFrame)):
await self.push_frame(frame, direction)
await self._handle_interruptions(frame)
await self._handle_frame(frame)
elif isinstance(frame, TransportMessageUrgentFrame):
await self.send_message(frame)
elif isinstance(frame, SystemFrame):
@@ -160,117 +177,420 @@ class BaseOutputTransport(FrameProcessor):
await self.stop(frame)
# Keep pushing EndFrame down so all the pipeline stops nicely.
await self.push_frame(frame, direction)
elif isinstance(frame, MixerControlFrame) and self._params.audio_out_mixer:
await self._params.audio_out_mixer.process_frame(frame)
elif isinstance(frame, MixerControlFrame):
await self._handle_frame(frame)
# Other frames.
elif isinstance(frame, OutputAudioRawFrame):
await self._handle_audio(frame)
await self._handle_frame(frame)
elif isinstance(frame, (OutputImageRawFrame, SpriteFrame)):
await self._handle_image(frame)
await self._handle_frame(frame)
# TODO(aleix): Images and audio should support presentation timestamps.
elif frame.pts:
await self._sink_clock_queue.put((frame.pts, frame.id, frame))
await self._handle_frame(frame)
elif direction == FrameDirection.UPSTREAM:
await self.push_frame(frame, direction)
else:
await self._sink_queue.put(frame)
await self._handle_frame(frame)
async def _handle_interruptions(self, frame: Frame):
if not self.interruptions_allowed:
async def _handle_frame(self, frame: Frame):
if frame.transport_destination not in self._media_senders:
logger.warning(
f"{self} destination [{frame.transport_destination}] not registered for frame {frame}"
)
return
sender = self._media_senders[frame.transport_destination]
if isinstance(frame, StartInterruptionFrame):
# Cancel sink and video tasks.
await self._cancel_sink_tasks()
await self._cancel_video_task()
# Create sink and video tasks.
await sender.handle_interruptions(frame)
elif isinstance(frame, OutputAudioRawFrame):
await sender.handle_audio_frame(frame)
elif isinstance(frame, (OutputImageRawFrame, SpriteFrame)):
await sender.handle_image_frame(frame)
elif isinstance(frame, MixerControlFrame):
await sender.handle_mixer_control_frame(frame)
elif frame.pts:
await sender.handle_timed_frame(frame)
else:
await sender.handle_sync_frame(frame)
#
# Media Sender
#
class MediaSender:
def __init__(
self,
transport: "BaseOutputTransport",
*,
destination: Optional[str],
sample_rate: int,
audio_chunk_size: int,
params: TransportParams,
):
self._transport = transport
self._destination = destination
self._sample_rate = sample_rate
self._audio_chunk_size = audio_chunk_size
self._params = params
# Buffer to keep track of incoming audio.
self._audio_buffer = bytearray()
# This will be used to resample incoming audio to the output sample rate.
self._resampler = create_default_resampler()
# The user can provide a single mixer, to be used by the default
# destination, or a destination/mixer mapping.
self._mixer: Optional[BaseAudioMixer] = None
# These are the images that we should send at our desired framerate.
self._video_images = None
# Indicates if the bot is currently speaking.
self._bot_speaking = False
self._audio_task: Optional[asyncio.Task] = None
self._video_task: Optional[asyncio.Task] = None
self._clock_task: Optional[asyncio.Task] = None
@property
def sample_rate(self) -> int:
return self._sample_rate
@property
def audio_chunk_size(self) -> int:
return self._audio_chunk_size
async def start(self, frame: StartFrame):
self._audio_buffer = bytearray()
# Create all tasks.
self._create_video_task()
self._create_sink_tasks()
self._create_clock_task()
self._create_audio_task()
# Check if we have an audio mixer for our destination.
if self._params.audio_out_mixer:
if isinstance(self._params.audio_out_mixer, Mapping):
self._mixer = self._params.audio_out_mixer.get(self._destination, None)
elif not self._destination:
# Only use the default mixer if we are the default destination.
self._mixer = self._params.audio_out_mixer
# Start audio mixer.
if self._mixer:
await self._mixer.start(self._sample_rate)
async def stop(self, frame: EndFrame):
# Let the sink tasks process the queue until they reach this EndFrame.
await self._clock_queue.put((sys.maxsize, frame.id, frame))
await self._audio_queue.put(frame)
# At this point we have enqueued an EndFrame and we need to wait for
# that EndFrame to be processed by the audio and clock tasks. We
# also need to wait for these tasks before cancelling the video task
# because it might be still rendering.
if self._audio_task:
await self._transport.wait_for_task(self._audio_task)
if self._clock_task:
await self._transport.wait_for_task(self._clock_task)
# Stop audio mixer.
if self._mixer:
await self._mixer.stop()
# We can now cancel the video task.
await self._cancel_video_task()
async def cancel(self, frame: CancelFrame):
# Since we are cancelling everything it doesn't matter what task we cancel first.
await self._cancel_audio_task()
await self._cancel_clock_task()
await self._cancel_video_task()
async def handle_interruptions(self, _: StartInterruptionFrame):
if not self._transport.interruptions_allowed:
return
# Cancel tasks.
await self._cancel_audio_task()
await self._cancel_clock_task()
await self._cancel_video_task()
# Create tasks.
self._create_video_task()
self._create_clock_task()
self._create_audio_task()
# Let's send a bot stopped speaking if we have to.
await self._bot_stopped_speaking()
async def _handle_audio(self, frame: OutputAudioRawFrame):
if not self._params.audio_out_enabled:
return
async def handle_audio_frame(self, frame: OutputAudioRawFrame):
if not self._params.audio_out_enabled:
return
# We might need to resample if incoming audio doesn't match the
# transport sample rate.
resampled = await self._resampler.resample(
frame.audio, frame.sample_rate, self._sample_rate
)
cls = type(frame)
self._audio_buffer.extend(resampled)
while len(self._audio_buffer) >= self._audio_chunk_size:
chunk = cls(
bytes(self._audio_buffer[: self._audio_chunk_size]),
sample_rate=self._sample_rate,
num_channels=frame.num_channels,
# We might need to resample if incoming audio doesn't match the
# transport sample rate.
resampled = await self._resampler.resample(
frame.audio, frame.sample_rate, self._sample_rate
)
await self._sink_queue.put(chunk)
self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
async def _handle_image(self, frame: OutputImageRawFrame | SpriteFrame):
if not self._params.video_out_enabled:
return
cls = type(frame)
self._audio_buffer.extend(resampled)
while len(self._audio_buffer) >= self._audio_chunk_size:
chunk = cls(
bytes(self._audio_buffer[: self._audio_chunk_size]),
sample_rate=self._sample_rate,
num_channels=frame.num_channels,
)
await self._audio_queue.put(chunk)
self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
if self._params.video_out_is_live:
await self._video_out_queue.put(frame)
else:
await self._sink_queue.put(frame)
async def handle_image_frame(self, frame: OutputImageRawFrame | SpriteFrame):
if not self._params.video_out_enabled:
return
async def _bot_started_speaking(self):
if not self._bot_speaking:
logger.debug("Bot started speaking")
await self.push_frame(BotStartedSpeakingFrame())
await self.push_frame(BotStartedSpeakingFrame(), FrameDirection.UPSTREAM)
self._bot_speaking = True
if self._params.video_out_is_live and isinstance(frame, OutputImageRawFrame):
await self._video_queue.put(frame)
elif isinstance(frame, OutputImageRawFrame):
await self._set_video_image(frame)
else:
await self._set_video_images(frame.images)
async def _bot_stopped_speaking(self):
if self._bot_speaking:
logger.debug("Bot stopped speaking")
await self.push_frame(BotStoppedSpeakingFrame())
await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
self._bot_speaking = False
# Clean audio buffer (there could be tiny left overs if not multiple
# to our output chunk size).
self._audio_buffer = bytearray()
async def handle_timed_frame(self, frame: Frame):
await self._clock_queue.put((frame.pts, frame.id, frame))
#
# Sink tasks
#
async def handle_sync_frame(self, frame: Frame):
await self._audio_queue.put(frame)
def _create_sink_tasks(self):
if not self._sink_task:
self._sink_queue = asyncio.Queue()
self._sink_task = self.create_task(self._sink_task_handler())
if not self._sink_clock_task:
self._sink_clock_queue = asyncio.PriorityQueue()
self._sink_clock_task = self.create_task(self._sink_clock_task_handler())
async def handle_mixer_control_frame(self, frame: MixerControlFrame):
if self._mixer:
await self._mixer.process_frame(frame)
async def _cancel_sink_tasks(self):
# Stop sink tasks.
if self._sink_task:
await self.cancel_task(self._sink_task)
self._sink_task = None
# Stop sink clock tasks.
if self._sink_clock_task:
await self.cancel_task(self._sink_clock_task)
self._sink_clock_task = None
#
# Audio handling
#
async def _sink_frame_handler(self, frame: Frame):
if isinstance(frame, OutputImageRawFrame):
await self._set_video_image(frame)
elif isinstance(frame, SpriteFrame):
await self._set_video_images(frame.images)
elif isinstance(frame, TransportMessageFrame):
await self.send_message(frame)
def _create_audio_task(self):
if not self._audio_task:
self._audio_queue = asyncio.Queue()
self._audio_task = self._transport.create_task(self._audio_task_handler())
async def _sink_clock_task_handler(self):
running = True
while running:
try:
timestamp, _, frame = await self._sink_clock_queue.get()
async def _cancel_audio_task(self):
if self._audio_task:
await self._transport.cancel_task(self._audio_task)
self._audio_task = None
async def _bot_started_speaking(self):
if not self._bot_speaking:
logger.debug(
f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking"
)
downstream_frame = BotStartedSpeakingFrame()
downstream_frame.transport_destination = self._destination
upstream_frame = BotStartedSpeakingFrame()
upstream_frame.transport_destination = self._destination
await self._transport.push_frame(downstream_frame)
await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM)
self._bot_speaking = True
async def _bot_stopped_speaking(self):
if self._bot_speaking:
logger.debug(
f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking"
)
downstream_frame = BotStoppedSpeakingFrame()
downstream_frame.transport_destination = self._destination
upstream_frame = BotStoppedSpeakingFrame()
upstream_frame.transport_destination = self._destination
await self._transport.push_frame(downstream_frame)
await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM)
self._bot_speaking = False
# Clean audio buffer (there could be tiny left overs if not multiple
# to our output chunk size).
self._audio_buffer = bytearray()
async def _handle_frame(self, frame: Frame):
if isinstance(frame, OutputImageRawFrame):
await self._set_video_image(frame)
elif isinstance(frame, SpriteFrame):
await self._set_video_images(frame.images)
elif isinstance(frame, TransportMessageFrame):
await self._transport.send_message(frame)
def _next_frame(self) -> AsyncGenerator[Frame, None]:
async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
while True:
try:
frame = await asyncio.wait_for(
self._audio_queue.get(), timeout=vad_stop_secs
)
yield frame
except asyncio.TimeoutError:
# Notify the bot stopped speaking upstream if necessary.
await self._bot_stopped_speaking()
async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
last_frame_time = 0
silence = b"\x00" * self._audio_chunk_size
while True:
try:
frame = self._audio_queue.get_nowait()
if isinstance(frame, OutputAudioRawFrame):
frame.audio = await self._mixer.mix(frame.audio)
last_frame_time = time.time()
yield frame
except asyncio.QueueEmpty:
# Notify the bot stopped speaking upstream if necessary.
diff_time = time.time() - last_frame_time
if diff_time > vad_stop_secs:
await self._bot_stopped_speaking()
# Generate an audio frame with only the mixer's part.
frame = OutputAudioRawFrame(
audio=await self._mixer.mix(silence),
sample_rate=self._sample_rate,
num_channels=self._params.audio_out_channels,
)
yield frame
if self._mixer:
return with_mixer(BOT_VAD_STOP_SECS)
else:
return without_mixer(BOT_VAD_STOP_SECS)
async def _audio_task_handler(self):
# Push a BotSpeakingFrame every 200ms, we don't really need to push it
# at every audio chunk. If the audio chunk is bigger than 200ms, push at
# every audio chunk.
TOTAL_CHUNK_MS = self._params.audio_out_10ms_chunks * 10
BOT_SPEAKING_CHUNK_PERIOD = max(int(200 / TOTAL_CHUNK_MS), 1)
bot_speaking_counter = 0
async for frame in self._next_frame():
# Notify the bot started speaking upstream if necessary and that
# it's actually speaking.
if isinstance(frame, TTSAudioRawFrame):
await self._bot_started_speaking()
if bot_speaking_counter % BOT_SPEAKING_CHUNK_PERIOD == 0:
await self._transport.push_frame(BotSpeakingFrame())
await self._transport.push_frame(
BotSpeakingFrame(), FrameDirection.UPSTREAM
)
bot_speaking_counter = 0
bot_speaking_counter += 1
# No need to push EndFrame, it's pushed from process_frame().
if isinstance(frame, EndFrame):
break
# Handle frame.
await self._handle_frame(frame)
# Also, push frame downstream in case anyone else needs it.
await self._transport.push_frame(frame)
# Send audio.
if isinstance(frame, OutputAudioRawFrame):
await self._transport.write_raw_audio_frames(frame.audio, self._destination)
#
# Video handling
#
def _create_video_task(self):
if not self._video_task and self._params.video_out_enabled:
self._video_queue = asyncio.Queue()
self._video_task = self._transport.create_task(self._video_task_handler())
async def _cancel_video_task(self):
# Stop video output task.
if self._video_task:
await self._transport.cancel_task(self._video_task)
self._video_task = None
async def _set_video_image(self, image: OutputImageRawFrame):
self._video_images = itertools.cycle([image])
async def _set_video_images(self, images: List[OutputImageRawFrame]):
self._video_images = itertools.cycle(images)
async def _video_task_handler(self):
self._video_start_time = None
self._video_frame_index = 0
self._video_frame_duration = 1 / self._params.video_out_framerate
self._video_frame_reset = self._video_frame_duration * 5
while True:
if self._params.video_out_is_live:
await self._video_is_live_handler()
elif self._video_images:
image = next(self._video_images)
await self._draw_image(image)
await asyncio.sleep(self._video_frame_duration)
else:
await asyncio.sleep(self._video_frame_duration)
async def _video_is_live_handler(self):
image = await self._video_queue.get()
# We get the start time as soon as we get the first image.
if not self._video_start_time:
self._video_start_time = time.time()
self._video_frame_index = 0
# Calculate how much time we need to wait before rendering next image.
real_elapsed_time = time.time() - self._video_start_time
real_render_time = self._video_frame_index * self._video_frame_duration
delay_time = self._video_frame_duration + real_render_time - real_elapsed_time
if abs(delay_time) > self._video_frame_reset:
self._video_start_time = time.time()
self._video_frame_index = 0
elif delay_time > 0:
await asyncio.sleep(delay_time)
self._video_frame_index += 1
# Render image
await self._draw_image(image)
self._video_queue.task_done()
async def _draw_image(self, frame: OutputImageRawFrame):
desired_size = (self._params.video_out_width, self._params.video_out_height)
# TODO: we should refactor in the future to support dynamic resolutions
# which is kind of what happens in P2P connections.
# We need to add support for that inside the DailyTransport
if frame.size != desired_size:
image = Image.frombytes(frame.format, frame.size, frame.image)
resized_image = image.resize(desired_size)
# logger.warning(f"{frame} does not have the expected size {desired_size}, resizing")
frame = OutputImageRawFrame(
resized_image.tobytes(), resized_image.size, resized_image.format
)
await self._transport.write_raw_video_frame(frame, self._destination)
#
# Clock handling
#
def _create_clock_task(self):
if not self._clock_task:
self._clock_queue = asyncio.PriorityQueue()
self._clock_task = self._transport.create_task(self._clock_task_handler())
async def _cancel_clock_task(self):
if self._clock_task:
await self._transport.cancel_task(self._clock_task)
self._clock_task = None
async def _clock_task_handler(self):
running = True
while running:
timestamp, _, frame = await self._clock_queue.get()
# If we hit an EndFrame, we can finish right away.
running = not isinstance(frame, EndFrame)
@@ -279,167 +599,12 @@ class BaseOutputTransport(FrameProcessor):
# has already passed we process it, otherwise we wait until it's
# time to process it.
if running:
current_time = self.get_clock().get_time()
current_time = self._transport.get_clock().get_time()
if timestamp > current_time:
wait_time = nanoseconds_to_seconds(timestamp - current_time)
await asyncio.sleep(wait_time)
# Handle frame.
await self._sink_frame_handler(frame)
# Push frame downstream.
await self._transport.push_frame(frame)
# Also, push frame downstream in case anyone else needs it.
await self.push_frame(frame)
self._sink_clock_queue.task_done()
except asyncio.CancelledError:
raise
except Exception as e:
logger.exception(f"{self} error processing sink clock queue: {e}")
def _next_frame(self) -> AsyncGenerator[Frame, None]:
async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
while True:
try:
frame = await asyncio.wait_for(self._sink_queue.get(), timeout=vad_stop_secs)
yield frame
except asyncio.TimeoutError:
# Notify the bot stopped speaking upstream if necessary.
await self._bot_stopped_speaking()
async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
last_frame_time = 0
silence = b"\x00" * self._audio_chunk_size
while True:
try:
frame = self._sink_queue.get_nowait()
if isinstance(frame, OutputAudioRawFrame):
frame.audio = await self._params.audio_out_mixer.mix(frame.audio)
last_frame_time = time.time()
yield frame
except asyncio.QueueEmpty:
# Notify the bot stopped speaking upstream if necessary.
diff_time = time.time() - last_frame_time
if diff_time > vad_stop_secs:
await self._bot_stopped_speaking()
# Generate an audio frame with only the mixer's part.
frame = OutputAudioRawFrame(
audio=await self._params.audio_out_mixer.mix(silence),
sample_rate=self._sample_rate,
num_channels=self._params.audio_out_channels,
)
yield frame
if self._params.audio_out_mixer:
return with_mixer(BOT_VAD_STOP_SECS)
else:
return without_mixer(BOT_VAD_STOP_SECS)
async def _sink_task_handler(self):
# Push a BotSpeakingFrame every 200ms, we don't really need to push it
# at every audio chunk. If the audio chunk is bigger than 200ms, push at
# every audio chunk.
TOTAL_CHUNK_MS = self._params.audio_out_10ms_chunks * 10
BOT_SPEAKING_CHUNK_PERIOD = max(int(200 / TOTAL_CHUNK_MS), 1)
bot_speaking_counter = 0
async for frame in self._next_frame():
# Notify the bot started speaking upstream if necessary and that
# it's actually speaking.
if isinstance(frame, TTSAudioRawFrame):
await self._bot_started_speaking()
if bot_speaking_counter % BOT_SPEAKING_CHUNK_PERIOD == 0:
await self.push_frame(BotSpeakingFrame())
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
bot_speaking_counter = 0
bot_speaking_counter += 1
# No need to push EndFrame, it's pushed from process_frame().
if isinstance(frame, EndFrame):
break
# Handle frame.
await self._sink_frame_handler(frame)
# Also, push frame downstream in case anyone else needs it.
await self.push_frame(frame)
# Send audio.
if isinstance(frame, OutputAudioRawFrame):
await self.write_raw_audio_frames(frame.audio)
#
# Video task
#
def _create_video_task(self):
# Create video output queue and task if needed.
if not self._video_out_task and self._params.video_out_enabled:
self._video_out_queue = asyncio.Queue()
self._video_out_task = self.create_task(self._video_out_task_handler())
async def _cancel_video_task(self):
# Stop video output task.
if self._video_out_task and self._params.video_out_enabled:
await self.cancel_task(self._video_out_task)
self._video_out_task = None
async def _draw_image(self, frame: OutputImageRawFrame):
desired_size = (self._params.video_out_width, self._params.video_out_height)
# TODO: we should refactor in the future to support dynamic resolutions
# which is kind of what happens in P2P connections.
# We need to add support for that inside the DailyTransport
if frame.size != desired_size:
image = Image.frombytes(frame.format, frame.size, frame.image)
resized_image = image.resize(desired_size)
# logger.warning(f"{frame} does not have the expected size {desired_size}, resizing")
frame = OutputImageRawFrame(
resized_image.tobytes(), resized_image.size, resized_image.format
)
await self.write_raw_video_frame(frame)
async def _set_video_image(self, image: OutputImageRawFrame):
self._video_images = itertools.cycle([image])
async def _set_video_images(self, images: List[OutputImageRawFrame]):
self._video_images = itertools.cycle(images)
async def _video_out_task_handler(self):
self._video_out_start_time = None
self._video_out_frame_index = 0
self._video_out_frame_duration = 1 / self._params.video_out_framerate
self._video_out_frame_reset = self._video_out_frame_duration * 5
while True:
if self._params.video_out_is_live:
await self._video_out_is_live_handler()
elif self._video_images:
image = next(self._video_images)
await self._draw_image(image)
await asyncio.sleep(self._video_out_frame_duration)
else:
await asyncio.sleep(self._video_out_frame_duration)
async def _video_out_is_live_handler(self):
image = await self._video_out_queue.get()
# We get the start time as soon as we get the first image.
if not self._video_out_start_time:
self._video_out_start_time = time.time()
self._video_out_frame_index = 0
# Calculate how much time we need to wait before rendering next image.
real_elapsed_time = time.time() - self._video_out_start_time
real_render_time = self._video_out_frame_index * self._video_out_frame_duration
delay_time = self._video_out_frame_duration + real_render_time - real_elapsed_time
if abs(delay_time) > self._video_out_frame_reset:
self._video_out_start_time = time.time()
self._video_out_frame_index = 0
elif delay_time > 0:
await asyncio.sleep(delay_time)
self._video_out_frame_index += 1
# Render image
await self._draw_image(image)
self._video_out_queue.task_done()
self._clock_queue.task_done()

6
src/pipecat/transports/base_transport.py
View File

@@ -5,7 +5,7 @@
#
from abc import abstractmethod
from typing import Optional
from typing import List, Mapping, Optional
from pydantic import BaseModel, ConfigDict
@@ -33,7 +33,8 @@ class TransportParams(BaseModel):
audio_out_channels: int = 1
audio_out_bitrate: int = 96000
audio_out_10ms_chunks: int = 4
audio_out_mixer: Optional[BaseAudioMixer] = None
audio_out_mixer: Optional[BaseAudioMixer | Mapping[Optional[str], BaseAudioMixer]] = None
audio_out_destinations: List[str] = []
audio_in_enabled: bool = False
audio_in_sample_rate: Optional[int] = None
audio_in_channels: int = 1
@@ -48,6 +49,7 @@ class TransportParams(BaseModel):
video_out_bitrate: int = 800000
video_out_framerate: int = 30
video_out_color_format: str = "RGB"
video_out_destinations: List[str] = []
vad_enabled: bool = False
vad_audio_passthrough: bool = False
vad_analyzer: Optional[VADAnalyzer] = None

6
src/pipecat/transports/local/audio.py
View File

@@ -61,6 +61,8 @@ class LocalAudioInputTransport(BaseInputTransport):
)
self._in_stream.start_stream()
await self.set_transport_ready(frame)
async def cleanup(self):
await super().cleanup()
if self._in_stream:
@@ -111,6 +113,8 @@ class LocalAudioOutputTransport(BaseOutputTransport):
)
self._out_stream.start_stream()
await self.set_transport_ready(frame)
async def cleanup(self):
await super().cleanup()
if self._out_stream:
@@ -118,7 +122,7 @@ class LocalAudioOutputTransport(BaseOutputTransport):
self._out_stream.close()
self._out_stream = None
async def write_raw_audio_frames(self, frames: bytes):
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
if self._out_stream:
await self.get_event_loop().run_in_executor(
self._executor, self._out_stream.write, frames

10
src/pipecat/transports/local/tk.py
View File

@@ -68,6 +68,8 @@ class TkInputTransport(BaseInputTransport):
)
self._in_stream.start_stream()
await self.set_transport_ready(frame)
async def cleanup(self):
await super().cleanup()
if self._in_stream:
@@ -124,6 +126,8 @@ class TkOutputTransport(BaseOutputTransport):
)
self._out_stream.start_stream()
await self.set_transport_ready(frame)
async def cleanup(self):
await super().cleanup()
if self._out_stream:
@@ -131,13 +135,15 @@ class TkOutputTransport(BaseOutputTransport):
self._out_stream.close()
self._out_stream = None
async def write_raw_audio_frames(self, frames: bytes):
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
if self._out_stream:
await self.get_event_loop().run_in_executor(
self._executor, self._out_stream.write, frames
)
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
def _write_frame_to_tk(self, frame: OutputImageRawFrame):

6
src/pipecat/transports/network/fastapi_websocket.py
View File

@@ -131,6 +131,7 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
await self._client.trigger_client_connected()
if not self._receive_task:
self._receive_task = self.create_task(self._receive_messages())
await self.set_transport_ready(frame)
async def _stop_tasks(self):
if self._monitor_websocket_task:
@@ -203,7 +204,8 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
await super().start(frame)
await self._client.setup(frame)
await self._params.serializer.setup(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
self._send_interval = (self.audio_chunk_size / self.sample_rate) / 2
await self.set_transport_ready(frame)
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -229,7 +231,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._write_frame(frame)
async def write_raw_audio_frames(self, frames: bytes):
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
if self._client.is_closing:
return

14
src/pipecat/transports/network/small_webrtc.py
View File

@@ -284,11 +284,13 @@ class SmallWebRTCClient:
)
yield audio_frame
async def write_raw_audio_frames(self, data: bytes):
async def write_raw_audio_frames(self, data: bytes, destination: Optional[str] = None):
if self._can_send() and self._audio_output_track:
await self._audio_output_track.add_audio_bytes(data)
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
if self._can_send() and self._video_output_track:
self._video_output_track.add_video_frame(frame)
@@ -393,6 +395,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
self._receive_audio_task = self.create_task(self._receive_audio())
if not self._receive_video_task and self._params.video_in_enabled:
self._receive_video_task = self.create_task(self._receive_video())
await self.set_transport_ready(frame)
async def _stop_tasks(self):
if self._receive_audio_task:
@@ -485,6 +488,7 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
await super().start(frame)
await self._client.setup(self._params, frame)
await self._client.connect()
await self.set_transport_ready(frame)
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -497,10 +501,12 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._client.send_message(frame)
async def write_raw_audio_frames(self, frames: bytes):
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
await self._client.write_raw_audio_frames(frames)
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
await self._client.write_raw_video_frame(frame)

20
src/pipecat/transports/network/webrtc_connection.py
View File

@@ -7,9 +7,8 @@
import asyncio
import json
import time
from typing import Any, Literal, Optional, Union
from typing import Any, List, Literal, Optional, Union
from av.frame import Frame
from loguru import logger
from pydantic import BaseModel, TypeAdapter
@@ -24,6 +23,7 @@ try:
RTCSessionDescription,
)
from aiortc.rtcrtpreceiver import RemoteStreamTrack
from av.frame import Frame
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the SmallWebRTC, you need to `pip install pipecat-ai[webrtc]`.")
@@ -87,13 +87,21 @@ class SmallWebRTCTrack:
return getattr(self._track, name)
# Alias so we don't need to expose RTCIceServer
IceServer = RTCIceServer
class SmallWebRTCConnection(BaseObject):
def __init__(self, ice_servers=None):
def __init__(self, ice_servers: Optional[Union[List[str], List[IceServer]]] = None):
super().__init__()
if ice_servers:
self.ice_servers = [RTCIceServer(urls=server) for server in ice_servers]
if not ice_servers:
self.ice_servers: List[IceServer] = []
elif all(isinstance(s, IceServer) for s in ice_servers):
self.ice_servers = ice_servers
elif all(isinstance(s, str) for s in ice_servers):
self.ice_servers = [IceServer(urls=s) for s in ice_servers]
else:
self.ice_servers = []
raise TypeError("ice_servers must be either List[str] or List[RTCIceServer]")
self._connect_invoked = False
self._track_map = {}
self._track_getters = {

6
src/pipecat/transports/network/websocket_client.py
View File

@@ -136,6 +136,7 @@ class WebsocketClientInputTransport(BaseInputTransport):
await self._params.serializer.setup(frame)
await self._session.setup(frame)
await self._session.connect()
await self.set_transport_ready(frame)
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -182,10 +183,11 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
self._send_interval = (self.audio_chunk_size / self.sample_rate) / 2
await self._params.serializer.setup(frame)
await self._session.setup(frame)
await self._session.connect()
await self.set_transport_ready(frame)
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -202,7 +204,7 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._write_frame(frame)
async def write_raw_audio_frames(self, frames: bytes):
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
frame = OutputAudioRawFrame(
audio=frames,
sample_rate=self.sample_rate,

6
src/pipecat/transports/network/websocket_server.py
View File

@@ -83,6 +83,7 @@ class WebsocketServerInputTransport(BaseInputTransport):
await self._params.serializer.setup(frame)
if not self._server_task:
self._server_task = self.create_task(self._server_task_handler())
await self.set_transport_ready(frame)
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -194,7 +195,8 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
self._send_interval = (self.audio_chunk_size / self.sample_rate) / 2
await self.set_transport_ready(frame)
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -218,7 +220,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._write_frame(frame)
async def write_raw_audio_frames(self, frames: bytes):
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
if not self._websocket:
# Simulate audio playback with a sleep.
await self._write_audio_sleep()

346
src/pipecat/transports/services/daily.py
View File

@@ -8,17 +8,13 @@ import asyncio
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Mapping, Optional
from typing import Any, Awaitable, Callable, Dict, Mapping, Optional
import aiohttp
from daily import (
VirtualCameraDevice,
VirtualMicrophoneDevice,
VirtualSpeakerDevice,
)
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import create_default_resampler
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
from pipecat.frames.frames import (
CancelFrame,
@@ -34,10 +30,11 @@ from pipecat.frames.frames import (
TranscriptionFrame,
TransportMessageFrame,
TransportMessageUrgentFrame,
UserAudioRawFrame,
UserImageRawFrame,
UserImageRequestFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup
from pipecat.transcriptions.language import Language
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
@@ -45,7 +42,17 @@ from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.utils.asyncio import BaseTaskManager
try:
from daily import CallClient, Daily, EventHandler
from daily import (
AudioData,
CallClient,
CustomAudioSource,
Daily,
EventHandler,
VideoFrame,
VirtualCameraDevice,
VirtualMicrophoneDevice,
VirtualSpeakerDevice,
)
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
@@ -149,6 +156,8 @@ class DailyParams(TransportParams):
api_url: Daily API base URL
api_key: Daily API authentication key
dialin_settings: Optional settings for dial-in functionality
camera_out_enabled: Whether to enable the main camera output track. If enabled, it still needs `video_out_enabled=True`
microphone_out_enabled: Whether to enable the main microphone track. If enabled, it still needs `audio_out_enabled=True`
transcription_enabled: Whether to enable speech transcription
transcription_settings: Configuration for transcription service
"""
@@ -156,6 +165,8 @@ class DailyParams(TransportParams):
api_url: str = "https://api.daily.co/v1"
api_key: str = ""
dialin_settings: Optional[DailyDialinSettings] = None
camera_out_enabled: bool = True
microphone_out_enabled: bool = True
transcription_enabled: bool = False
transcription_settings: DailyTranscriptionSettings = DailyTranscriptionSettings()
@@ -164,6 +175,7 @@ class DailyCallbacks(BaseModel):
"""Callback handlers for Daily events.
Attributes:
on_active_speaker_changed: Called when the active speaker of the call has changed.
on_joined: Called when bot successfully joined a room.
on_left: Called when bot left a room.
on_error: Called when an error occurs.
@@ -190,6 +202,7 @@ class DailyCallbacks(BaseModel):
on_recording_error: Called when recording encounters an error.
"""
on_active_speaker_changed: Callable[[Mapping[str, Any]], Awaitable[None]]
on_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
on_left: Callable[[], Awaitable[None]]
on_error: Callable[[str], Awaitable[None]]
@@ -275,6 +288,7 @@ class DailyTransportClient(EventHandler):
self._transport_name = transport_name
self._participant_id: str = ""
self._audio_renderers = {}
self._video_renderers = {}
self._transcription_ids = []
self._transcription_status = None
@@ -310,6 +324,7 @@ class DailyTransportClient(EventHandler):
self._camera: Optional[VirtualCameraDevice] = None
self._mic: Optional[VirtualMicrophoneDevice] = None
self._speaker: Optional[VirtualSpeakerDevice] = None
self._audio_sources: Dict[str, CustomAudioSource] = {}
def _camera_name(self):
return f"camera-{self}"
@@ -328,6 +343,14 @@ class DailyTransportClient(EventHandler):
def participant_id(self) -> str:
return self._participant_id
@property
def in_sample_rate(self) -> int:
return self._in_sample_rate
@property
def out_sample_rate(self) -> int:
return self._out_sample_rate
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
if not self._joined:
return
@@ -365,21 +388,47 @@ class DailyTransportClient(EventHandler):
await asyncio.sleep(0.01)
return None
async def write_raw_audio_frames(self, frames: bytes):
if not self._mic:
return None
async def register_audio_destination(self, destination: str):
self._audio_sources[destination] = await self.add_custom_audio_track(destination)
self._client.update_publishing({"customAudio": {destination: True}})
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
future = self._get_event_loop().create_future()
self._mic.write_frames(frames, completion=completion_callback(future))
if not destination and self._mic:
self._mic.write_frames(frames, completion=completion_callback(future))
elif destination and destination in self._audio_sources:
source = self._audio_sources[destination]
source.write_frames(frames, completion=completion_callback(future))
else:
logger.warning(f"{self} unable to write audio frames to destination [{destination}]")
future.set_result(None)
await future
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
if not self._camera:
return None
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
if not destination and self._camera:
self._camera.write_frame(frame.image)
self._camera.write_frame(frame.image)
async def setup(self, setup: FrameProcessorSetup):
if self._task_manager:
return
async def setup(self, frame: StartFrame):
self._task_manager = setup.task_manager
self._callback_task = self._task_manager.create_task(
self._callback_task_handler(),
f"{self}::callback_task",
)
async def cleanup(self):
if self._callback_task and self._task_manager:
await self._task_manager.cancel_task(self._callback_task)
self._callback_task = None
# Make sure we don't block the event loop in case `client.release()`
# takes extra time.
await self._get_event_loop().run_in_executor(self._executor, self._cleanup)
async def start(self, frame: StartFrame):
self._in_sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
@@ -408,13 +457,6 @@ class DailyTransportClient(EventHandler):
)
Daily.select_speaker_device(self._speaker_name())
if not self._task_manager:
self._task_manager = frame.task_manager
self._callback_task = self._task_manager.create_task(
self._callback_task_handler(),
f"{self}::callback_task",
)
async def join(self):
# Transport already joined or joining, ignore.
if self._joined or self._joining:
@@ -480,6 +522,9 @@ class DailyTransportClient(EventHandler):
async def _join(self):
future = self._get_event_loop().create_future()
camera_enabled = self._params.video_out_enabled and self._params.camera_out_enabled
microphone_enabled = self._params.audio_out_enabled and self._params.microphone_out_enabled
self._client.join(
self._room_url,
self._token,
@@ -487,13 +532,13 @@ class DailyTransportClient(EventHandler):
client_settings={
"inputs": {
"camera": {
"isEnabled": self._params.video_out_enabled,
"isEnabled": camera_enabled,
"settings": {
"deviceId": self._camera_name(),
},
},
"microphone": {
"isEnabled": self._params.audio_out_enabled,
"isEnabled": microphone_enabled,
"settings": {
"deviceId": self._mic_name(),
"customConstraints": {
@@ -546,6 +591,10 @@ class DailyTransportClient(EventHandler):
if self._params.transcription_enabled:
await self._stop_transcription()
# Remove any custom tracks, if any.
for track_name, _ in self._audio_sources.items():
await self.remove_custom_audio_track(track_name)
try:
error = await self._leave()
if not error:
@@ -574,14 +623,6 @@ class DailyTransportClient(EventHandler):
self._client.leave(completion=completion_callback(future))
return await asyncio.wait_for(future, timeout=10)
async def cleanup(self):
if self._callback_task and self._task_manager:
await self._task_manager.cancel_task(self._callback_task)
self._callback_task = None
# Make sure we don't block the event loop in case `client.release()`
# takes extra time.
await self._get_event_loop().run_in_executor(self._executor, self._cleanup)
def _cleanup(self):
if self._client:
self._client.release()
@@ -648,6 +689,32 @@ class DailyTransportClient(EventHandler):
if self._joined and self._transcription_status:
await self.update_transcription(self._transcription_ids)
async def capture_participant_audio(
self,
participant_id: str,
callback: Callable,
audio_source: str = "microphone",
):
# Only enable the desired audio source subscription on this participant.
if audio_source in ("microphone", "screenAudio"):
media = {"media": {audio_source: "subscribed"}}
else:
media = {"media": {"customAudio": {audio_source: "subscribed"}}}
await self.update_subscriptions(participant_settings={participant_id: media})
self._audio_renderers.setdefault(participant_id, {})[audio_source] = callback
logger.info(
f"Starting to capture audio from participant {participant_id} to {audio_source}"
)
self._client.set_audio_renderer(
participant_id,
self._audio_data_received,
audio_source=audio_source,
)
async def capture_participant_video(
self,
participant_id: str,
@@ -656,12 +723,15 @@ class DailyTransportClient(EventHandler):
video_source: str = "camera",
color_format: str = "RGB",
):
# Only enable the desired video source subscription on this participant.
await self.update_subscriptions(
participant_settings={participant_id: {"media": {video_source: "subscribed"}}}
)
# Only enable the desired audio source subscription on this participant.
if video_source in ("camera", "screenVideo"):
media = {"media": {video_source: "subscribed"}}
else:
media = {"media": {"customVideo": {video_source: "subscribed"}}}
self._video_renderers[participant_id] = callback
await self.update_subscriptions(participant_settings={participant_id: media})
self._video_renderers.setdefault(participant_id, {})[video_source] = callback
self._client.set_video_renderer(
participant_id,
@@ -670,6 +740,28 @@ class DailyTransportClient(EventHandler):
color_format=color_format,
)
async def add_custom_audio_track(self, track_name: str) -> CustomAudioSource:
future = self._get_event_loop().create_future()
audio_source = CustomAudioSource(self._out_sample_rate, 1)
self._client.add_custom_audio_track(
track_name=track_name,
audio_source=audio_source,
completion=completion_callback(future),
)
await future
return audio_source
async def remove_custom_audio_track(self, track_name: str):
future = self._get_event_loop().create_future()
self._client.remove_custom_audio_track(
track_name=track_name,
completion=completion_callback(future),
)
await future
async def update_transcription(self, participants=None, instance_id=None):
future = self._get_event_loop().create_future()
self._client.update_transcription(
@@ -686,7 +778,15 @@ class DailyTransportClient(EventHandler):
)
await future
async def update_remote_participants(self, remote_participants: Mapping[str, Any] = None):
async def update_publishing(self, publishing_settings: Mapping[str, Any]):
future = self._get_event_loop().create_future()
self._client.update_publishing(
publishing_settings=publishing_settings,
completion=completion_callback(future),
)
await future
async def update_remote_participants(self, remote_participants: Mapping[str, Any]):
future = self._get_event_loop().create_future()
self._client.update_remote_participants(
remote_participants=remote_participants, completion=completion_callback(future)
@@ -698,6 +798,9 @@ class DailyTransportClient(EventHandler):
# Daily (EventHandler)
#
def on_active_speaker_changed(self, participant):
self._call_async_callback(self._callbacks.on_active_speaker_changed, participant)
def on_app_message(self, message: Any, sender: str):
self._call_async_callback(self._callbacks.on_app_message, message, sender)
@@ -773,15 +876,15 @@ class DailyTransportClient(EventHandler):
# Daily (CallClient callbacks)
#
def _video_frame_received(self, participant_id, video_frame):
callback = self._video_renderers[participant_id]
self._call_async_callback(
callback,
participant_id,
video_frame.buffer,
(video_frame.width, video_frame.height),
video_frame.color_format,
)
def _audio_data_received(self, participant_id: str, audio_data: AudioData, audio_source: str):
callback = self._audio_renderers[participant_id][audio_source]
self._call_async_callback(callback, participant_id, audio_data, audio_source)
def _video_frame_received(
self, participant_id: str, video_frame: VideoFrame, video_source: str
):
callback = self._video_renderers[participant_id][video_source]
self._call_async_callback(callback, participant_id, video_frame, video_source)
def _call_async_callback(self, callback, *args):
future = asyncio.run_coroutine_threadsafe(
@@ -837,6 +940,8 @@ class DailyInputTransport(BaseInputTransport):
# internally to be processed.
self._audio_in_task = None
self._resampler = create_default_resampler()
self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer
@property
@@ -850,19 +955,33 @@ class DailyInputTransport(BaseInputTransport):
logger.debug(f"Start receiving audio")
self._audio_in_task = self.create_task(self._audio_in_task_handler())
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._client.setup(setup)
async def cleanup(self):
await super().cleanup()
await self._client.cleanup()
await self._transport.cleanup()
async def start(self, frame: StartFrame):
if self._initialized:
return
self._initialized = True
# Parent start.
await super().start(frame)
# Setup client.
await self._client.setup(frame)
await self._client.start(frame)
# Join the room.
await self._client.join()
# Indicate the transport that we are connected.
await self.set_transport_ready(frame)
if self._params.audio_in_stream_on_start:
self.start_audio_in_streaming()
@@ -886,11 +1005,6 @@ class DailyInputTransport(BaseInputTransport):
await self.cancel_task(self._audio_in_task)
self._audio_in_task = None
async def cleanup(self):
await super().cleanup()
await self._client.cleanup()
await self._transport.cleanup()
#
# FrameProcessor
#
@@ -916,6 +1030,31 @@ class DailyInputTransport(BaseInputTransport):
# Audio in
#
async def capture_participant_audio(
self,
participant_id: str,
audio_source: str = "microphone",
):
await self._client.capture_participant_audio(
participant_id, self._on_participant_audio_data, audio_source
)
async def _on_participant_audio_data(
self, participant_id: str, audio: AudioData, audio_source: str
):
resampled = await self._resampler.resample(
audio.audio_frames, audio.sample_rate, self._client.out_sample_rate
)
frame = UserAudioRawFrame(
user_id=participant_id,
audio=resampled,
sample_rate=self._client.out_sample_rate,
num_channels=audio.num_channels,
)
frame.transport_source = audio_source
await self.push_frame(frame)
async def _audio_in_task_handler(self):
while True:
frame = await self._client.read_next_audio_frame()
@@ -933,7 +1072,10 @@ class DailyInputTransport(BaseInputTransport):
video_source: str = "camera",
color_format: str = "RGB",
):
self._video_renderers[participant_id] = {
if participant_id not in self._video_renderers:
self._video_renderers[participant_id] = {}
self._video_renderers[participant_id][video_source] = {
"framerate": framerate,
"timestamp": 0,
"render_next_frame": [],
@@ -945,14 +1087,17 @@ class DailyInputTransport(BaseInputTransport):
async def request_participant_image(self, frame: UserImageRequestFrame):
if frame.user_id in self._video_renderers:
self._video_renderers[frame.user_id]["render_next_frame"].append(frame)
video_source = frame.video_source if frame.video_source else "camera"
self._video_renderers[frame.user_id][video_source]["render_next_frame"].append(frame)
async def _on_participant_video_frame(self, participant_id: str, buffer, size, format):
async def _on_participant_video_frame(
self, participant_id: str, video_frame: VideoFrame, video_source: str
):
render_frame = False
curr_time = time.time()
prev_time = self._video_renderers[participant_id]["timestamp"]
framerate = self._video_renderers[participant_id]["framerate"]
prev_time = self._video_renderers[participant_id][video_source]["timestamp"]
framerate = self._video_renderers[participant_id][video_source]["framerate"]
# Some times we render frames because of a request.
request_frame = None
@@ -961,20 +1106,23 @@ class DailyInputTransport(BaseInputTransport):
next_time = prev_time + 1 / framerate
render_frame = (next_time - curr_time) < 0.1
elif self._video_renderers[participant_id]["render_next_frame"]:
request_frame = self._video_renderers[participant_id]["render_next_frame"].pop(0)
elif self._video_renderers[participant_id][video_source]["render_next_frame"]:
request_frame = self._video_renderers[participant_id][video_source][
"render_next_frame"
].pop(0)
render_frame = True
if render_frame:
frame = UserImageRawFrame(
user_id=participant_id,
request=request_frame,
image=buffer,
size=size,
format=format,
image=video_frame.buffer,
size=(video_frame.width, video_frame.height),
format=video_frame.color_format,
)
frame.transport_source = video_source
await self.push_frame(frame)
self._video_renderers[participant_id]["timestamp"] = curr_time
self._video_renderers[participant_id][video_source]["timestamp"] = curr_time
class DailyOutputTransport(BaseOutputTransport):
@@ -998,20 +1146,33 @@ class DailyOutputTransport(BaseOutputTransport):
# Whether we have seen a StartFrame already.
self._initialized = False
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._client.setup(setup)
async def cleanup(self):
await super().cleanup()
await self._client.cleanup()
await self._transport.cleanup()
async def start(self, frame: StartFrame):
if self._initialized:
return
self._initialized = True
# Parent start.
await super().start(frame)
# Setup client.
await self._client.setup(frame)
await self._client.start(frame)
# Join the room.
await self._client.join()
# Indicate the transport that we are connected.
await self.set_transport_ready(frame)
async def stop(self, frame: EndFrame):
# Parent stop.
await super().stop(frame)
@@ -1024,19 +1185,22 @@ class DailyOutputTransport(BaseOutputTransport):
# Leave the room.
await self._client.leave()
async def cleanup(self):
await super().cleanup()
await self._client.cleanup()
await self._transport.cleanup()
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._client.send_message(frame)
async def write_raw_audio_frames(self, frames: bytes):
await self._client.write_raw_audio_frames(frames)
async def register_video_destination(self, destination: str):
logger.warning(f"{self} registering video destinations is not supported yet")
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
await self._client.write_raw_video_frame(frame)
async def register_audio_destination(self, destination: str):
await self._client.register_audio_destination(destination)
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
await self._client.write_raw_audio_frames(frames, destination)
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
await self._client.write_raw_video_frame(frame, destination)
class DailyTransport(BaseTransport):
@@ -1066,6 +1230,7 @@ class DailyTransport(BaseTransport):
super().__init__(input_name=input_name, output_name=output_name)
callbacks = DailyCallbacks(
on_active_speaker_changed=self._on_active_speaker_changed,
on_joined=self._on_joined,
on_left=self._on_left,
on_error=self._on_error,
@@ -1101,6 +1266,7 @@ class DailyTransport(BaseTransport):
# Register supported handlers. The user will only be able to register
# these handlers.
self._register_event_handler("on_active_speaker_changed")
self._register_event_handler("on_joined")
self._register_event_handler("on_left")
self._register_event_handler("on_error")
@@ -1204,6 +1370,14 @@ class DailyTransport(BaseTransport):
async def capture_participant_transcription(self, participant_id: str):
await self._client.capture_participant_transcription(participant_id)
async def capture_participant_audio(
self,
participant_id: str,
audio_source: str = "microphone",
):
if self._input:
await self._input.capture_participant_audio(participant_id, audio_source)
async def capture_participant_video(
self,
participant_id: str,
@@ -1216,14 +1390,20 @@ class DailyTransport(BaseTransport):
participant_id, framerate, video_source, color_format
)
async def update_publishing(self, publishing_settings: Mapping[str, Any]):
await self._client.update_publishing(publishing_settings=publishing_settings)
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
await self._client.update_subscriptions(
participant_settings=participant_settings, profile_settings=profile_settings
)
async def update_remote_participants(self, remote_participants: Mapping[str, Any] = None):
async def update_remote_participants(self, remote_participants: Mapping[str, Any]):
await self._client.update_remote_participants(remote_participants=remote_participants)
async def _on_active_speaker_changed(self, participant: Any):
await self._call_event_handler("on_active_speaker_changed", participant)
async def _on_joined(self, data):
await self._call_event_handler("on_joined", data)

54
src/pipecat/transports/services/livekit.py
View File

@@ -23,7 +23,7 @@ from pipecat.frames.frames import (
TransportMessageFrame,
TransportMessageUrgentFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -100,20 +100,27 @@ class LiveKitTransportClient:
raise Exception(f"{self}: missing room object (pipeline not started?)")
return self._room
async def setup(self, frame: StartFrame):
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
if not self._task_manager:
self._task_manager = frame.task_manager
self._room = rtc.Room(loop=self._task_manager.get_event_loop())
async def setup(self, setup: FrameProcessorSetup):
if self._task_manager:
return
# Set up room event handlers
self.room.on("participant_connected")(self._on_participant_connected_wrapper)
self.room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
self.room.on("track_subscribed")(self._on_track_subscribed_wrapper)
self.room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
self.room.on("data_received")(self._on_data_received_wrapper)
self.room.on("connected")(self._on_connected_wrapper)
self.room.on("disconnected")(self._on_disconnected_wrapper)
self._task_manager = setup.task_manager
self._room = rtc.Room(loop=self._task_manager.get_event_loop())
# Set up room event handlers
self.room.on("participant_connected")(self._on_participant_connected_wrapper)
self.room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
self.room.on("track_subscribed")(self._on_track_subscribed_wrapper)
self.room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
self.room.on("data_received")(self._on_data_received_wrapper)
self.room.on("connected")(self._on_connected_wrapper)
self.room.on("disconnected")(self._on_disconnected_wrapper)
async def cleanup(self):
await self.disconnect()
async def start(self, frame: StartFrame):
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
async def connect(self):
@@ -333,9 +340,6 @@ class LiveKitTransportClient:
else:
logger.warning(f"Received unexpected event type: {type(event)}")
async def cleanup(self):
await self.disconnect()
async def get_next_audio_frame(self):
frame, participant_id = await self._audio_queue.get()
return frame, participant_id
@@ -366,10 +370,11 @@ class LiveKitInputTransport(BaseInputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.setup(frame)
await self._client.start(frame)
await self._client.connect()
if not self._audio_in_task and self._params.audio_in_enabled:
self._audio_in_task = self.create_task(self._audio_in_task_handler())
await self.set_transport_ready(frame)
logger.info("LiveKitInputTransport started")
async def stop(self, frame: EndFrame):
@@ -385,6 +390,10 @@ class LiveKitInputTransport(BaseInputTransport):
if self._audio_in_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_in_task)
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._client.setup(setup)
async def cleanup(self):
await super().cleanup()
await self._transport.cleanup()
@@ -439,8 +448,9 @@ class LiveKitOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.setup(frame)
await self._client.start(frame)
await self._client.connect()
await self.set_transport_ready(frame)
logger.info("LiveKitOutputTransport started")
async def stop(self, frame: EndFrame):
@@ -452,6 +462,10 @@ class LiveKitOutputTransport(BaseOutputTransport):
await super().cancel(frame)
await self._client.disconnect()
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._client.setup(setup)
async def cleanup(self):
await super().cleanup()
await self._transport.cleanup()
@@ -462,7 +476,7 @@ class LiveKitOutputTransport(BaseOutputTransport):
else:
await self._client.send_data(frame.message.encode())
async def write_raw_audio_frames(self, frames: bytes):
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
livekit_audio = self._convert_pipecat_audio_to_livekit(frames)
await self._client.publish_audio(livekit_audio)

15
src/pipecat/utils/asyncio.py
View File

@@ -6,7 +6,7 @@
import asyncio
from abc import ABC, abstractmethod
from typing import Coroutine, Optional, Set
from typing import Coroutine, Dict, Optional, Sequence, Set
from loguru import logger
@@ -69,14 +69,14 @@ class BaseTaskManager(ABC):
pass
@abstractmethod
def current_tasks(self) -> Set[asyncio.Task]:
def current_tasks(self) -> Sequence[asyncio.Task]:
"""Returns the list of currently created/registered tasks."""
pass
class TaskManager(BaseTaskManager):
def __init__(self) -> None:
self._tasks: Set[asyncio.Task] = set()
self._tasks: Dict[str, asyncio.Task] = {}
self._loop: Optional[asyncio.AbstractEventLoop] = None
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
@@ -179,16 +179,17 @@ class TaskManager(BaseTaskManager):
finally:
self._remove_task(task)
def current_tasks(self) -> Set[asyncio.Task]:
def current_tasks(self) -> Sequence[asyncio.Task]:
"""Returns the list of currently created/registered tasks."""
return self._tasks
return list(self._tasks.values())
def _add_task(self, task: asyncio.Task):
self._tasks.add(task)
name = task.get_name()
self._tasks[name] = task
def _remove_task(self, task: asyncio.Task):
name = task.get_name()
try:
self._tasks.remove(task)
del self._tasks[name]
except KeyError as e:
logger.trace(f"{name}: unable to remove task (already removed?): {e}")

7
src/pipecat/utils/tracing/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""OpenTelemetry tracing utilities for Pipecat."""

219
src/pipecat/utils/tracing/class_decorators.py Normal file
View File

@@ -0,0 +1,219 @@
#
# Copyright (c) 2024–2025, Daily
# Portions Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base OpenTelemetry tracing decorators and utilities for Pipecat.
This module provides class and method level tracing capabilities
similar to the original NVIDIA implementation.
"""
import asyncio
import contextlib
import enum
import functools
import inspect
from typing import Callable, Optional, TypeVar
from pipecat.utils.tracing.setup import is_tracing_available
# Import OpenTelemetry if available
if is_tracing_available():
import opentelemetry.trace
from opentelemetry import metrics, trace
# Type variables for better typing support
T = TypeVar("T")
C = TypeVar("C", bound=type)
class AttachmentStrategy(enum.Enum):
"""Controls how spans are attached to the trace hierarchy.
Attributes:
CHILD: Attached to class span if no parent, otherwise to parent.
LINK: Attached to class span with link to parent.
NONE: Always attached to class span regardless of context.
"""
CHILD = enum.auto()
LINK = enum.auto()
NONE = enum.auto()
class Traceable:
"""Base class for objects that can be traced with OpenTelemetry.
Provides the foundational tracing capabilities used by @traced methods.
"""
def __init__(self, name: str, **kwargs):
"""Initialize a traceable object.
Args:
name: Name of the traceable object for the span.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
if not is_tracing_available():
self._tracer = self._meter = self._parent_span_id = self._span = None
return
self._tracer = trace.get_tracer("pipecat")
self._meter = metrics.get_meter("pipecat")
self._parent_span_id = trace.get_current_span().get_span_context().span_id
self._span = self._tracer.start_span(name)
self._span.end()
@property
def meter(self):
"""Returns the OpenTelemetry meter instance.
Returns:
Meter: The OpenTelemetry meter instance for this object.
"""
return self._meter
@contextlib.contextmanager
def __traced_context_manager(
self: Traceable, func: Callable, name: str | None, attachment_strategy: AttachmentStrategy
):
"""Internal context manager for the traced decorator."""
if not isinstance(self, Traceable):
raise RuntimeError(
"@traced annotation can only be used in classes inheriting from Traceable"
)
stack = contextlib.ExitStack()
try:
current_span = trace.get_current_span()
is_span_class_parent_span = current_span.get_span_context().span_id == self._parent_span_id
match attachment_strategy:
case AttachmentStrategy.CHILD if not is_span_class_parent_span:
stack.enter_context(
self._tracer.start_as_current_span(func.__name__ if name is None else name) # type: ignore
)
case AttachmentStrategy.LINK:
if is_span_class_parent_span:
link = trace.Link(self._span.get_span_context()) # type: ignore
else:
link = trace.Link(current_span.get_span_context())
stack.enter_context(
opentelemetry.trace.use_span(span=self._span, end_on_exit=False) # type: ignore
)
stack.enter_context(
self._tracer.start_as_current_span( # type: ignore
func.__name__ if name is None else name, links=[link]
)
)
case AttachmentStrategy.NONE | AttachmentStrategy.CHILD:
stack.enter_context(
opentelemetry.trace.use_span(span=self._span, end_on_exit=False) # type: ignore
)
stack.enter_context(
self._tracer.start_as_current_span(func.__name__ if name is None else name) # type: ignore
)
yield
finally:
stack.close()
def __traced_decorator(func, name, attachment_strategy: AttachmentStrategy):
"""Implementation of the traced decorator."""
@functools.wraps(func)
async def coroutine_wrapper(self: Traceable, *args, **kwargs):
exception = None
with __traced_context_manager(self, func, name, attachment_strategy):
try:
return await func(self, *args, **kwargs)
except asyncio.CancelledError as e:
exception = e
if exception:
raise exception
@functools.wraps(func)
async def generator_wrapper(self: Traceable, *args, **kwargs):
exception = None
with __traced_context_manager(self, func, name, attachment_strategy):
try:
async for v in func(self, *args, **kwargs):
yield v
except asyncio.CancelledError as e:
exception = e
if exception:
raise exception
if inspect.iscoroutinefunction(func):
return coroutine_wrapper
if inspect.isasyncgenfunction(func):
return generator_wrapper
raise ValueError("@traced annotation can only be used on async or async generator functions")
def traced(
func: Optional[Callable] = None,
*,
name: Optional[str] = None,
attachment_strategy: AttachmentStrategy = AttachmentStrategy.CHILD,
) -> Callable:
"""Adds tracing to an async function in a Traceable class.
Args:
func: The async function to trace.
name: Custom span name. Defaults to function name.
attachment_strategy: How to attach this span (CHILD, LINK, NONE).
Returns:
Wrapped async function with tracing.
Raises:
RuntimeError: If used in a class not inheriting from Traceable.
ValueError: If used on a non-async function.
"""
if not is_tracing_available():
# Just return the original function or a simple decorator
def decorator(f):
return f
return decorator if func is None else func
if func is not None:
return __traced_decorator(func, name=name, attachment_strategy=attachment_strategy)
else:
return functools.partial(
__traced_decorator, name=name, attachment_strategy=attachment_strategy
)
def traceable(cls: C) -> C:
"""Makes a class traceable for OpenTelemetry.
Creates a new class that inherits from both the original class
and Traceable, enabling tracing for class methods.
Args:
cls: The class to make traceable.
Returns:
A new class with tracing capabilities.
"""
if not is_tracing_available():
return cls
@functools.wraps(cls, updated=())
class TracedClass(cls, Traceable):
def __init__(self, *args, **kwargs):
cls.__init__(self, *args, **kwargs)
if hasattr(self, "name"):
Traceable.__init__(self, self.name)
else:
Traceable.__init__(self, cls.__name__)
return TracedClass

104
src/pipecat/utils/tracing/conversation_context_provider.py Normal file
View File

@@ -0,0 +1,104 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import uuid
from typing import TYPE_CHECKING, Optional
# Import types for type checking only
if TYPE_CHECKING:
from opentelemetry.context import Context
from opentelemetry.trace import SpanContext
from pipecat.utils.tracing.setup import is_tracing_available
if is_tracing_available():
from opentelemetry.context import Context
from opentelemetry.trace import NonRecordingSpan, SpanContext, set_span_in_context
class ConversationContextProvider:
"""Provides access to the current conversation's tracing context.
This is a singleton that can be used to get the current conversation's
span context to create child spans (like turns).
"""
_instance = None
_current_conversation_context: Optional["Context"] = None
_conversation_id: Optional[str] = None
@classmethod
def get_instance(cls):
"""Get the singleton instance."""
if cls._instance is None:
cls._instance = ConversationContextProvider()
return cls._instance
def set_current_conversation_context(
self, span_context: Optional["SpanContext"], conversation_id: Optional[str] = None
):
"""Set the current conversation context.
Args:
span_context: The span context for the current conversation or None to clear it.
conversation_id: Optional ID for the conversation.
"""
if not is_tracing_available():
return
self._conversation_id = conversation_id
if span_context:
# Create a non-recording span from the span context
non_recording_span = NonRecordingSpan(span_context)
self._current_conversation_context = set_span_in_context(non_recording_span)
else:
self._current_conversation_context = None
def get_current_conversation_context(self) -> Optional["Context"]:
"""Get the OpenTelemetry context for the current conversation.
Returns:
The current conversation context or None if not available.
"""
return self._current_conversation_context
def get_conversation_id(self) -> Optional[str]:
"""Get the ID for the current conversation.
Returns:
The current conversation ID or None if not available.
"""
return self._conversation_id
def generate_conversation_id(self) -> str:
"""Generate a new conversation ID.
Returns:
A new randomly generated UUID string.
"""
return str(uuid.uuid4())
# Create a simple helper function to get the current conversation context
def get_current_conversation_context() -> Optional["Context"]:
"""Get the OpenTelemetry context for the current conversation.
Returns:
The current conversation context or None if not available.
"""
provider = ConversationContextProvider.get_instance()
return provider.get_current_conversation_context()
def get_conversation_id() -> Optional[str]:
"""Get the ID for the current conversation.
Returns:
The current conversation ID or None if not available.
"""
provider = ConversationContextProvider.get_instance()
return provider.get_conversation_id()

202
src/pipecat/utils/tracing/service_attributes.py Normal file
View File

@@ -0,0 +1,202 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Functions for adding attributes to OpenTelemetry spans."""
from typing import TYPE_CHECKING, Any, Dict, Optional
# Import for type checking only
if TYPE_CHECKING:
from opentelemetry.trace import Span
from pipecat.utils.tracing.setup import is_tracing_available
if is_tracing_available():
from opentelemetry.trace import Span
def add_tts_span_attributes(
span: "Span",
service_name: str,
model: str,
voice_id: str,
text: Optional[str] = None,
settings: Optional[Dict[str, Any]] = None,
character_count: Optional[int] = None,
operation_name: str = "tts",
ttfb_ms: Optional[float] = None,
**kwargs,
) -> None:
"""Add TTS-specific attributes to a span.
Args:
span: The span to add attributes to
service_name: Name of the TTS service (e.g., "cartesia")
model: Model name/identifier
voice_id: Voice identifier
text: The text being synthesized
settings: Service configuration settings
character_count: Number of characters in the text
operation_name: Name of the operation (default: "tts")
ttfb_ms: Time to first byte in milliseconds
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("service.name", service_name)
span.set_attribute("model", model)
span.set_attribute("voice_id", voice_id)
span.set_attribute("operation", operation_name)
# Add optional attributes
if text:
span.set_attribute("text", text)
if character_count is not None:
span.set_attribute("metrics.tts.character_count", character_count)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
# Add settings if provided
if settings:
for key, value in settings.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(f"settings.{key}", value)
# Add any additional keyword arguments as attributes
for key, value in kwargs.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(key, value)
def add_stt_span_attributes(
span: "Span",
service_name: str,
model: str,
transcript: Optional[str] = None,
is_final: Optional[bool] = None,
language: Optional[str] = None,
settings: Optional[Dict[str, Any]] = None,
vad_enabled: bool = False,
ttfb_ms: Optional[float] = None,
**kwargs,
) -> None:
"""Add STT-specific attributes to a span.
Args:
span: The span to add attributes to
service_name: Name of the STT service (e.g., "deepgram")
model: Model name/identifier
transcript: The transcribed text
is_final: Whether this is a final transcript
language: Detected or configured language
settings: Service configuration settings
vad_enabled: Whether voice activity detection is enabled
ttfb_ms: Time to first byte in milliseconds
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("service.name", service_name)
span.set_attribute("model", model)
span.set_attribute("vad_enabled", vad_enabled)
# Add optional attributes
if transcript:
span.set_attribute("transcript", transcript)
if is_final is not None:
span.set_attribute("is_final", is_final)
if language:
span.set_attribute("language", language)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
# Add settings if provided
if settings:
for key, value in settings.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(f"settings.{key}", value)
# Add any additional keyword arguments as attributes
for key, value in kwargs.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(key, value)
def add_llm_span_attributes(
span: "Span",
service_name: str,
model: str,
stream: bool = True,
messages: Optional[str] = None,
tools: Optional[str] = None,
tool_count: Optional[int] = None,
tool_choice: Optional[str] = None,
system: Optional[str] = None,
parameters: Optional[Dict[str, Any]] = None,
extra_parameters: Optional[Dict[str, Any]] = None,
ttfb_ms: Optional[float] = None,
**kwargs,
) -> None:
"""Add LLM-specific attributes to a span.
Args:
span: The span to add attributes to
service_name: Name of the LLM service (e.g., "openai")
model: Model name/identifier
stream: Whether streaming is enabled
messages: JSON-serialized messages
tools: JSON-serialized tools configuration
tool_count: Number of tools available
tool_choice: Tool selection configuration
system: System message
parameters: Service parameters
extra_parameters: Additional parameters
ttfb_ms: Time to first byte in milliseconds
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("service.name", service_name)
span.set_attribute("model", model)
span.set_attribute("stream", stream)
# Add optional attributes
if messages:
span.set_attribute("messages", messages)
if tools:
span.set_attribute("tools", tools)
if tool_count is not None:
span.set_attribute("tool_count", tool_count)
if tool_choice:
span.set_attribute("tool_choice", tool_choice)
if system:
span.set_attribute("system", system)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
# Add parameters if provided
if parameters:
for key, value in parameters.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(f"param.{key}", value)
# Add extra parameters if provided
if extra_parameters:
for key, value in extra_parameters.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(f"extra.{key}", value)
# Add any additional keyword arguments as attributes
for key, value in kwargs.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(key, value)

452
src/pipecat/utils/tracing/service_decorators.py Normal file
View File

@@ -0,0 +1,452 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Service-specific OpenTelemetry tracing decorators for Pipecat.
This module provides specialized decorators that automatically capture
rich information about service execution including configuration,
parameters, and performance metrics.
"""
import contextlib
import functools
import inspect
import json
import logging
from typing import TYPE_CHECKING, Callable, Optional, TypeVar
# Type imports for type checking only
if TYPE_CHECKING:
from opentelemetry import context as context_api
from opentelemetry import trace
from pipecat.utils.tracing.service_attributes import (
add_llm_span_attributes,
add_stt_span_attributes,
add_tts_span_attributes,
)
from pipecat.utils.tracing.setup import is_tracing_available
from pipecat.utils.tracing.turn_context_provider import get_current_turn_context
if is_tracing_available():
from opentelemetry import context as context_api
from opentelemetry import trace
T = TypeVar("T")
R = TypeVar("R")
# Internal helper functions
def _noop_decorator(func):
"""No-op fallback decorator when tracing is unavailable."""
return func
def _get_parent_service_context(self):
"""Get the parent service span context (internal use only).
This looks for the service span that was created when the service was initialized.
Args:
self: The service instance
Returns:
Context or None: The parent service context, or None if unavailable
"""
if not is_tracing_available():
return None
# The parent span was created when Traceable was initialized and stored as self._span
if hasattr(self, "_span") and self._span:
return trace.set_span_in_context(self._span)
# If we can't find a stored span, default to current context
return context_api.get_current()
def _get_service_name(self, service_prefix: str) -> str:
"""Generate a default span name using service type and class name.
Args:
self: The service instance.
service_prefix: The service type (e.g., 'llm', 'stt', 'tts').
Returns:
A default span name string like "type_classname" (e.g. llm_openaillmservice).
"""
service_class_name = self.__class__.__name__.lower()
return f"{service_prefix}_{service_class_name}"
def _add_token_usage_to_span(span, token_usage):
"""Add token usage metrics to a span (internal use only).
Args:
span: The span to add token metrics to
token_usage: Dictionary or object containing token usage information
"""
if not is_tracing_available() or not token_usage:
return
if isinstance(token_usage, dict):
if "prompt_tokens" in token_usage:
span.set_attribute("llm.prompt_tokens", token_usage["prompt_tokens"])
if "completion_tokens" in token_usage:
span.set_attribute("llm.completion_tokens", token_usage["completion_tokens"])
else:
# Handle LLMTokenUsage object
span.set_attribute("llm.prompt_tokens", getattr(token_usage, "prompt_tokens", 0))
span.set_attribute("llm.completion_tokens", getattr(token_usage, "completion_tokens", 0))
def traced_tts(func: Optional[Callable] = None, *, name: Optional[str] = None) -> Callable:
"""Traces TTS service methods with TTS-specific attributes.
Automatically captures and records:
- Service name and model information
- Voice ID and settings
- Character count and text content
- Performance metrics like TTFB
Works with both async functions and generators.
Args:
func: The TTS method to trace.
name: Custom span name. Defaults to service type and class name.
Returns:
Wrapped method with TTS-specific tracing.
"""
if not is_tracing_available():
return _noop_decorator if func is None else _noop_decorator(func)
def decorator(f):
is_async_generator = inspect.isasyncgenfunction(f)
@contextlib.asynccontextmanager
async def tracing_context(self, text):
"""Async context manager for TTS tracing."""
if not is_tracing_available():
yield None
return
service_class_name = self.__class__.__name__
span_name = name or _get_service_name(self, "tts")
# Get parent context
turn_context = get_current_turn_context()
parent_context = turn_context or _get_parent_service_context(self)
# Create span
tracer = trace.get_tracer("pipecat")
with tracer.start_as_current_span(span_name, context=parent_context) as span:
try:
add_tts_span_attributes(
span=span,
service_name=service_class_name,
model=getattr(self, "model_name", "unknown"),
voice_id=getattr(self, "_voice_id", "unknown"),
text=text,
settings=getattr(self, "_settings", {}),
character_count=len(text),
operation_name="tts",
cartesia_version=getattr(self, "_cartesia_version", None),
context_id=getattr(self, "_context_id", None),
)
yield span
except Exception as e:
logging.warning(f"Error in TTS tracing: {e}")
raise
finally:
# Update TTFB metric at the end
ttfb_ms = getattr(getattr(self, "_metrics", None), "ttfb_ms", None)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
if is_async_generator:
@functools.wraps(f)
async def gen_wrapper(self, text, *args, **kwargs):
try:
if not is_tracing_available():
async for item in f(self, text, *args, **kwargs):
yield item
return
async with tracing_context(self, text):
async for item in f(self, text, *args, **kwargs):
yield item
except Exception as e:
logging.error(f"Error in TTS tracing (continuing without tracing): {e}")
# If tracing fails, fall back to the original function
async for item in f(self, text, *args, **kwargs):
yield item
return gen_wrapper
else:
@functools.wraps(f)
async def wrapper(self, text, *args, **kwargs):
try:
if not is_tracing_available():
return await f(self, text, *args, **kwargs)
async with tracing_context(self, text):
return await f(self, text, *args, **kwargs)
except Exception as e:
logging.error(f"Error in TTS tracing (continuing without tracing): {e}")
# If tracing fails, fall back to the original function
return await f(self, text, *args, **kwargs)
return wrapper
if func is not None:
return decorator(func)
return decorator
def traced_stt(func: Optional[Callable] = None, *, name: Optional[str] = None) -> Callable:
"""Traces STT service methods with transcription attributes.
Automatically captures and records:
- Service name and model information
- Transcription text and final status
- Language information
- Performance metrics like TTFB
Args:
func: The STT method to trace.
name: Custom span name. Defaults to function name.
Returns:
Wrapped method with STT-specific tracing.
"""
if not is_tracing_available():
return _noop_decorator if func is None else _noop_decorator(func)
def decorator(f):
@functools.wraps(f)
async def wrapper(self, transcript, is_final, language=None):
try:
if not is_tracing_available():
return await f(self, transcript, is_final, language)
service_class_name = self.__class__.__name__
span_name = name or _get_service_name(self, "stt")
# Get the turn context first, then fall back to service context
turn_context = get_current_turn_context()
parent_context = turn_context or _get_parent_service_context(self)
# Create a new span as child of the turn span or service span
tracer = trace.get_tracer("pipecat")
with tracer.start_as_current_span(
span_name, context=parent_context
) as current_span:
try:
# Get TTFB metric if available
ttfb_ms = getattr(getattr(self, "_metrics", None), "ttfb_ms", None)
# Use settings from the service if available
settings = getattr(self, "_settings", {})
add_stt_span_attributes(
span=current_span,
service_name=service_class_name,
model=getattr(self, "model_name", settings.get("model", "unknown")),
transcript=transcript,
is_final=is_final,
language=str(language) if language else None,
vad_enabled=getattr(self, "vad_enabled", False),
settings=settings,
ttfb_ms=ttfb_ms,
)
# Call the original function
return await f(self, transcript, is_final, language)
except Exception as e:
# Log any exception but don't disrupt the main flow
logging.warning(f"Error in STT transcription tracing: {e}")
raise
except Exception as e:
logging.error(f"Error in STT tracing (continuing without tracing): {e}")
# If tracing fails, fall back to the original function
return await f(self, transcript, is_final, language)
return wrapper
if func is not None:
return decorator(func)
return decorator
def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) -> Callable:
"""Traces LLM service methods with LLM-specific attributes.
Automatically captures and records:
- Service name and model information
- Context content and messages
- Tool configurations
- Token usage metrics
- Performance metrics like TTFB
Args:
func: The LLM method to trace.
name: Custom span name. Defaults to service type and class name.
Returns:
Wrapped method with LLM-specific tracing.
"""
if not is_tracing_available():
return _noop_decorator if func is None else _noop_decorator(func)
def decorator(f):
@functools.wraps(f)
async def wrapper(self, context, *args, **kwargs):
try:
if not is_tracing_available():
return await f(self, context, *args, **kwargs)
service_class_name = self.__class__.__name__
span_name = name or _get_service_name(self, "llm")
# Get the parent context - turn context if available, otherwise service context
turn_context = get_current_turn_context()
parent_context = turn_context or _get_parent_service_context(self)
# Create a new span as child of the turn span or service span
tracer = trace.get_tracer("pipecat")
with tracer.start_as_current_span(
span_name, context=parent_context
) as current_span:
try:
# For token usage monitoring
original_start_llm_usage_metrics = None
if hasattr(self, "start_llm_usage_metrics"):
original_start_llm_usage_metrics = self.start_llm_usage_metrics
# Override the method to capture token usage
@functools.wraps(original_start_llm_usage_metrics)
async def wrapped_start_llm_usage_metrics(tokens):
# Call the original method
await original_start_llm_usage_metrics(tokens)
# Add token usage to the current span
_add_token_usage_to_span(current_span, tokens)
# Replace the method temporarily
self.start_llm_usage_metrics = wrapped_start_llm_usage_metrics
try:
# Detect if we're using Google's service
is_google_service = "google" in service_class_name.lower()
# Try to get messages based on service type
messages = None
serialized_messages = None
# TODO: Revisit once we unify the messages across services
if is_google_service:
# Handle Google service specifically
if hasattr(context, "get_messages_for_logging"):
messages = context.get_messages_for_logging()
else:
# Handle other services like OpenAI
if hasattr(context, "get_messages"):
messages = context.get_messages()
elif hasattr(context, "messages"):
messages = context.messages
# Serialize messages if available
if messages:
try:
serialized_messages = json.dumps(messages)
except Exception as e:
serialized_messages = f"Error serializing messages: {str(e)}"
# Get tools, system message, etc. based on the service type
tools = getattr(context, "tools", None)
serialized_tools = None
tool_count = 0
if tools:
try:
serialized_tools = json.dumps(tools)
tool_count = len(tools) if isinstance(tools, list) else 1
except Exception as e:
serialized_tools = f"Error serializing tools: {str(e)}"
# Handle system message for different services
system_message = None
if hasattr(context, "system"):
system_message = context.system
elif hasattr(context, "system_message"):
system_message = context.system_message
elif hasattr(self, "_system_instruction"):
system_message = self._system_instruction
# Get settings from the service
params = {}
if hasattr(self, "_settings"):
for key, value in self._settings.items():
if key == "extra":
continue
# Add value directly if it's a basic type
if isinstance(value, (int, float, bool, str)):
params[key] = value
elif value is None or (
hasattr(value, "__name__") and value.__name__ == "NOT_GIVEN"
):
params[key] = "NOT_GIVEN"
# Add all available attributes to the span
attribute_kwargs = {
"service_name": service_class_name,
"model": getattr(self, "model_name", "unknown"),
"stream": True, # Most LLM services use streaming
"parameters": params,
}
# Add optional attributes only if they exist
if serialized_messages:
attribute_kwargs["messages"] = serialized_messages
if serialized_tools:
attribute_kwargs["tools"] = serialized_tools
attribute_kwargs["tool_count"] = tool_count
if system_message:
attribute_kwargs["system"] = system_message
# Add all gathered attributes to the span
add_llm_span_attributes(span=current_span, **attribute_kwargs)
except Exception as e:
logging.warning(f"Error adding initial LLM attributes: {e}")
# Call the original function
return await f(self, context, *args, **kwargs)
finally:
# Restore the original methods if we overrode them
if (
"original_start_llm_usage_metrics" in locals()
and original_start_llm_usage_metrics
):
self.start_llm_usage_metrics = original_start_llm_usage_metrics
# Update TTFB metric
ttfb_ms = getattr(getattr(self, "_metrics", None), "ttfb_ms", None)
if ttfb_ms is not None:
current_span.set_attribute("metrics.ttfb_ms", ttfb_ms)
except Exception as e:
logging.error(f"Error in LLM tracing (continuing without tracing): {e}")
# If tracing fails, fall back to the original function
return await f(self, context, *args, **kwargs)
return wrapper
if func is not None:
return decorator(func)
return decorator

84
src/pipecat/utils/tracing/setup.py Normal file
View File

@@ -0,0 +1,84 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Core OpenTelemetry tracing utilities and setup for Pipecat."""
import os
# Check if OpenTelemetry is available
try:
from opentelemetry import trace
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
OPENTELEMETRY_AVAILABLE = True
except ImportError:
OPENTELEMETRY_AVAILABLE = False
def is_tracing_available() -> bool:
"""Returns True if OpenTelemetry tracing is available and configured.
Returns:
bool: True if tracing is available, False otherwise.
"""
return OPENTELEMETRY_AVAILABLE
def setup_tracing(
service_name: str = "pipecat",
exporter=None, # User-provided exporter
console_export: bool = False,
) -> bool:
"""Set up OpenTelemetry tracing with a user-provided exporter.
Args:
service_name: The name of the service for traces
exporter: A pre-configured OpenTelemetry span exporter instance.
If None, only console export will be available if enabled.
console_export: Whether to also export traces to console (useful for debugging)
Returns:
bool: True if setup was successful, False otherwise
Example:
# With OTLP exporter
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True)
setup_tracing("my-service", exporter=exporter)
"""
if not OPENTELEMETRY_AVAILABLE:
return False
try:
# Create a resource with service info
resource = Resource.create(
{
"service.name": service_name,
"service.instance.id": os.getenv("HOSTNAME", "unknown"),
"deployment.environment": os.getenv("ENVIRONMENT", "development"),
}
)
# Set up the tracer provider with the resource
tracer_provider = TracerProvider(resource=resource)
trace.set_tracer_provider(tracer_provider)
# Add console exporter if requested (good for debugging)
if console_export:
console_exporter = ConsoleSpanExporter()
tracer_provider.add_span_processor(BatchSpanProcessor(console_exporter))
# Add user-provided exporter if available
if exporter:
tracer_provider.add_span_processor(BatchSpanProcessor(exporter))
return True
except Exception as e:
print(f"Error setting up tracing: {e}")
return False

71
src/pipecat/utils/tracing/turn_context_provider.py Normal file
View File

@@ -0,0 +1,71 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import TYPE_CHECKING, Optional
# Import types for type checking only
if TYPE_CHECKING:
from opentelemetry.context import Context
from opentelemetry.trace import SpanContext
from pipecat.utils.tracing.setup import is_tracing_available
if is_tracing_available():
from opentelemetry.context import Context
from opentelemetry.trace import NonRecordingSpan, SpanContext, set_span_in_context
class TurnContextProvider:
"""Provides access to the current turn's tracing context.
This is a singleton that services can use to get the current turn's
span context to create child spans.
"""
_instance = None
_current_turn_context: Optional["Context"] = None
@classmethod
def get_instance(cls):
"""Get the singleton instance."""
if cls._instance is None:
cls._instance = TurnContextProvider()
return cls._instance
def set_current_turn_context(self, span_context: Optional["SpanContext"]):
"""Set the current turn context.
Args:
span_context: The span context for the current turn or None to clear it.
"""
if not is_tracing_available():
return
if span_context:
# Create a non-recording span from the span context
non_recording_span = NonRecordingSpan(span_context)
self._current_turn_context = set_span_in_context(non_recording_span)
else:
self._current_turn_context = None
def get_current_turn_context(self) -> Optional["Context"]:
"""Get the OpenTelemetry context for the current turn.
Returns:
The current turn context or None if not available.
"""
return self._current_turn_context
# Create a simple helper function to get the current turn context
def get_current_turn_context() -> Optional["Context"]:
"""Get the OpenTelemetry context for the current turn.
Returns:
The current turn context or None if not available.
"""
provider = TurnContextProvider.get_instance()
return provider.get_current_turn_context()

205
src/pipecat/utils/tracing/turn_trace_observer.py Normal file
View File

@@ -0,0 +1,205 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import TYPE_CHECKING, Dict, Optional
from loguru import logger
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
from pipecat.utils.tracing.conversation_context_provider import ConversationContextProvider
from pipecat.utils.tracing.setup import is_tracing_available
from pipecat.utils.tracing.turn_context_provider import TurnContextProvider
# Import types for type checking only
if TYPE_CHECKING:
from opentelemetry.trace import Span, SpanContext
if is_tracing_available():
from opentelemetry import trace
from opentelemetry.trace import Span, SpanContext
class TurnTraceObserver(BaseObserver):
"""Observer that creates trace spans for each conversation turn.
This observer uses TurnTrackingObserver to track turns and creates
OpenTelemetry spans for each turn. Service spans (STT, LLM, TTS)
become children of the turn spans.
If conversation tracing is enabled, turns become children of a
conversation span that encapsulates the entire session.
"""
def __init__(self, turn_tracker: TurnTrackingObserver, conversation_id: Optional[str] = None):
super().__init__()
self._turn_tracker = turn_tracker
self._current_span: Optional["Span"] = None
self._current_turn_number: int = 0
self._trace_context_map: Dict[int, "SpanContext"] = {}
self._tracer = trace.get_tracer("pipecat.turn") if is_tracing_available() else None
# Conversation tracking properties
self._conversation_span: Optional["Span"] = None
self._conversation_id = conversation_id
if turn_tracker:
@turn_tracker.event_handler("on_turn_started")
async def on_turn_started(tracker, turn_number):
await self._handle_turn_started(turn_number)
@turn_tracker.event_handler("on_turn_ended")
async def on_turn_ended(tracker, turn_number, duration, was_interrupted):
await self._handle_turn_ended(turn_number, duration, was_interrupted)
async def on_push_frame(self, data: FramePushed):
"""Process a frame without modifying it.
This observer doesn't need to process individual frames as it
relies on turn start/end events from the turn tracker.
"""
pass
def start_conversation_tracing(self, conversation_id: Optional[str] = None):
"""Start a new conversation span.
Args:
conversation_id: Optional custom ID for the conversation. If None, a UUID will be generated.
"""
if not is_tracing_available() or not self._tracer:
return
# Generate a conversation ID if not provided
context_provider = ConversationContextProvider.get_instance()
if conversation_id is None:
conversation_id = context_provider.generate_conversation_id()
logger.debug(f"Generated new conversation ID: {conversation_id}")
self._conversation_id = conversation_id
# Create a new span for this conversation
self._conversation_span = self._tracer.start_span(f"conversation-{conversation_id}")
# Set span attributes
self._conversation_span.set_attribute("conversation.id", conversation_id)
self._conversation_span.set_attribute("conversation.type", "voice")
# Update the conversation context provider
context_provider.set_current_conversation_context(
self._conversation_span.get_span_context(), conversation_id
)
logger.debug(f"Started tracing for Conversation {conversation_id}")
def end_conversation_tracing(self):
"""End the current conversation span and ensure the last turn is closed."""
if not is_tracing_available():
return
# First, ensure any active turn is closed properly
if self._current_span:
# If we have an active turn span, end it with a standard duration
logger.debug(f"Ending Turn {self._current_turn_number} due to conversation end")
self._current_span.set_attribute("turn.was_interrupted", True)
self._current_span.set_attribute("turn.ended_by_conversation_end", True)
self._current_span.end()
self._current_span = None
# Clear the turn context provider
context_provider = TurnContextProvider.get_instance()
context_provider.set_current_turn_context(None)
# Now end the conversation span if it exists
if self._conversation_span:
# End the span
self._conversation_span.end()
self._conversation_span = None
# Clear the context provider
context_provider = ConversationContextProvider.get_instance()
context_provider.set_current_conversation_context(None)
logger.debug(f"Ended tracing for Conversation {self._conversation_id}")
self._conversation_id = None
async def _handle_turn_started(self, turn_number: int):
"""Handle a turn start event by creating a new span."""
if not is_tracing_available() or not self._tracer:
return
# If this is the first turn and no conversation span exists yet,
# start the conversation tracing (will generate ID if needed)
if turn_number == 1 and not self._conversation_span:
self.start_conversation_tracing(self._conversation_id)
# Get the parent context - conversation if available, otherwise use root context
parent_context = None
if self._conversation_span:
context_provider = ConversationContextProvider.get_instance()
parent_context = context_provider.get_current_conversation_context()
# Create a new span for this turn
self._current_span = self._tracer.start_span(f"turn-{turn_number}", context=parent_context)
self._current_turn_number = turn_number
# Set span attributes
self._current_span.set_attribute("turn.number", turn_number)
self._current_span.set_attribute("turn.type", "conversation")
# Add conversation ID attribute if available
if self._conversation_id:
self._current_span.set_attribute("conversation.id", self._conversation_id)
# Store the span context so services can become children of this span
self._trace_context_map[turn_number] = self._current_span.get_span_context()
# Update the context provider so services can access this span
context_provider = TurnContextProvider.get_instance()
context_provider.set_current_turn_context(self._current_span.get_span_context())
logger.debug(f"Started tracing for Turn {turn_number}")
async def _handle_turn_ended(self, turn_number: int, duration: float, was_interrupted: bool):
"""Handle a turn end event by ending the current span."""
if not is_tracing_available() or not self._current_span:
return
# Only end the span if it matches the current turn
if turn_number == self._current_turn_number:
# Set additional attributes
self._current_span.set_attribute("turn.duration_seconds", duration)
self._current_span.set_attribute("turn.was_interrupted", was_interrupted)
# End the span
self._current_span.end()
self._current_span = None
# Clear the context provider
context_provider = TurnContextProvider.get_instance()
context_provider.set_current_turn_context(None)
logger.debug(f"Ended tracing for Turn {turn_number}")
def get_current_turn_context(self) -> Optional["SpanContext"]:
"""Get the span context for the current turn.
This can be used by services to create child spans.
"""
if not is_tracing_available() or not self._current_span:
return None
return self._current_span.get_span_context()
def get_turn_context(self, turn_number: int) -> Optional["SpanContext"]:
"""Get the span context for a specific turn.
This can be used by services to create child spans.
"""
if not is_tracing_available():
return None
return self._trace_context_map.get(turn_number)