wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:pipecat-ai/pipecat

Browse Source
This commit is contained in:
Matej Marinko
2025-06-12 11:32:38 +02:00
parent ee5fea4221 228afe01ed
commit 6d3a38842d
355 changed files with 17096 additions and 6988 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/pipecat/__init__.py
View File

@@ -4,10 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import sys
from importlib.metadata import version
from loguru import logger
__version__ = version("pipecat-ai")
logger.info(f"ᓚᘏᗢ Pipecat {__version__} ᓚᘏᗢ")
logger.info(f"ᓚᘏᗢ Pipecat {__version__} (Python {sys.version}) ᓚᘏᗢ")

0
src/pipecat/audio/interruptions/__init__.py Normal file
View File

38
src/pipecat/audio/interruptions/base_interruption_strategy.py Normal file
View File

@@ -0,0 +1,38 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from abc import ABC, abstractmethod
class BaseInterruptionStrategy(ABC):
"""This is a base class for interruption strategies. Interruption strategies
decide when the user can interrupt the bot while the bot is speaking. For
example, there could be strategies based on audio volume or strategies based
on the number of words the user spoke.
"""
async def append_audio(self, audio: bytes, sample_rate: int):
"""Appends audio to the strategy. Not all strategies handle audio."""
pass
async def append_text(self, text: str):
"""Appends text to the strategy. Not all strategies handle text."""
pass
@abstractmethod
async def should_interrupt(self) -> bool:
"""This is called when the user stops speaking and it's time to decide
whether the user should interrupt the bot. The decision will be based on
the aggregated audio and/or text.
"""
pass
@abstractmethod
async def reset(self):
"""Reset the current accumulated text and/or audio."""
pass

40
src/pipecat/audio/interruptions/min_words_interruption_strategy.py Normal file
View File

@@ -0,0 +1,40 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from loguru import logger
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
"""This is an interruption strategy based on a minimum number of words said
by the user. That is, the strategy will be true if the user has said at
least that amount of words.
"""
def __init__(self, *, min_words: int):
super().__init__()
self._min_words = min_words
self._text = ""
async def append_text(self, text: str):
"""Appends text for later analysis. Not all strategies need to handle
text.
"""
self._text += text
async def should_interrupt(self) -> bool:
word_count = len(self._text.split())
interrupt = word_count >= self._min_words
logger.debug(
f"should_interrupt={interrupt} num_spoken_words={word_count} min_words={self._min_words}"
)
return interrupt
async def reset(self):
self._text = ""

2
src/pipecat/audio/vad/vad_analyzer.py
View File

@@ -71,7 +71,7 @@ class VADAnalyzer(ABC):
self.set_params(self._params)
def set_params(self, params: VADParams):
logger.info(f"Setting VAD params to: {params}")
logger.debug(f"Setting VAD params to: {params}")
self._params = params
self._vad_frames = self.num_frames_required()
self._vad_frames_num_bytes = self._vad_frames * self._num_channels * 2

0
src/pipecat/examples/__init__.py Normal file
View File

64
src/pipecat/examples/daily_runner.py Normal file
View File

@@ -0,0 +1,64 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import os
from typing import Optional
import aiohttp
from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
async def configure(aiohttp_session: aiohttp.ClientSession):
(url, token, _) = await configure_with_args(aiohttp_session)
return (url, token)
async def configure_with_args(
aiohttp_session: aiohttp.ClientSession, parser: Optional[argparse.ArgumentParser] = None
):
if not parser:
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=False, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=False,
help="Daily API Key (needed to create an owner token for the room)",
)
args, unknown = parser.parse_known_args()
url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
key = args.apikey or os.getenv("DAILY_API_KEY")
if not url:
raise Exception(
"No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL."
)
if not key:
raise Exception(
"No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers."
)
daily_rest_helper = DailyRESTHelper(
daily_api_key=key,
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
aiohttp_session=aiohttp_session,
)
# Create a meeting token for the given room with an expiration 1 hour in
# the future.
expiry_time: float = 60 * 60
token = await daily_rest_helper.get_token(url, expiry_time)
return (url, token, args)

263
src/pipecat/examples/run.py Normal file
View File

@@ -0,0 +1,263 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import asyncio
import json
import os
import sys
from contextlib import asynccontextmanager
from typing import Any, Callable, Dict, Mapping, Optional
import aiohttp
import uvicorn
from dotenv import load_dotenv
from fastapi import BackgroundTasks, FastAPI, WebSocket
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse, RedirectResponse
from loguru import logger
from pipecat.serializers.twilio import TwilioFrameSerializer
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.network.fastapi_websocket import (
FastAPIWebsocketParams,
FastAPIWebsocketTransport,
)
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import IceServer, SmallWebRTCConnection
from pipecat.transports.services.daily import DailyParams, DailyTransport
# Load environment variables
load_dotenv(override=True)
def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
if isinstance(transport, SmallWebRTCTransport):
return client.pc_id
elif isinstance(transport, DailyTransport):
return client["id"]
logger.warning(f"Unable to get client id from unsupported transport {type(transport)}")
return ""
async def maybe_capture_participant_camera(
transport: BaseTransport, client: Any, framerate: int = 0
):
if isinstance(transport, DailyTransport):
await transport.capture_participant_video(
client["id"], framerate=framerate, video_source="camera"
)
async def maybe_capture_participant_screen(
transport: BaseTransport, client: Any, framerate: int = 0
):
if isinstance(transport, DailyTransport):
await transport.capture_participant_video(
client["id"], framerate=framerate, video_source="screenVideo"
)
def run_example_daily(
run_example: Callable,
args: argparse.Namespace,
params: DailyParams,
):
logger.info("Running example with DailyTransport...")
from pipecat.examples.daily_runner import configure
async def run():
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
# Run example function with DailyTransport transport arguments.
transport = DailyTransport(room_url, token, "Pipecat", params=params)
await run_example(transport, args, True)
asyncio.run(run())
def run_example_webrtc(
run_example: Callable,
args: argparse.Namespace,
params: TransportParams,
):
logger.info("Running example with SmallWebRTCTransport...")
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
app = FastAPI()
# Store connections by pc_id
pcs_map: Dict[str, SmallWebRTCConnection] = {}
ice_servers = [
IceServer(
urls="stun:stun.l.google.com:19302",
)
]
# Mount the frontend at /
app.mount("/client", SmallWebRTCPrebuiltUI)
@app.get("/", include_in_schema=False)
async def root_redirect():
return RedirectResponse(url="/client/")
@app.post("/api/offer")
async def offer(request: dict, background_tasks: BackgroundTasks):
pc_id = request.get("pc_id")
if pc_id and pc_id in pcs_map:
pipecat_connection = pcs_map[pc_id]
logger.info(f"Reusing existing connection for pc_id: {pc_id}")
await pipecat_connection.renegotiate(
sdp=request["sdp"],
type=request["type"],
restart_pc=request.get("restart_pc", False),
)
else:
pipecat_connection = SmallWebRTCConnection(ice_servers)
await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
@pipecat_connection.event_handler("closed")
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
pcs_map.pop(webrtc_connection.pc_id, None)
# Run example function with SmallWebRTC transport arguments.
transport = SmallWebRTCTransport(params=params, webrtc_connection=pipecat_connection)
background_tasks.add_task(run_example, transport, args, False)
answer = pipecat_connection.get_answer()
# Updating the peer connection inside the map
pcs_map[answer["pc_id"]] = pipecat_connection
return answer
@asynccontextmanager
async def lifespan(app: FastAPI):
yield # Run app
coros = [pc.disconnect() for pc in pcs_map.values()]
await asyncio.gather(*coros)
pcs_map.clear()
uvicorn.run(app, host=args.host, port=args.port)
def run_example_twilio(
run_example: Callable,
args: argparse.Namespace,
params: FastAPIWebsocketParams,
):
logger.info("Running example with FastAPIWebsocketTransport (Twilio)...")
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], # Allow all origins for testing
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
@app.post("/")
async def start_call():
logger.debug("POST TwiML")
xml_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Connect>
<Stream url="wss://{args.proxy}/ws"></Stream>
</Connect>
<Pause length="40"/>
</Response>
"""
return HTMLResponse(content=xml_content, media_type="application/xml")
@app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
logger.debug("WebSocket connection accepted")
# Reading Twilio data.
start_data = websocket.iter_text()
await start_data.__anext__()
call_data = json.loads(await start_data.__anext__())
print(call_data, flush=True)
stream_sid = call_data["start"]["streamSid"]
call_sid = call_data["start"]["callSid"]
# Create websocket transport and update params.
params.add_wav_header = False
params.serializer = TwilioFrameSerializer(
stream_sid=stream_sid,
call_sid=call_sid,
account_sid=os.getenv("TWILIO_ACCOUNT_SID", ""),
auth_token=os.getenv("TWILIO_AUTH_TOKEN", ""),
)
transport = FastAPIWebsocketTransport(websocket=websocket, params=params)
await run_example(transport, args, False)
uvicorn.run(app, host=args.host, port=args.port)
def run_main(
run_example: Callable,
args: argparse.Namespace,
transport_params: Mapping[str, Callable] = {},
):
if args.transport not in transport_params:
logger.error(f"Transport '{args.transport}' not supported by this example")
return
params = transport_params[args.transport]()
match args.transport:
case "daily":
run_example_daily(run_example, args, params)
case "webrtc":
run_example_webrtc(run_example, args, params)
case "twilio":
run_example_twilio(run_example, args, params)
def main(
run_example: Callable,
*,
parser: Optional[argparse.ArgumentParser] = None,
transport_params: Mapping[str, Callable] = {},
):
if not parser:
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
parser.add_argument(
"--host", default="localhost", help="Host for HTTP server (default: localhost)"
)
parser.add_argument(
"--port", type=int, default=7860, help="Port for HTTP server (default: 7860)"
)
parser.add_argument(
"--transport",
"-t",
type=str,
choices=["daily", "webrtc", "twilio"],
default="webrtc",
help="The transport this example should use",
)
parser.add_argument(
"--proxy", "-x", help="A public proxy host name (no protocol, e.g. proxy.example.com)"
)
parser.add_argument("--verbose", "-v", action="count", default=0)
args = parser.parse_args()
# Log level
logger.remove(0)
logger.add(sys.stderr, level="TRACE" if args.verbose else "DEBUG")
# Import the bot file
run_main(run_example, args, transport_params)

74
src/pipecat/frames/frames.py
View File

@@ -15,9 +15,11 @@ from typing import (
Literal,
Mapping,
Optional,
Sequence,
Tuple,
)
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
@@ -228,14 +230,15 @@ class TTSTextFrame(TextFrame):
@dataclass
class TranscriptionFrame(TextFrame):
"""A text frame with transcription-specific data. Will be placed in the
transport's receive queue when a participant speaks.
"""A text frame with transcription-specific data. The `result` field
contains the result from the STT service if available.
"""
user_id: str
timestamp: str
language: Optional[Language] = None
result: Optional[Any] = None
def __str__(self):
return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})"
@@ -243,14 +246,16 @@ class TranscriptionFrame(TextFrame):
@dataclass
class InterimTranscriptionFrame(TextFrame):
"""A text frame with interim transcription-specific data. Will be placed in
the transport's receive queue when a participant speaks.
"""A text frame with interim transcription-specific data. The `result` field
contains the result from the STT service if available.
"""
text: str
user_id: str
timestamp: str
language: Optional[Language] = None
result: Optional[Any] = None
def __str__(self):
return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})"
@@ -413,22 +418,19 @@ class TransportMessageFrame(DataFrame):
@dataclass
class DTMFFrame(DataFrame):
class DTMFFrame:
"""A DTMF button frame"""
button: KeypadEntry
@dataclass
class InputDTMFFrame(DTMFFrame):
"""A DTMF button input"""
class OutputDTMFFrame(DTMFFrame, DataFrame):
"""A DTMF keypress output that will be queued. If your transport supports
multiple dial-out destinations, use the `transport_destination` field to
specify where the DTMF keypress should be sent.
pass
@dataclass
class OutputDTMFFrame(DTMFFrame):
"""A DTMF button output"""
"""
pass
@@ -448,6 +450,7 @@ class StartFrame(SystemFrame):
enable_metrics: bool = False
enable_usage_metrics: bool = False
report_only_initial_ttfb: bool = False
interruption_strategies: List[BaseInterruptionStrategy] = field(default_factory=list)
@dataclass
@@ -643,6 +646,32 @@ class MetricsFrame(SystemFrame):
data: List[MetricsData]
@dataclass
class FunctionCallFromLLM:
"""Represents a function call returned by the LLM to be registered for execution.
Attributes:
function_name (str): The name of the function.
tool_call_id (str): A unique identifier for the function call.
arguments (Mapping[str, Any]): The arguments for the function.
context (OpenAILLMContext): The LLM context.
"""
function_name: str
tool_call_id: str
arguments: Mapping[str, Any]
context: Any
@dataclass
class FunctionCallsStartedFrame(SystemFrame):
"""A frame signaling that one or more function call execution is going to
start."""
function_calls: Sequence[FunctionCallFromLLM]
@dataclass
class FunctionCallInProgressFrame(SystemFrame):
"""A frame signaling that a function call is in progress."""
@@ -677,6 +706,7 @@ class FunctionCallResultFrame(SystemFrame):
tool_call_id: str
arguments: Any
result: Any
run_llm: Optional[bool] = None
properties: Optional[FunctionCallResultProperties] = None
@@ -777,6 +807,24 @@ class VisionImageRawFrame(InputImageRawFrame):
return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})"
@dataclass
class InputDTMFFrame(DTMFFrame, SystemFrame):
"""A DTMF keypress input."""
pass
@dataclass
class OutputDTMFUrgentFrame(DTMFFrame, SystemFrame):
"""A DTMF keypress output that will be sent right away. If your transport
supports multiple dial-out destinations, use the `transport_destination`
field to specify where the DTMF keypress should be sent.
"""
pass
#
# Control frames
#

4
src/pipecat/pipeline/runner.py
View File

@@ -59,7 +59,7 @@ class PipelineRunner(BaseObject):
await asyncio.gather(*[t.stop_when_done() for t in self._tasks.values()])
async def cancel(self):
logger.debug(f"Canceling runner {self}")
logger.debug(f"Cancelling runner {self}")
await asyncio.gather(*[t.cancel() for t in self._tasks.values()])
def _setup_sigint(self):
@@ -72,7 +72,7 @@ class PipelineRunner(BaseObject):
self._sig_task = asyncio.create_task(self._sig_cancel())
async def _sig_cancel(self):
logger.warning(f"Interruption detected. Canceling runner {self}")
logger.warning(f"Interruption detected. Cancelling runner {self}")
await self.cancel()
def _gc_collect(self):

39
src/pipecat/pipeline/task.py
View File

@@ -6,11 +6,12 @@
import asyncio
import time
from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type
from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Sequence, Tuple, Type
from loguru import logger
from pydantic import BaseModel, ConfigDict, Field
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
from pipecat.clocks.base_clock import BaseClock
from pipecat.clocks.system_clock import SystemClock
from pipecat.frames.frames import (
@@ -54,10 +55,11 @@ class PipelineParams(BaseModel):
enable_metrics: Whether to enable metrics collection.
enable_usage_metrics: Whether to enable usage metrics.
heartbeats_period_secs: Period between heartbeats in seconds.
observers: List of observers for monitoring pipeline execution.
observers: [deprecated] Use `observers` arg in `PipelineTask` class.
report_only_initial_ttfb: Whether to report only initial time to first byte.
send_initial_empty_metrics: Whether to send initial empty metrics.
start_metadata: Additional metadata for pipeline start.
interruption_strategies: Strategies for bot interruption behavior.
"""
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -73,6 +75,7 @@ class PipelineParams(BaseModel):
report_only_initial_ttfb: bool = False
send_initial_empty_metrics: bool = True
start_metadata: Dict[str, Any] = Field(default_factory=dict)
interruption_strategies: List[BaseInterruptionStrategy] = Field(default_factory=list)
class PipelineTaskSource(FrameProcessor):
@@ -181,7 +184,9 @@ class PipelineTask(BaseTask):
the idle timeout is reached.
enable_turn_tracking: Whether to enable turn tracking.
enable_turn_tracing: Whether to enable turn tracing.
conversation_id: Optional custom ID for the conversation.
additional_span_attributes: Optional dictionary of attributes to propagate as
OpenTelemetry conversation span attributes.
"""
def __init__(
@@ -202,6 +207,7 @@ class PipelineTask(BaseTask):
enable_turn_tracking: bool = True,
enable_tracing: bool = False,
conversation_id: Optional[str] = None,
additional_span_attributes: Optional[dict] = None,
):
super().__init__()
self._pipeline = pipeline
@@ -214,6 +220,7 @@ class PipelineTask(BaseTask):
self._enable_turn_tracking = enable_turn_tracking
self._enable_tracing = enable_tracing and is_tracing_available()
self._conversation_id = conversation_id
self._additional_span_attributes = additional_span_attributes or {}
if self._params.observers:
import warnings
@@ -232,10 +239,13 @@ class PipelineTask(BaseTask):
observers.append(self._turn_tracking_observer)
if self._enable_tracing and self._turn_tracking_observer:
self._turn_trace_observer = TurnTraceObserver(
self._turn_tracking_observer, conversation_id=self._conversation_id
self._turn_tracking_observer,
conversation_id=self._conversation_id,
additional_span_attributes=self._additional_span_attributes,
)
observers.append(self._turn_trace_observer)
self._finished = False
self._cancelled = False
# This queue receives frames coming from the pipeline upstream.
self._up_queue = asyncio.Queue()
@@ -306,8 +316,8 @@ class PipelineTask(BaseTask):
"""Return the turn trace observer if enabled."""
return self._turn_trace_observer
async def add_observer(self, observer: BaseObserver):
await self._observer.add_observer(observer)
def add_observer(self, observer: BaseObserver):
self._observer.add_observer(observer)
async def remove_observer(self, observer: BaseObserver):
await self._observer.remove_observer(observer)
@@ -346,7 +356,6 @@ class PipelineTask(BaseTask):
async def cancel(self):
"""Stops the running pipeline immediately."""
logger.debug(f"Canceling pipeline task {self}")
await self._cancel()
async def run(self):
@@ -406,12 +415,15 @@ class PipelineTask(BaseTask):
await self.queue_frame(frame)
async def _cancel(self):
# Make sure everything is cleaned up downstream. This is sent
# out-of-band from the main streaming task which is what we want since
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
# Only cancel the push task. Everything else will be cancelled in run().
await self._task_manager.cancel_task(self._process_push_task)
if not self._cancelled:
logger.debug(f"Canceling pipeline task {self}")
self._cancelled = True
# Make sure everything is cleaned up downstream. This is sent
# out-of-band from the main streaming task which is what we want since
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
# Only cancel the push task. Everything else will be cancelled in run().
await self._task_manager.cancel_task(self._process_push_task)
async def _create_tasks(self):
self._process_up_task = self._task_manager.create_task(
@@ -515,6 +527,7 @@ class PipelineTask(BaseTask):
enable_metrics=self._params.enable_metrics,
enable_usage_metrics=self._params.enable_usage_metrics,
report_only_initial_ttfb=self._params.report_only_initial_ttfb,
interruption_strategies=self._params.interruption_strategies,
)
start_frame.metadata = self._params.start_metadata
await self._source.queue_frame(start_frame, FrameDirection.DOWNSTREAM)

23
src/pipecat/pipeline/task_observer.py
View File

@@ -49,21 +49,31 @@ class TaskObserver(BaseObserver):
super().__init__(**kwargs)
self._observers = observers or []
self._task_manager = task_manager
self._proxies: Dict[BaseObserver, Proxy] = {}
self._proxies: Optional[Dict[BaseObserver, Proxy]] = (
None # Becomes a dict after start() is called
)
async def add_observer(self, observer: BaseObserver):
proxy = self._create_proxy(observer)
self._proxies[observer] = proxy
def add_observer(self, observer: BaseObserver):
# Add the observer to the list.
self._observers.append(observer)
# If we already started, create a new proxy for the observer.
# Otherwise, it will be created in start().
if self._started():
proxy = self._create_proxy(observer)
self._proxies[observer] = proxy
async def remove_observer(self, observer: BaseObserver):
# If the observer has a proxy, remove it.
if observer in self._proxies:
proxy = self._proxies[observer]
# Remove the proxy so it doesn't get called anymore.
del self._proxies[observer]
# Cancel the proxy task right away.
await self._task_manager.cancel_task(proxy.task)
# Remove the observer.
# Remove the observer from the list.
if observer in self._observers:
self._observers.remove(observer)
async def start(self):
@@ -79,6 +89,9 @@ class TaskObserver(BaseObserver):
for proxy in self._proxies.values():
await proxy.queue.put(data)
def _started(self) -> bool:
return self._proxies is not None
def _create_proxy(self, observer: BaseObserver) -> Proxy:
queue = asyncio.Queue()
task = self._task_manager.create_task(

143
src/pipecat/processors/aggregators/dtmf_aggregator.py Normal file
View File

@@ -0,0 +1,143 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from typing import Optional
from pipecat.frames.frames import (
BotInterruptionFrame,
CancelFrame,
EndFrame,
Frame,
InputDTMFFrame,
KeypadEntry,
StartFrame,
TranscriptionFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.time import time_now_iso8601
class DTMFAggregator(FrameProcessor):
"""Aggregates DTMF frames into meaningful sequences for LLM processing.
The aggregator accumulates digits from InputDTMFFrame instances and flushes
when:
- Timeout occurs (configurable idle period)
- Termination digit is received (default: '#')
- EndFrame or CancelFrame is received
Emits TranscriptionFrame for compatibility with existing LLM context aggregators.
Args:
timeout: Idle timeout in seconds before flushing
termination_digit: Digit that triggers immediate flush
prefix: Prefix added to DTMF sequence in transcription
"""
def __init__(
self,
timeout: float = 2.0,
termination_digit: KeypadEntry = KeypadEntry.POUND,
prefix: str = "DTMF: ",
**kwargs,
):
super().__init__(**kwargs)
self._aggregation = ""
self._idle_timeout = timeout
self._termination_digit = termination_digit
self._prefix = prefix
self._digit_event = asyncio.Event()
self._aggregation_task: Optional[asyncio.Task] = None
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
self._create_aggregation_task()
await self.push_frame(frame, direction)
elif isinstance(frame, (EndFrame, CancelFrame)):
if self._aggregation:
await self._flush_aggregation()
await self._stop_aggregation_task()
await self.push_frame(frame, direction)
elif isinstance(frame, InputDTMFFrame):
# Push the DTMF frame downstream first
await self.push_frame(frame, direction)
# Then handle it in order for the TranscriptionFrame to be emitted
# after the InputDTMFFrame
await self._handle_dtmf_frame(frame)
else:
await self.push_frame(frame, direction)
async def _handle_dtmf_frame(self, frame: InputDTMFFrame):
"""Handle DTMF input frame."""
is_first_digit = not self._aggregation
digit_value = frame.button.value
self._aggregation += digit_value
# For first digit, schedule interruption in separate task
if is_first_digit:
asyncio.create_task(self._send_interruption_task())
# Check for immediate flush conditions
if frame.button == self._termination_digit:
await self._flush_aggregation()
else:
# Signal digit received for timeout handling
self._digit_event.set()
async def _send_interruption_task(self):
"""Send interruption frame safely in a separate task."""
try:
# Send the interruption frame
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
except Exception as e:
# Log error but don't propagate
print(f"Error sending interruption: {e}")
def _create_aggregation_task(self) -> None:
"""Creates the aggregation task if it hasn't been created yet."""
if not self._aggregation_task:
self._aggregation_task = self.create_task(self._aggregation_task_handler())
async def _stop_aggregation_task(self) -> None:
"""Stops the aggregation task."""
if self._aggregation_task:
await self.cancel_task(self._aggregation_task)
self._aggregation_task = None
async def _aggregation_task_handler(self):
"""Background task that handles timeout-based flushing."""
while True:
try:
await asyncio.wait_for(self._digit_event.wait(), timeout=self._idle_timeout)
self._digit_event.clear()
except asyncio.TimeoutError:
if self._aggregation:
await self._flush_aggregation()
async def _flush_aggregation(self):
"""Flush the current aggregation as a TranscriptionFrame."""
if not self._aggregation:
return
sequence = self._aggregation
transcription_text = f"{self._prefix}{sequence}"
transcription_frame = TranscriptionFrame(
text=transcription_text, user_id="", timestamp=time_now_iso8601()
)
await self.push_frame(transcription_frame)
self._aggregation = ""
async def cleanup(self) -> None:
"""Clean up resources."""
await super().cleanup()
await self._stop_aggregation_task()

95
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -11,7 +11,9 @@ from typing import Dict, List, Literal, Optional, Set
from loguru import logger
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
from pipecat.frames.frames import (
BotInterruptionFrame,
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
CancelFrame,
@@ -22,6 +24,8 @@ from pipecat.frames.frames import (
FunctionCallCancelFrame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
FunctionCallsStartedFrame,
InputAudioRawFrame,
InterimTranscriptionFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
@@ -159,7 +163,7 @@ class BaseLLMResponseAggregator(FrameProcessor):
pass
@abstractmethod
def reset(self):
async def reset(self):
"""Reset the internals of this aggregator. This should not modify the
internal messages.
"""
@@ -193,7 +197,7 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator):
self._context = context
self._role = role
self._aggregation = ""
self._aggregation: str = ""
@property
def messages(self) -> List[dict]:
@@ -226,7 +230,7 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator):
def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict):
self._context.set_tool_choice(tool_choice)
def reset(self):
async def reset(self):
self._aggregation = ""
@@ -269,10 +273,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
self._aggregation_event = asyncio.Event()
self._aggregation_task = None
def reset(self):
super().reset()
async def reset(self):
await super().reset()
self._seen_interim_results = False
self._waiting_for_aggregation = False
[await s.reset() for s in self._interruption_strategies]
async def handle_aggregation(self, aggregation: str):
self._context.add_message({"role": self.role, "content": aggregation})
@@ -293,6 +298,9 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
elif isinstance(frame, CancelFrame):
await self._cancel(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, InputAudioRawFrame):
await self._handle_input_audio(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, UserStartedSpeakingFrame):
await self._handle_user_started_speaking(frame)
await self.push_frame(frame, direction)
@@ -320,18 +328,42 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
else:
await self.push_frame(frame, direction)
async def _process_aggregation(self):
"""Process the current aggregation and push it downstream."""
aggregation = self._aggregation
await self.reset()
await self.handle_aggregation(aggregation)
frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)
async def push_aggregation(self):
"""Pushes the current aggregation based on interruption strategies and conditions."""
if len(self._aggregation) > 0:
aggregation = self._aggregation
if self.interruption_strategies and self._bot_speaking:
should_interrupt = await self._should_interrupt_based_on_strategies()
# Reset the aggregation. Reset it before pushing it down, otherwise
# if the tasks gets cancelled we won't be able to clear things up.
self.reset()
if should_interrupt:
logger.debug(
"Interruption conditions met - pushing BotInterruptionFrame and aggregation"
)
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
await self._process_aggregation()
else:
logger.debug("Interruption conditions not met - not pushing aggregation")
# Don't process aggregation, just reset it
await self.reset()
else:
# No interruption config - normal behavior (always push aggregation)
await self._process_aggregation()
await self.handle_aggregation(aggregation)
async def _should_interrupt_based_on_strategies(self) -> bool:
"""Check if interruption should occur based on configured strategies."""
frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)
async def should_interrupt(strategy: BaseInterruptionStrategy):
await strategy.append_text(self._aggregation)
return await strategy.should_interrupt()
return any([await should_interrupt(s) for s in self._interruption_strategies])
async def _start(self, frame: StartFrame):
self._create_aggregation_task()
@@ -342,6 +374,10 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
async def _cancel(self, frame: CancelFrame):
await self._cancel_aggregation_task()
async def _handle_input_audio(self, frame: InputAudioRawFrame):
for s in self.interruption_strategies:
await s.append_audio(frame.audio, frame.sample_rate)
async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame):
self._user_speaking = True
self._waiting_for_aggregation = True
@@ -427,7 +463,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
# If we reached this case and the bot is speaking, let's ignore
# what the user said.
logger.debug("Ignoring user speaking emulation, bot is speaking.")
self.reset()
await self.reset()
else:
# The bot is not speaking so, let's trigger user speaking
# emulation.
@@ -465,7 +501,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
self._params.expect_stripped_words = kwargs["expect_stripped_words"]
self._started = 0
self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {}
self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {}
self._context_updated_tasks: Set[asyncio.Task] = set()
async def handle_aggregation(self, aggregation: str):
@@ -503,6 +539,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
self.set_tools(frame.tools)
elif isinstance(frame, LLMSetToolChoiceFrame):
self.set_tool_choice(frame.tool_choice)
elif isinstance(frame, FunctionCallsStartedFrame):
await self._handle_function_calls_started(frame)
elif isinstance(frame, FunctionCallInProgressFrame):
await self._handle_function_call_in_progress(frame)
elif isinstance(frame, FunctionCallResultFrame):
@@ -522,7 +560,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
return
aggregation = self._aggregation.strip()
self.reset()
await self.reset()
if aggregation:
await self.handle_aggregation(aggregation)
@@ -537,7 +575,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
async def _handle_interruptions(self, frame: StartInterruptionFrame):
await self.push_aggregation()
self._started = 0
self.reset()
await self.reset()
async def _handle_function_calls_started(self, frame: FunctionCallsStartedFrame):
function_names = [f"{f.function_name}:{f.tool_call_id}" for f in frame.function_calls]
logger.debug(f"{self} FunctionCallsStartedFrame: {function_names}")
for function_call in frame.function_calls:
self._function_calls_in_progress[function_call.tool_call_id] = None
async def _handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
logger.debug(
@@ -562,19 +606,22 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
await self.handle_function_call_result(frame)
run_llm = False
# Run inference if the function call result requires it.
if frame.result:
run_llm = False
if properties and properties.run_llm is not None:
# If the tool call result has a run_llm property, use it
# If the tool call result has a run_llm property, use it.
run_llm = properties.run_llm
elif frame.run_llm is not None:
# If the frame is indicating we should run the LLM, do it.
run_llm = frame.run_llm
else:
# Default behavior is to run the LLM if there are no function calls in progress
# If this is the last function call in progress, run the LLM.
run_llm = not bool(self._function_calls_in_progress)
if run_llm:
await self.push_context_frame(FrameDirection.UPSTREAM)
if run_llm:
await self.push_context_frame(FrameDirection.UPSTREAM)
# Call the `on_context_updated` callback once the function call result
# is added to the context. Also, run this in a separate task to make
@@ -653,7 +700,7 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
# Reset the aggregation. Reset it before pushing it down, otherwise
# if the tasks gets cancelled we won't be able to clear things up.
self.reset()
await self.reset()
frame = LLMMessagesFrame(self._context.messages)
await self.push_frame(frame)
@@ -675,7 +722,7 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
# Reset the aggregation. Reset it before pushing it down, otherwise
# if the tasks gets cancelled we won't be able to clear things up.
self.reset()
await self.reset()
frame = LLMMessagesFrame(self._context.messages)
await self.push_frame(frame)

2
src/pipecat/processors/aggregators/openai_llm_context.py
View File

@@ -106,7 +106,7 @@ class OpenAILLMContext:
if "mime_type" in msg and msg["mime_type"].startswith("image/"):
msg["data"] = "..."
msgs.append(msg)
return json.dumps(msgs)
return json.dumps(msgs, ensure_ascii=False)
def from_standard_message(self, message):
"""Convert from OpenAI message format to OpenAI message format (passthrough).

2
src/pipecat/processors/aggregators/user_response.py
View File

@@ -23,4 +23,4 @@ class UserResponseAggregator(LLMUserResponseAggregator):
await self.push_frame(frame)
# Reset our accumulator state.
self.reset()
await self.reset()

9
src/pipecat/processors/frame_processor.py
View File

@@ -7,10 +7,11 @@
import asyncio
from dataclasses import dataclass
from enum import Enum
from typing import Awaitable, Callable, Coroutine, Optional
from typing import Awaitable, Callable, Coroutine, List, Optional, Sequence
from loguru import logger
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
from pipecat.clocks.base_clock import BaseClock
from pipecat.frames.frames import (
CancelFrame,
@@ -67,6 +68,7 @@ class FrameProcessor(BaseObject):
self._enable_metrics = False
self._enable_usage_metrics = False
self._report_only_initial_ttfb = False
self._interruption_strategies: List[BaseInterruptionStrategy] = []
# Indicates whether we have received the StartFrame.
self.__started = False
@@ -119,6 +121,10 @@ class FrameProcessor(BaseObject):
def report_only_initial_ttfb(self):
return self._report_only_initial_ttfb
@property
def interruption_strategies(self) -> Sequence[BaseInterruptionStrategy]:
return self._interruption_strategies
def can_generate_metrics(self) -> bool:
return False
@@ -272,6 +278,7 @@ class FrameProcessor(BaseObject):
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
self._interruption_strategies = frame.interruption_strategies
self.__create_input_task()
self.__create_push_task()

2
src/pipecat/processors/metrics/frame_processor_metrics.py
View File

@@ -28,7 +28,7 @@ class FrameProcessorMetrics:
self._should_report_ttfb = True
@property
def ttfb_ms(self) -> Optional[float]:
def ttfb(self) -> Optional[float]:
"""Get the current TTFB value in seconds.
Returns:

161
src/pipecat/serializers/exotel.py Normal file
View File

@@ -0,0 +1,161 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import base64
import json
from typing import Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
InputAudioRawFrame,
InputDTMFFrame,
KeypadEntry,
StartFrame,
StartInterruptionFrame,
TransportMessageFrame,
TransportMessageUrgentFrame,
)
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
class ExotelFrameSerializer(FrameSerializer):
"""Serializer for Exotel Media Streams WebSocket protocol.
This serializer handles converting between Pipecat frames and Exotel's WebSocket
media streams protocol. It supports audio conversion, DTMF events, and automatic
call termination.
Ref Doc for events - https://support.exotel.com/support/solutions/articles/3000108630-working-with-the-stream-and-voicebot-applet
"""
class InputParams(BaseModel):
"""Configuration parameters for ExotelFrameSerializer.
Attributes:
exotel_sample_rate: Sample rate used by Exotel, defaults to 8000 Hz.
sample_rate: Optional override for pipeline input sample rate.
"""
exotel_sample_rate: int = 8000
sample_rate: Optional[int] = None
def __init__(
self, stream_sid: str, call_sid: Optional[str] = None, params: Optional[InputParams] = None
):
"""Initialize the ExotelFrameSerializer.
Args:
stream_sid: The Exotel Media Stream SID.
call_sid: The associated Exotel Call SID (optional, not used in this implementation).
params: Configuration parameters.
"""
self._stream_sid = stream_sid
self._call_sid = call_sid
self._params = params or ExotelFrameSerializer.InputParams()
self._exotel_sample_rate = self._params.exotel_sample_rate
self._sample_rate = 0 # Pipeline input rate
self._resampler = create_default_resampler()
@property
def type(self) -> FrameSerializerType:
"""Gets the serializer type.
Returns:
The serializer type, either TEXT or BINARY.
"""
return FrameSerializerType.TEXT
async def setup(self, frame: StartFrame):
"""Sets up the serializer with pipeline configuration.
Args:
frame: The StartFrame containing pipeline configuration.
"""
self._sample_rate = self._params.sample_rate or frame.audio_in_sample_rate
async def serialize(self, frame: Frame) -> str | bytes | None:
"""Serializes a Pipecat frame to Exotel WebSocket format.
Handles conversion of various frame types to Exotel WebSocket messages.
Args:
frame: The Pipecat frame to serialize.
Returns:
Serialized data as string or bytes, or None if the frame isn't handled.
"""
if isinstance(frame, StartInterruptionFrame):
answer = {"event": "clear", "streamSid": self._stream_sid}
return json.dumps(answer)
elif isinstance(frame, AudioRawFrame):
data = frame.audio
# Output: Exotel outputs PCM audio, but we need to resample to match requested sample_rate
serialized_data = await self._resampler.resample(
data, frame.sample_rate, self._exotel_sample_rate
)
payload = base64.b64encode(serialized_data).decode("ascii")
answer = {
"event": "media",
"streamSid": self._stream_sid,
"media": {"payload": payload},
}
return json.dumps(answer)
elif isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
return json.dumps(frame.message)
return None
async def deserialize(self, data: str | bytes) -> Frame | None:
"""Deserializes Exotel WebSocket data to Pipecat frames.
Handles conversion of Exotel media events to appropriate Pipecat frames.
Args:
data: The raw WebSocket data from Exotel.
Returns:
A Pipecat frame corresponding to the Exotel event, or None if unhandled.
"""
message = json.loads(data)
if message["event"] == "media":
payload_base64 = message["media"]["payload"]
payload = base64.b64decode(payload_base64)
deserialized_data = await self._resampler.resample(
payload,
self._exotel_sample_rate,
self._sample_rate,
)
# Input: Exotel takes PCM data, so just resample to match sample_rate
audio_frame = InputAudioRawFrame(
audio=deserialized_data,
num_channels=1, # Assuming mono audio from Exotel
sample_rate=self._sample_rate, # Use the configured pipeline input rate
)
return audio_frame
elif message["event"] == "dtmf":
digit = message.get("dtmf", {}).get("digit")
try:
return InputDTMFFrame(KeypadEntry(digit))
except ValueError:
# Handle case where string doesn't match any enum value
logger.info(f"Invalid DTMF digit: {digit}")
return None
return None

15
src/pipecat/serializers/protobuf.py
View File

@@ -41,6 +41,7 @@ class ProtobufFrameSerializer(FrameSerializer):
TextFrame: "text",
InputAudioRawFrame: "audio",
TranscriptionFrame: "transcription",
MessageFrame: "message",
}
DESERIALIZABLE_FIELDS = {v: k for k, v in DESERIALIZABLE_TYPES.items()}
@@ -97,8 +98,18 @@ class ProtobufFrameSerializer(FrameSerializer):
if "pts" in args_dict:
del args_dict["pts"]
# Create the instance
instance = class_name(**args_dict)
# Special handling for MessageFrame -> TransportMessageUrgentFrame
if class_name == MessageFrame:
try:
msg = json.loads(args_dict["data"])
instance = TransportMessageUrgentFrame(message=msg)
logger.debug(f"ProtobufFrameSerializer: Transport message {instance}")
except Exception as e:
logger.error(f"Error parsing MessageFrame data: {e}")
return None
else:
# Normal deserialization, create the instance
instance = class_name(**args_dict)
# Set special fields
if id:

20
src/pipecat/services/anthropic/llm.py
View File

@@ -45,7 +45,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -202,9 +202,8 @@ class AnthropicLLMService(LLMService):
tool_use_block = None
json_accumulator = ""
function_calls = []
async for event in response:
# logger.debug(f"Anthropic LLM event: {event}")
# Aggregate streaming content, create frames, trigger events
if event.type == "content_block_delta":
@@ -226,11 +225,14 @@ class AnthropicLLMService(LLMService):
and event.delta.stop_reason == "tool_use"
):
if tool_use_block:
await self.call_function(
context=context,
tool_call_id=tool_use_block.id,
function_name=tool_use_block.name,
arguments=json.loads(json_accumulator) if json_accumulator else dict(),
args = json.loads(json_accumulator) if json_accumulator else {}
function_calls.append(
FunctionCallFromLLM(
context=context,
tool_call_id=tool_use_block.id,
function_name=tool_use_block.name,
arguments=args,
)
)
# Calculate usage. Do this here in its own if statement, because there may be usage
@@ -277,6 +279,8 @@ class AnthropicLLMService(LLMService):
if total_input_tokens >= 1024:
context.turns_above_cache_threshold += 1
await self.run_function_calls(function_calls)
except asyncio.CancelledError:
# If we're interrupted, we won't get a complete usage report. So set our flag to use the
# token estimate. The reraise the exception so all the processors running in this task

61
src/pipecat/services/assemblyai/models.py Normal file
View File

@@ -0,0 +1,61 @@
from typing import List, Literal, Optional
from pydantic import BaseModel, Field
class Word(BaseModel):
"""Represents a single word in a transcription with timing and confidence."""
start: int
end: int
text: str
confidence: float
word_is_final: bool = Field(..., alias="word_is_final")
class BaseMessage(BaseModel):
"""Base class for all AssemblyAI WebSocket messages."""
type: str
class BeginMessage(BaseMessage):
"""Message sent when a new session begins."""
type: Literal["Begin"] = "Begin"
id: str
expires_at: int
class TurnMessage(BaseMessage):
"""Message containing transcription data for a turn of speech."""
type: Literal["Turn"] = "Turn"
turn_order: int
turn_is_formatted: bool
end_of_turn: bool
transcript: str
end_of_turn_confidence: float
words: List[Word]
class TerminationMessage(BaseMessage):
"""Message sent when the session is terminated."""
type: Literal["Termination"] = "Termination"
audio_duration_seconds: float
session_duration_seconds: float
# Union type for all possible message types
AnyMessage = BeginMessage | TurnMessage | TerminationMessage
class AssemblyAIConnectionParams(BaseModel):
sample_rate: int = 16000
encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le"
formatted_finals: bool = True
word_finalization_max_wait_time: Optional[int] = None
end_of_turn_confidence_threshold: Optional[float] = None
min_end_of_turn_silence_when_confident: Optional[int] = None
max_turn_silence: Optional[int] = None

311
src/pipecat/services/assemblyai/stt.py
View File

@@ -5,30 +5,42 @@
#
import asyncio
from typing import AsyncGenerator, Optional
import json
from typing import Any, AsyncGenerator, Dict
from urllib.parse import urlencode
from loguru import logger
from pipecat import __version__ as pipecat_version
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
from .models import (
AssemblyAIConnectionParams,
BaseMessage,
BeginMessage,
TerminationMessage,
TurnMessage,
)
try:
import assemblyai as aai
from assemblyai import AudioEncoding
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use AssemblyAI, you need to `pip install pipecat-ai[assemblyai]`.")
logger.error('In order to use AssemblyAI, you need to `pip install "pipecat-ai[assemblyai]"`.')
raise Exception(f"Missing module: {e}")
@@ -37,31 +49,37 @@ class AssemblyAISTTService(STTService):
self,
*,
api_key: str,
sample_rate: Optional[int] = None,
encoding: Optional[AudioEncoding] = None,
language=Language.EN, # Only English is supported for Realtime
language: Language = Language.EN, # AssemblyAI only supports English
api_endpoint_base_url: str = "wss://streaming.assemblyai.com/v3/ws",
connection_params: AssemblyAIConnectionParams = AssemblyAIConnectionParams(),
vad_force_turn_endpoint: bool = True,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._language = language
self._api_endpoint_base_url = api_endpoint_base_url
self._connection_params = connection_params
self._vad_force_turn_endpoint = vad_force_turn_endpoint
encoding = encoding or AudioEncoding("pcm_s16le")
aai.settings.api_key = api_key
self._transcriber: Optional[aai.RealtimeTranscriber] = None
super().__init__(sample_rate=self._connection_params.sample_rate, **kwargs)
self._settings = {
"encoding": encoding,
"language": language,
}
self._websocket = None
self._termination_event = asyncio.Event()
self._received_termination = False
self._connected = False
self._receive_task = None
self._audio_buffer = bytearray()
self._chunk_size_ms = 50
self._chunk_size_bytes = 0
def can_generate_metrics(self) -> bool:
return True
async def set_language(self, language: Language):
logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language
async def start(self, frame: StartFrame):
await super().start(frame)
self._chunk_size_bytes = int(self._chunk_size_ms * self._sample_rate * 2 / 1000)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -73,97 +91,182 @@ class AssemblyAISTTService(STTService):
await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process an audio chunk for STT transcription.
self._audio_buffer.extend(audio)
This method streams the audio data to AssemblyAI for real-time transcription.
Transcription results are handled asynchronously via callback functions.
while len(self._audio_buffer) >= self._chunk_size_bytes:
chunk = bytes(self._audio_buffer[: self._chunk_size_bytes])
self._audio_buffer = self._audio_buffer[self._chunk_size_bytes :]
await self._websocket.send(chunk)
:param audio: Audio data as bytes
:yield: None (transcription frames are pushed via self.push_frame in callbacks)
"""
if self._transcriber:
await self.start_ttfb_metrics()
await self.start_processing_metrics()
self._transcriber.stream(audio)
yield None
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame):
await self.start_ttfb_metrics()
elif isinstance(frame, UserStoppedSpeakingFrame):
if self._vad_force_turn_endpoint:
await self._websocket.send(json.dumps({"type": "ForceEndpoint"}))
await self.start_processing_metrics()
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
async def _trace_transcription(self, transcript: str, is_final: bool, language: Language):
"""Record transcription event for tracing."""
pass
def _build_ws_url(self) -> str:
"""Build WebSocket URL with query parameters using urllib.parse.urlencode."""
params = {
k: str(v).lower() if isinstance(v, bool) else v
for k, v in self._connection_params.model_dump().items()
if v is not None
}
if params:
query_string = urlencode(params)
return f"{self._api_endpoint_base_url}?{query_string}"
return self._api_endpoint_base_url
async def _connect(self):
"""Establish a connection to the AssemblyAI real-time transcription service.
This method sets up the necessary callback functions and initializes the
AssemblyAI transcriber.
"""
if self._transcriber:
return
def on_open(session_opened: aai.RealtimeSessionOpened):
"""Callback for when the connection to AssemblyAI is opened."""
logger.info(f"{self}: Connected to AssemblyAI")
def on_data(transcript: aai.RealtimeTranscript):
"""Callback for handling incoming transcription data.
This function runs in a separate thread from the main asyncio event loop.
It creates appropriate transcription frames and schedules them to be
pushed to the next stage of the pipeline in the main event loop.
"""
if not transcript.text:
return
timestamp = time_now_iso8601()
is_final = isinstance(transcript, aai.RealtimeFinalTranscript)
language = self._settings["language"]
if is_final:
frame = TranscriptionFrame(transcript.text, "", timestamp, language)
else:
frame = InterimTranscriptionFrame(transcript.text, "", timestamp, language)
asyncio.run_coroutine_threadsafe(
self._handle_transcription(transcript.text, is_final, language),
self.get_event_loop(),
try:
ws_url = self._build_ws_url()
headers = {
"Authorization": self._api_key,
"User-Agent": f"AssemblyAI/1.0 (integration=Pipecat/{pipecat_version})",
}
self._websocket = await websockets.connect(
ws_url,
extra_headers=headers,
)
# Schedule the coroutine to run in the main event loop
# This is necessary because this callback runs in a different thread
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
def on_error(error: aai.RealtimeError):
"""Callback for handling errors from AssemblyAI.
Like on_data, this runs in a separate thread and schedules error
handling in the main event loop.
"""
logger.error(f"{self}: An error occurred: {error}")
# Schedule the coroutine to run in the main event loop
asyncio.run_coroutine_threadsafe(
self.push_frame(ErrorFrame(str(error))), self.get_event_loop()
)
def on_close():
"""Callback for when the connection to AssemblyAI is closed."""
logger.info(f"{self}: Disconnected from AssemblyAI")
self._transcriber = aai.RealtimeTranscriber(
sample_rate=self.sample_rate,
encoding=self._settings["encoding"],
on_data=on_data,
on_error=on_error,
on_open=on_open,
on_close=on_close,
)
self._transcriber.connect()
self._connected = True
self._receive_task = self.create_task(self._receive_task_handler())
except Exception as e:
logger.error(f"Failed to connect to AssemblyAI: {e}")
self._connected = False
raise
async def _disconnect(self):
"""Disconnect from the AssemblyAI service and clean up resources."""
if self._transcriber:
self._transcriber.close()
self._transcriber = None
"""Disconnect from AssemblyAI WebSocket and wait for termination message."""
if not self._connected or not self._websocket:
return
try:
self._termination_event.clear()
self._received_termination = False
if len(self._audio_buffer) > 0:
await self._websocket.send(bytes(self._audio_buffer))
self._audio_buffer.clear()
try:
await self._websocket.send(json.dumps({"type": "Terminate"}))
try:
await asyncio.wait_for(
self._termination_event.wait(),
timeout=5.0,
)
except asyncio.TimeoutError:
logger.warning("Timed out waiting for termination message from server")
except Exception as e:
logger.warning(f"Error during termination handshake: {e}")
if self._receive_task:
await self.cancel_task(self._receive_task)
await self._websocket.close()
except Exception as e:
logger.error(f"Error during disconnect: {e}")
finally:
self._websocket = None
self._connected = False
self._receive_task = None
async def _receive_task_handler(self):
"""Handle incoming WebSocket messages."""
try:
while self._connected:
try:
message = await self._websocket.recv()
data = json.loads(message)
await self._handle_message(data)
except websockets.exceptions.ConnectionClosedOK:
break
except Exception as e:
logger.error(f"Error processing WebSocket message: {e}")
break
except Exception as e:
logger.error(f"Fatal error in receive handler: {e}")
def _parse_message(self, message: Dict[str, Any]) -> BaseMessage:
"""Parse a raw message into the appropriate message type."""
msg_type = message.get("type")
if msg_type == "Begin":
return BeginMessage.model_validate(message)
elif msg_type == "Turn":
return TurnMessage.model_validate(message)
elif msg_type == "Termination":
return TerminationMessage.model_validate(message)
else:
raise ValueError(f"Unknown message type: {msg_type}")
async def _handle_message(self, message: Dict[str, Any]):
"""Handle AssemblyAI WebSocket messages."""
try:
parsed_message = self._parse_message(message)
if isinstance(parsed_message, BeginMessage):
logger.debug(
f"Session Begin: {parsed_message.id} (expires at {parsed_message.expires_at})"
)
elif isinstance(parsed_message, TurnMessage):
await self._handle_transcription(parsed_message)
elif isinstance(parsed_message, TerminationMessage):
await self._handle_termination(parsed_message)
except Exception as e:
logger.error(f"Error handling message: {e}")
async def _handle_termination(self, message: TerminationMessage):
"""Handle termination message."""
self._received_termination = True
self._termination_event.set()
logger.info(
f"Session Terminated: Audio Duration={message.audio_duration_seconds}s, "
f"Session Duration={message.session_duration_seconds}s"
)
await self.push_frame(EndFrame())
async def _handle_transcription(self, message: TurnMessage):
"""Handle transcription results."""
if not message.transcript:
return
await self.stop_ttfb_metrics()
if message.end_of_turn and (
not self._connection_params.formatted_finals or message.turn_is_formatted
):
await self.push_frame(
TranscriptionFrame(
message.transcript,
"", # participant
time_now_iso8601(),
self._language,
message,
)
)
await self._trace_transcription(message.transcript, True, self._language)
await self.stop_processing_metrics()
else:
await self.push_frame(
InterimTranscriptionFrame(
message.transcript,
"", # participant
time_now_iso8601(),
self._language,
message,
)
)

66
src/pipecat/services/aws/llm.py
View File

@@ -21,6 +21,7 @@ from pipecat.adapters.services.bedrock_adapter import AWSBedrockLLMAdapter
from pipecat.frames.frames import (
Frame,
FunctionCallCancelFrame,
FunctionCallFromLLM,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
LLMFullResponseEndFrame,
@@ -606,6 +607,21 @@ class AWSBedrockLLMService(LLMService):
assistant = AWSBedrockAssistantContextAggregator(context, params=assistant_params)
return AWSBedrockContextAggregatorPair(_user=user, _assistant=assistant)
def _create_no_op_tool(self):
"""Create a no-operation tool for AWS Bedrock when tool content exists but no tools are defined.
This is required because AWS Bedrock doesn't allow empty tool configurations after tools were
previously set. Other LLM vendors allow NOT_GIVEN or empty tool configurations,
but AWS Bedrock requires at least one tool to be defined.
"""
return {
"toolSpec": {
"name": "no_operation",
"description": "Internal placeholder function. Do not call this function.",
"inputSchema": {"json": {"type": "object", "properties": {}, "required": []}},
}
}
@traced_llm
async def _process_context(self, context: AWSBedrockLLMContext):
# Usage tracking
@@ -616,6 +632,8 @@ class AWSBedrockLLMService(LLMService):
cache_creation_input_tokens = 0
use_completion_tokens_estimate = False
using_noop_tool = False
try:
await self.push_frame(LLMFullResponseStartFrame())
await self.start_processing_metrics()
@@ -640,12 +658,28 @@ class AWSBedrockLLMService(LLMService):
# Add system message
request_params["system"] = context.system
# Add tools if present
if context.tools:
tool_config = {"tools": context.tools}
# Check if messages contain tool use or tool result content blocks
has_tool_content = False
for message in context.messages:
if isinstance(message.get("content"), list):
for content_item in message["content"]:
if "toolUse" in content_item or "toolResult" in content_item:
has_tool_content = True
break
if has_tool_content:
break
# Add tool_choice if specified
if context.tool_choice:
# Handle tools: use current tools, or no-op if tool content exists but no current tools
tools = context.tools or []
if has_tool_content and not tools:
tools = [self._create_no_op_tool()]
using_noop_tool = True
if tools:
tool_config = {"tools": tools}
# Only add tool_choice if we have real tools (not just no-op)
if not using_noop_tool and context.tool_choice:
if context.tool_choice == "auto":
tool_config["toolChoice"] = {"auto": {}}
elif context.tool_choice == "none":
@@ -675,6 +709,7 @@ class AWSBedrockLLMService(LLMService):
tool_use_block = None
json_accumulator = ""
function_calls = []
for event in response["stream"]:
# Handle text content
if "contentBlockDelta" in event:
@@ -704,12 +739,19 @@ class AWSBedrockLLMService(LLMService):
if event["messageStop"]["stopReason"] == "tool_use" and tool_use_block:
try:
arguments = json.loads(json_accumulator) if json_accumulator else {}
await self.call_function(
context=context,
tool_call_id=tool_use_block["id"],
function_name=tool_use_block["name"],
arguments=arguments,
)
# Only call function if it's not the no_operation tool
if not using_noop_tool:
function_calls.append(
FunctionCallFromLLM(
context=context,
tool_call_id=tool_use_block["id"],
function_name=tool_use_block["name"],
arguments=arguments,
)
)
else:
logger.debug("Ignoring no_operation tool call")
except json.JSONDecodeError:
logger.error(f"Failed to parse tool arguments: {json_accumulator}")
@@ -720,7 +762,7 @@ class AWSBedrockLLMService(LLMService):
completion_tokens += usage.get("outputTokens", 0)
cache_read_input_tokens += usage.get("cacheReadInputTokens", 0)
cache_creation_input_tokens += usage.get("cacheWriteInputTokens", 0)
await self.run_function_calls(function_calls)
except asyncio.CancelledError:
# If we're interrupted, we won't get a complete usage report. So set our flag to use the
# token estimate. The reraise the exception so all the processors running in this task

2
src/pipecat/services/aws/stt.py
View File

@@ -305,6 +305,7 @@ class AWSTranscribeSTTService(STTService):
"",
time_now_iso8601(),
self._settings["language"],
result=result,
)
)
await self._handle_transcription(
@@ -320,6 +321,7 @@ class AWSTranscribeSTTService(STTService):
"",
time_now_iso8601(),
self._settings["language"],
result=result,
)
)
elif headers.get(":message-type") == "exception":

3
src/pipecat/services/aws/tts.py
View File

@@ -253,7 +253,8 @@ class AWSPollyTTSService(TTSService):
yield TTSStartedFrame()
CHUNK_SIZE = 1024
CHUNK_SIZE = self.chunk_size
for i in range(0, len(audio_data), CHUNK_SIZE):
chunk = audio_data[i : i + CHUNK_SIZE]
if len(chunk) > 0:

8
src/pipecat/services/azure/stt.py
View File

@@ -121,7 +121,13 @@ class AzureSTTService(STTService):
def _on_handle_recognized(self, event):
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
language = getattr(event.result, "language", None) or self._settings.get("language")
frame = TranscriptionFrame(event.result.text, "", time_now_iso8601(), language)
frame = TranscriptionFrame(
event.result.text,
"",
time_now_iso8601(),
language,
result=event,
)
asyncio.run_coroutine_threadsafe(
self._handle_transcription(event.result.text, True, language), self.get_event_loop()
)

3
src/pipecat/services/cartesia/__init__.py
View File

@@ -8,6 +8,7 @@ import sys
from pipecat.services import DeprecatedModuleProxy
from .stt import *
from .tts import *
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "cartesia", "cartesia.tts")
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "cartesia", "cartesia.[stt,tts]")

239
src/pipecat/services/cartesia/stt.py Normal file
View File

@@ -0,0 +1,239 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import json
import urllib.parse
from typing import AsyncGenerator, Optional
import websockets
from loguru import logger
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
class CartesiaLiveOptions:
def __init__(
self,
*,
model: str = "ink-whisper",
language: str = Language.EN.value,
encoding: str = "pcm_s16le",
sample_rate: int = 16000,
**kwargs,
):
self.model = model
self.language = language
self.encoding = encoding
self.sample_rate = sample_rate
self.additional_params = kwargs
def to_dict(self):
params = {
"model": self.model,
"language": self.language if isinstance(self.language, str) else self.language.value,
"encoding": self.encoding,
"sample_rate": str(self.sample_rate),
}
return params
def items(self):
return self.to_dict().items()
def get(self, key, default=None):
if hasattr(self, key):
return getattr(self, key)
return self.additional_params.get(key, default)
@classmethod
def from_json(cls, json_str: str) -> "CartesiaLiveOptions":
return cls(**json.loads(json_str))
class CartesiaSTTService(STTService):
def __init__(
self,
*,
api_key: str,
base_url: str = "",
sample_rate: int = 16000,
live_options: Optional[CartesiaLiveOptions] = None,
**kwargs,
):
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, **kwargs)
default_options = CartesiaLiveOptions(
model="ink-whisper",
language=Language.EN.value,
encoding="pcm_s16le",
sample_rate=sample_rate,
)
merged_options = default_options
if live_options:
merged_options_dict = default_options.to_dict()
merged_options_dict.update(live_options.to_dict())
merged_options = CartesiaLiveOptions(
**{
k: v
for k, v in merged_options_dict.items()
if not isinstance(v, str) or v != "None"
}
)
self._settings = merged_options
self.set_model_name(merged_options.model)
self._api_key = api_key
self._base_url = base_url or "api.cartesia.ai"
self._connection = None
self._receiver_task = None
def can_generate_metrics(self) -> bool:
return True
async def start(self, frame: StartFrame):
await super().start(frame)
await self._connect()
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
# If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again
if not self._connection or self._connection.closed:
await self._connect()
await self._connection.send(audio)
yield None
async def _connect(self):
params = self._settings.to_dict()
ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}"
logger.debug(f"Connecting to Cartesia: {ws_url}")
headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key}
try:
self._connection = await websockets.connect(ws_url, extra_headers=headers)
# Setup the receiver task to handle the incoming messages from the Cartesia server
if self._receiver_task is None or self._receiver_task.done():
self._receiver_task = asyncio.create_task(self._receive_messages())
logger.debug(f"Connected to Cartesia")
except Exception as e:
logger.error(f"{self}: unable to connect to Cartesia: {e}")
async def _receive_messages(self):
try:
while True:
if not self._connection or self._connection.closed:
break
message = await self._connection.recv()
try:
data = json.loads(message)
await self._process_response(data)
except json.JSONDecodeError:
logger.warning(f"Received non-JSON message: {message}")
except asyncio.CancelledError:
pass
except websockets.exceptions.ConnectionClosed as e:
logger.debug(f"WebSocket connection closed: {e}")
except Exception as e:
logger.error(f"Error in message receiver: {e}")
async def _process_response(self, data):
if "type" in data:
if data["type"] == "transcript":
await self._on_transcript(data)
elif data["type"] == "error":
logger.error(f"Cartesia error: {data.get('message', 'Unknown error')}")
@traced_stt
async def _handle_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
async def _on_transcript(self, data):
if "text" not in data:
return
transcript = data.get("text", "")
is_final = data.get("is_final", False)
language = None
if "language" in data:
try:
language = Language(data["language"])
except (ValueError, KeyError):
pass
if len(transcript) > 0:
await self.stop_ttfb_metrics()
if is_final:
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), language)
)
await self._handle_transcription(transcript, is_final, language)
await self.stop_processing_metrics()
else:
# For interim transcriptions, just push the frame without tracing
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)
)
async def _disconnect(self):
if self._receiver_task:
self._receiver_task.cancel()
try:
await self._receiver_task
except asyncio.CancelledError:
pass
except Exception as e:
logger.exception(f"Unexpected exception while cancelling task: {e}")
self._receiver_task = None
if self._connection and self._connection.open:
logger.debug("Disconnecting from Cartesia")
await self._connection.close()
self._connection = None
async def start_metrics(self):
await self.start_ttfb_metrics()
await self.start_processing_metrics()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame):
await self.start_metrics()
elif isinstance(frame, UserStoppedSpeakingFrame):
# Send finalize command to flush the transcription session
if self._connection and self._connection.open:
await self._connection.send("finalize")

17
src/pipecat/services/deepgram/stt.py
View File

@@ -90,6 +90,7 @@ class DeepgramSTTService(STTService):
if "language" in merged_options and isinstance(merged_options["language"], Language):
merged_options["language"] = merged_options["language"].value
self.set_model_name(merged_options["model"])
self._settings = merged_options
self._addons = addons
@@ -211,14 +212,26 @@ class DeepgramSTTService(STTService):
await self.stop_ttfb_metrics()
if is_final:
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), language)
TranscriptionFrame(
transcript,
"",
time_now_iso8601(),
language,
result=result,
)
)
await self._handle_transcription(transcript, is_final, language)
await self.stop_processing_metrics()
else:
# For interim transcriptions, just push the frame without tracing
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)
InterimTranscriptionFrame(
transcript,
"",
time_now_iso8601(),
language,
result=result,
)
)
async def process_frame(self, frame: Frame, direction: FrameDirection):

31
src/pipecat/services/elevenlabs/tts.py
View File

@@ -279,9 +279,12 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await self._disconnect()
async def flush_audio(self):
if self._websocket and self._context_id:
msg = {"context_id": self._context_id, "flush": True}
await self._websocket.send(json.dumps(msg))
if not self._context_id or not self._websocket:
return
logger.trace(f"{self}: flushing audio")
msg = {"context_id": self._context_id, "flush": True}
await self._websocket.send(json.dumps(msg))
self._context_id = None
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
await super().push_frame(frame, direction)
@@ -389,14 +392,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
async for message in self._get_websocket():
msg = json.loads(message)
# Check if this message belongs to the current context
# The default context may return null/None for context_id
received_ctx_id = msg.get("contextId")
if (
self._context_id is not None
and received_ctx_id is not None
and received_ctx_id != self._context_id
):
logger.trace(f"Ignoring message from different context: {received_ctx_id}")
if not self.audio_context_available(received_ctx_id):
logger.trace(f"Ignoring message from unavailable context: {received_ctx_id}")
continue
if msg.get("audio"):
@@ -405,14 +403,15 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
audio = base64.b64decode(msg["audio"])
frame = TTSAudioRawFrame(audio, self.sample_rate, 1)
await self.push_frame(frame)
await self.append_to_audio_context(received_ctx_id, frame)
if msg.get("alignment"):
word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
await self.add_word_timestamps(word_times)
self._cumulative_time = word_times[-1][1]
if msg.get("isFinal"):
logger.trace(f"Received final message for context {received_ctx_id}")
# Context has finished
await self.remove_audio_context(received_ctx_id)
# Reset context tracking if this was our active context
if self._context_id == received_ctx_id:
self._context_id = None
self._started = False
@@ -464,6 +463,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await self._websocket.send(
json.dumps({"context_id": self._context_id, "close_context": True})
)
await self.remove_audio_context(self._context_id)
self._context_id = None
if not self._started:
@@ -471,6 +471,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
yield TTSStartedFrame()
self._started = True
self._cumulative_time = 0
# Create new context ID and register it
self._context_id = str(uuid.uuid4())
await self.create_audio_context(self._context_id)
await self._send_text(text)
await self.start_tts_usage_metrics(text)
@@ -478,7 +481,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
logger.error(f"{self} error sending message: {e}")
yield TTSStoppedFrame()
self._started = False
self._context_id = None
if self._context_id:
await self.remove_audio_context(self._context_id)
self._context_id = None
return
yield None
except Exception as e:

6
src/pipecat/services/fal/stt.py
View File

@@ -252,7 +252,11 @@ class FalSTTService(SegmentedSTTService):
await self._handle_transcription(text, True, self._settings["language"])
logger.debug(f"Transcription: [{text}]")
yield TranscriptionFrame(
text, "", time_now_iso8601(), Language(self._settings["language"])
text,
"",
time_now_iso8601(),
Language(self._settings["language"]),
result=response,
)
except Exception as e:

73
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -52,7 +52,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
@@ -60,6 +60,7 @@ from pipecat.services.openai.llm import (
from pipecat.transcriptions.language import Language
from pipecat.utils.string import match_endofsentence
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt, traced_tts
from . import events
@@ -335,7 +336,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
*,
api_key: str,
base_url: str = "generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent",
model="models/gemini-2.5-flash-preview-native-audio-dialog",
model="models/gemini-2.0-flash-live-001",
voice_id: str = "Charon",
start_audio_paused: bool = False,
start_video_paused: bool = False,
@@ -378,6 +379,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._last_transcription_sent = ""
self._bot_audio_buffer = bytearray()
self._bot_text_buffer = ""
self._llm_output_buffer = ""
self._sample_rate = 24000
@@ -471,6 +473,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
async def _handle_user_stopped_speaking(self, frame):
self._user_is_speaking = False
self._user_audio_buffer = bytearray()
await self.start_ttfb_metrics()
if self._needs_turn_complete_message:
self._needs_turn_complete_message = False
evt = events.ClientContentMessage.model_validate(
@@ -752,6 +755,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
logger.debug(f"Creating initial response: {messages}")
await self.start_ttfb_metrics()
evt = events.ClientContentMessage.model_validate(
{
"clientContent": {
@@ -793,6 +798,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
return
logger.debug(f"Creating response: {messages}")
await self.start_ttfb_metrics()
evt = events.ClientContentMessage.model_validate(
{
"clientContent": {
@@ -803,6 +810,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
)
await self.send_client_event(evt)
@traced_gemini_live(operation="llm_tool_result")
async def _tool_result(self, tool_result_message):
# For now we're shoving the name into the tool_call_id field, so this
# will work until we revisit that.
@@ -827,6 +835,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._websocket.send(response_message)
# await self._websocket.send(json.dumps({"clientContent": {"turnComplete": True}}))
@traced_gemini_live(operation="llm_setup")
async def _handle_evt_setup_complete(self, evt):
# If this is our first context frame, run the LLM
self._api_session_ready = True
@@ -840,6 +849,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
if not part:
return
await self.stop_ttfb_metrics()
# part.text is added when `modalities` is set to TEXT; otherwise, it's None
text = part.text
if text:
@@ -873,26 +884,48 @@ class GeminiMultimodalLiveLLMService(LLMService):
)
await self.push_frame(frame)
@traced_gemini_live(operation="llm_tool_call")
async def _handle_evt_tool_call(self, evt):
function_calls = evt.toolCall.functionCalls
if not function_calls:
return
if not self._context:
logger.error("Function calls are not supported without a context object.")
for call in function_calls:
await self.call_function(
context=self._context,
tool_call_id=call.id,
function_name=call.name,
arguments=call.args,
)
function_calls_llm = [
FunctionCallFromLLM(
context=self._context,
tool_call_id=f.id,
function_name=f.name,
arguments=f.args,
)
for f in function_calls
]
await self.run_function_calls(function_calls_llm)
@traced_gemini_live(operation="llm_response")
async def _handle_evt_turn_complete(self, evt):
self._bot_is_speaking = False
text = self._bot_text_buffer
self._bot_text_buffer = ""
# Only push the TTSStoppedFrame the bot is outputting audio
# Determine output and modality for tracing
if text:
# TEXT modality
output_text = text
output_modality = "TEXT"
else:
# AUDIO modality
output_text = self._llm_output_buffer
output_modality = "AUDIO"
# Trace the complete LLM response (this will be handled by the decorator)
# The decorator will extract the output text and usage metadata from the event
self._bot_text_buffer = ""
self._llm_output_buffer = ""
# Only push the TTSStoppedFrame if the bot is outputting audio
# when text is found, modalities is set to TEXT and no audio
# is produced.
if not text:
@@ -900,6 +933,13 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self.push_frame(LLMFullResponseEndFrame())
@traced_stt
async def _handle_user_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
async def _handle_evt_input_transcription(self, evt):
"""Handle the input transcription event.
@@ -935,9 +975,15 @@ class GeminiMultimodalLiveLLMService(LLMService):
# Send a TranscriptionFrame with the complete sentence
logger.debug(f"[Transcription:user] [{complete_sentence}]")
await self._handle_user_transcription(
complete_sentence, True, self._settings["language"]
)
await self.push_frame(
TranscriptionFrame(
text=complete_sentence, user_id="", timestamp=time_now_iso8601()
text=complete_sentence,
user_id="",
timestamp=time_now_iso8601(),
result=evt,
),
FrameDirection.UPSTREAM,
)
@@ -954,6 +1000,9 @@ class GeminiMultimodalLiveLLMService(LLMService):
if not text:
return
# Collect text for tracing
self._llm_output_buffer += text
await self.push_frame(LLMTextFrame(text=text))
await self.push_frame(TTSTextFrame(text=text))

2
src/pipecat/services/gladia/config.py
View File

@@ -74,11 +74,13 @@ class TranslationConfig(BaseModel):
target_languages: List of target language codes for translation
model: Translation model to use ("base" or "enhanced")
match_original_utterances: Whether to align translations with original utterances
informal: Force informal language forms when available
"""
target_languages: Optional[List[str]] = None
model: Optional[str] = None
match_original_utterances: Optional[bool] = None
informal: Optional[bool] = None
class RealtimeProcessingConfig(BaseModel):

14
src/pipecat/services/gladia/stt.py
View File

@@ -408,7 +408,13 @@ class GladiaSTTService(STTService):
if confidence >= self._confidence:
if is_final:
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), language)
TranscriptionFrame(
transcript,
"",
time_now_iso8601(),
language,
result=content,
)
)
await self._handle_transcription(
transcript=transcript,
@@ -418,7 +424,11 @@ class GladiaSTTService(STTService):
else:
await self.push_frame(
InterimTranscriptionFrame(
transcript, "", time_now_iso8601(), language
transcript,
"",
time_now_iso8601(),
language,
result=content,
)
)
elif content["type"] == "translation":

19
src/pipecat/services/google/llm.py
View File

@@ -42,7 +42,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.google.frames import LLMSearchResponseFrame
from pipecat.services.llm_service import LLMService
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
@@ -83,7 +83,7 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator):
await self.push_frame(frame)
# Reset our accumulator state.
self.reset()
await self.reset()
class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
@@ -557,6 +557,7 @@ class GoogleLLMService(LLMService):
)
await self.stop_ttfb_metrics()
function_calls = []
async for chunk in response:
if chunk.usage_metadata:
prompt_tokens += chunk.usage_metadata.prompt_token_count or 0
@@ -576,11 +577,13 @@ class GoogleLLMService(LLMService):
function_call = part.function_call
id = function_call.id or str(uuid.uuid4())
logger.debug(f"Function call: {function_call.name}:{id}")
await self.call_function(
context=context,
tool_call_id=id,
function_name=function_call.name,
arguments=function_call.args or {},
function_calls.append(
FunctionCallFromLLM(
context=context,
tool_call_id=id,
function_name=function_call.name,
arguments=function_call.args or {},
)
)
if (
@@ -621,6 +624,8 @@ class GoogleLLMService(LLMService):
"rendered_content": rendered_content,
"origins": origins,
}
await self.run_function_calls(function_calls)
except DeadlineExceeded:
await self._call_event_handler("on_completion_timeout")
except Exception as e:

28
src/pipecat/services/google/llm_openai.py
View File

@@ -10,6 +10,8 @@ import os
from openai import AsyncStream
from openai.types.chat import ChatCompletionChunk
from pipecat.services.llm_service import FunctionCallFromLLM
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
@@ -18,7 +20,6 @@ from loguru import logger
from pipecat.frames.frames import LLMTextFrame
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.openai.base_llm import OpenAIUnhandledFunctionException
from pipecat.services.openai.llm import OpenAILLMService
@@ -112,25 +113,26 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService):
logger.debug(
f"Function list: {functions_list}, Arguments list: {arguments_list}, Tool ID list: {tool_id_list}"
)
for index, (function_name, arguments, tool_id) in enumerate(
zip(functions_list, arguments_list, tool_id_list), start=1
function_calls = []
for function_name, arguments, tool_id in zip(
functions_list, arguments_list, tool_id_list
):
if function_name == "":
# TODO: Remove the _process_context method once Google resolves the bug
# where the index is incorrectly set to None instead of returning the actual index,
# which currently results in an empty function name('').
continue
if self.has_function(function_name):
run_llm = False
arguments = json.loads(arguments)
await self.call_function(
arguments = json.loads(arguments)
function_calls.append(
FunctionCallFromLLM(
context=context,
tool_call_id=tool_id,
function_name=function_name,
arguments=arguments,
tool_call_id=tool_id,
run_llm=run_llm,
)
else:
raise OpenAIUnhandledFunctionException(
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
)
)
await self.run_function_calls(function_calls)

14
src/pipecat/services/google/stt.py
View File

@@ -816,7 +816,13 @@ class GoogleSTTService(STTService):
if result.is_final:
self._last_transcript_was_final = True
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), primary_language)
TranscriptionFrame(
transcript,
"",
time_now_iso8601(),
primary_language,
result=result,
)
)
await self.stop_processing_metrics()
await self._handle_transcription(
@@ -829,7 +835,11 @@ class GoogleSTTService(STTService):
await self.stop_ttfb_metrics()
await self.push_frame(
InterimTranscriptionFrame(
transcript, "", time_now_iso8601(), primary_language
transcript,
"",
time_now_iso8601(),
primary_language,
result=result,
)
)

7
src/pipecat/services/google/tts.py
View File

@@ -362,8 +362,8 @@ class GoogleHttpTTSService(TTSService):
# Skip the first 44 bytes to remove the WAV header
audio_content = response.audio_content[44:]
# Read and yield audio data in chunks
CHUNK_SIZE = 1024
CHUNK_SIZE = self.chunk_size
for i in range(0, len(audio_content), CHUNK_SIZE):
chunk = audio_content[i : i + CHUNK_SIZE]
if not chunk:
@@ -505,9 +505,10 @@ class GoogleTTSService(TTSService):
yield TTSStartedFrame()
audio_buffer = b""
CHUNK_SIZE = 1024
first_chunk_for_ttfb = False
CHUNK_SIZE = self.chunk_size
async for response in streaming_responses:
chunk = response.audio_content
if not chunk:

279
src/pipecat/services/llm_service.py
View File

@@ -7,18 +7,23 @@
import asyncio
import inspect
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Mapping, Optional, Protocol, Set, Tuple, Type
from typing import Any, Awaitable, Callable, Dict, Mapping, Optional, Protocol, Sequence, Type
from loguru import logger
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
FunctionCallCancelFrame,
FunctionCallFromLLM,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
FunctionCallResultProperties,
FunctionCallsStartedFrame,
StartFrame,
StartInterruptionFrame,
UserImageRequestFrame,
)
@@ -41,22 +46,6 @@ class FunctionCallResultCallback(Protocol):
) -> None: ...
@dataclass
class FunctionCallEntry:
"""Represents an internal entry for a function call.
Attributes:
function_name (Optional[str]): The name of the function.
handler (FunctionCallHandler): The handler for processing function call parameters.
cancel_on_interruption (bool): Flag indicating whether to cancel the call on interruption.
"""
function_name: Optional[str]
handler: FunctionCallHandler
cancel_on_interruption: bool
@dataclass
class FunctionCallParams:
"""Parameters for a function call.
@@ -79,20 +68,78 @@ class FunctionCallParams:
result_callback: FunctionCallResultCallback
@dataclass
class FunctionCallRegistryItem:
"""Represents an entry in our function call registry. This is what the user
registers.
Attributes:
function_name (Optional[str]): The name of the function.
handler (FunctionCallHandler): The handler for processing function call parameters.
cancel_on_interruption (bool): Flag indicating whether to cancel the call on interruption.
"""
function_name: Optional[str]
handler: FunctionCallHandler
cancel_on_interruption: bool
@dataclass
class FunctionCallRunnerItem:
"""Represents an internal function call entry to our function call
runner. The runner executes function calls in order.
Attributes:
registry_name (Optional[str]): The function call name registration (could be None).
function_name (str): The name of the function.
tool_call_id (str): A unique identifier for the function call.
arguments (Mapping[str, Any]): The arguments for the function.
context (OpenAILLMContext): The LLM context.
"""
registry_item: FunctionCallRegistryItem
function_name: str
tool_call_id: str
arguments: Mapping[str, Any]
context: OpenAILLMContext
run_llm: Optional[bool] = None
class LLMService(AIService):
"""This class is a no-op but serves as a base class for LLM services."""
"""This is the base class for all LLM services. It handles function calling
registration and execution. The class also provides event handlers.
An event to know when an LLM service completion timeout occurs:
@task.event_handler("on_completion_timeout")
async def on_completion_timeout(service):
...
And an event to know that function calls have been received from the LLM
service and that we are going to start executing them:
@task.event_handler("on_function_calls_started")
async def on_function_calls_started(service, function_calls: Sequence[FunctionCallFromLLM]):
...
"""
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
# However, subclasses should override this with a more specific adapter when necessary.
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
def __init__(self, **kwargs):
def __init__(self, run_in_parallel: bool = True, **kwargs):
super().__init__(**kwargs)
self._functions = {}
self._run_in_parallel = run_in_parallel
self._start_callbacks = {}
self._adapter = self.adapter_class()
self._function_call_tasks: Set[Tuple[asyncio.Task, str, str]] = set()
self._functions: Dict[Optional[str], FunctionCallRegistryItem] = {}
self._function_call_tasks: Dict[asyncio.Task, FunctionCallRunnerItem] = {}
self._sequential_runner_task: Optional[asyncio.Task] = None
self._register_event_handler("on_function_calls_started")
self._register_event_handler("on_completion_timeout")
def get_llm_adapter(self) -> BaseLLMAdapter:
@@ -107,13 +154,28 @@ class LLMService(AIService):
) -> Any:
pass
async def start(self, frame: StartFrame):
await super().start(frame)
if not self._run_in_parallel:
await self._create_sequential_runner_task()
async def stop(self, frame: EndFrame):
await super().stop(frame)
if not self._run_in_parallel:
await self._cancel_sequential_runner_task()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
if not self._run_in_parallel:
await self._cancel_sequential_runner_task()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame):
await self._handle_interruptions(frame)
async def _handle_interruptions(self, frame: StartInterruptionFrame):
async def _handle_interruptions(self, _: StartInterruptionFrame):
for function_name, entry in self._functions.items():
if entry.cancel_on_interruption:
await self._cancel_function_call(function_name)
@@ -124,11 +186,11 @@ class LLMService(AIService):
handler: Any,
start_callback=None,
*,
cancel_on_interruption: bool = False,
cancel_on_interruption: bool = True,
):
# Registering a function with the function_name set to None will run
# that handler for all functions
self._functions[function_name] = FunctionCallEntry(
self._functions[function_name] = FunctionCallRegistryItem(
function_name=function_name,
handler=handler,
cancel_on_interruption=cancel_on_interruption,
@@ -157,25 +219,43 @@ class LLMService(AIService):
return True
return function_name in self._functions.keys()
async def call_function(
self,
*,
context: OpenAILLMContext,
tool_call_id: str,
function_name: str,
arguments: Mapping[str, Any],
run_llm: bool = True,
):
if not function_name in self._functions.keys() and not None in self._functions.keys():
async def run_function_calls(self, function_calls: Sequence[FunctionCallFromLLM]):
if len(function_calls) == 0:
return
task = self.create_task(
self._run_function_call(context, tool_call_id, function_name, arguments, run_llm)
)
await self._call_event_handler("on_function_calls_started", function_calls)
self._function_call_tasks.add((task, tool_call_id, function_name))
# Push frame both downstream and upstream
started_frame_downstream = FunctionCallsStartedFrame(function_calls=function_calls)
started_frame_upstream = FunctionCallsStartedFrame(function_calls=function_calls)
await self.push_frame(started_frame_downstream, FrameDirection.DOWNSTREAM)
await self.push_frame(started_frame_upstream, FrameDirection.UPSTREAM)
task.add_done_callback(self._function_call_task_finished)
for function_call in function_calls:
if function_call.function_name in self._functions.keys():
item = self._functions[function_call.function_name]
elif None in self._functions.keys():
item = self._functions[None]
else:
logger.warning(
f"{self} is calling '{function_call.function_name}', but it's not registered."
)
continue
runner_item = FunctionCallRunnerItem(
registry_item=item,
function_name=function_call.function_name,
tool_call_id=function_call.tool_call_id,
arguments=function_call.arguments,
context=function_call.context,
)
if self._run_in_parallel:
task = self.create_task(self._run_function_call(runner_item))
self._function_call_tasks[task] = runner_item
task.add_done_callback(self._function_call_task_finished)
else:
await self._sequential_runner_queue.put(runner_item)
async def call_start_function(self, context: OpenAILLMContext, function_name: str):
if function_name in self._start_callbacks.keys():
@@ -203,43 +283,57 @@ class LLMService(AIService):
FrameDirection.UPSTREAM,
)
async def _run_function_call(
self,
context: OpenAILLMContext,
tool_call_id: str,
function_name: str,
arguments: Mapping[str, Any],
run_llm: bool = True,
):
if function_name in self._functions.keys():
entry = self._functions[function_name]
async def _create_sequential_runner_task(self):
if not self._sequential_runner_task:
self._sequential_runner_queue = asyncio.Queue()
self._sequential_runner_task = self.create_task(self._sequential_runner_handler())
async def _cancel_sequential_runner_task(self):
if self._sequential_runner_task:
await self.cancel_task(self._sequential_runner_task)
self._sequential_runner_task = None
async def _sequential_runner_handler(self):
while True:
runner_item = await self._sequential_runner_queue.get()
task = self.create_task(self._run_function_call(runner_item))
self._function_call_tasks[task] = runner_item
# Since we run tasks sequentially we don't need to call
# task.add_done_callback(self._function_call_task_finished).
await self.wait_for_task(task)
del self._function_call_tasks[task]
async def _run_function_call(self, runner_item: FunctionCallRunnerItem):
if runner_item.function_name in self._functions.keys():
item = self._functions[runner_item.function_name]
elif None in self._functions.keys():
entry = self._functions[None]
item = self._functions[None]
else:
return
logger.debug(
f"{self} Calling function [{function_name}:{tool_call_id}] with arguments {arguments}"
f"{self} Calling function [{runner_item.function_name}:{runner_item.tool_call_id}] with arguments {runner_item.arguments}"
)
# NOTE(aleix): This needs to be removed after we remove the deprecation.
await self.call_start_function(context, function_name)
await self.call_start_function(runner_item.context, runner_item.function_name)
# Push a SystemFrame downstream. This frame will let our assistant context aggregator
# know that we are in the middle of a function call. Some contexts/aggregators may
# not need this. But some definitely do (Anthropic, for example).
# Also push a SystemFrame upstream for use by other processors, like STTMuteFilter.
# Push a function call in-progress downstream. This frame will let our
# assistant context aggregator know that we are in the middle of a
# function call. Some contexts/aggregators may not need this. But some
# definitely do (Anthropic, for example). Also push it upstream for use
# by other processors, like STTMuteFilter.
progress_frame_downstream = FunctionCallInProgressFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
cancel_on_interruption=entry.cancel_on_interruption,
function_name=runner_item.function_name,
tool_call_id=runner_item.tool_call_id,
arguments=runner_item.arguments,
cancel_on_interruption=item.cancel_on_interruption,
)
progress_frame_upstream = FunctionCallInProgressFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
cancel_on_interruption=entry.cancel_on_interruption,
function_name=runner_item.function_name,
tool_call_id=runner_item.tool_call_id,
arguments=runner_item.arguments,
cancel_on_interruption=item.cancel_on_interruption,
)
# Push frame both downstream and upstream
@@ -251,24 +345,26 @@ class LLMService(AIService):
result: Any, *, properties: Optional[FunctionCallResultProperties] = None
):
result_frame_downstream = FunctionCallResultFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
function_name=runner_item.function_name,
tool_call_id=runner_item.tool_call_id,
arguments=runner_item.arguments,
result=result,
run_llm=runner_item.run_llm,
properties=properties,
)
result_frame_upstream = FunctionCallResultFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
function_name=runner_item.function_name,
tool_call_id=runner_item.tool_call_id,
arguments=runner_item.arguments,
result=result,
run_llm=runner_item.run_llm,
properties=properties,
)
await self.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM)
await self.push_frame(result_frame_upstream, FrameDirection.UPSTREAM)
signature = inspect.signature(entry.handler)
signature = inspect.signature(item.handler)
if len(signature.parameters) > 1:
import warnings
@@ -279,24 +375,32 @@ class LLMService(AIService):
DeprecationWarning,
)
await entry.handler(
function_name, tool_call_id, arguments, self, context, function_call_result_callback
await item.handler(
runner_item.function_name,
runner_item.tool_call_id,
runner_item.arguments,
self,
runner_item.context,
function_call_result_callback,
)
else:
params = FunctionCallParams(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
function_name=runner_item.function_name,
tool_call_id=runner_item.tool_call_id,
arguments=runner_item.arguments,
llm=self,
context=context,
context=runner_item.context,
result_callback=function_call_result_callback,
)
await entry.handler(params)
await item.handler(params)
async def _cancel_function_call(self, function_name: str):
async def _cancel_function_call(self, function_name: Optional[str]):
cancelled_tasks = set()
for task, tool_call_id, name in self._function_call_tasks:
if name == function_name:
for task, runner_item in self._function_call_tasks.items():
if runner_item.registry_item.function_name == function_name:
name = runner_item.function_name
tool_call_id = runner_item.tool_call_id
# We remove the callback because we are going to cancel the task
# now, otherwise we will be removing it from the set while we
# are iterating.
@@ -306,23 +410,20 @@ class LLMService(AIService):
await self.cancel_task(task)
frame = FunctionCallCancelFrame(
function_name=function_name, tool_call_id=tool_call_id
)
frame = FunctionCallCancelFrame(function_name=name, tool_call_id=tool_call_id)
await self.push_frame(frame)
logger.debug(f"{self} Function call [{name}:{tool_call_id}] has been cancelled")
cancelled_tasks.add(task)
logger.debug(f"{self} Function call [{name}:{tool_call_id}] has been cancelled")
# Remove all cancelled tasks from our set.
for task in cancelled_tasks:
self._function_call_task_finished(task)
def _function_call_task_finished(self, task: asyncio.Task):
tuple_to_remove = next((t for t in self._function_call_tasks if t[0] == task), None)
if tuple_to_remove:
self._function_call_tasks.discard(tuple_to_remove)
if task in self._function_call_tasks:
del self._function_call_tasks[task]
# The task is finished so this should exit immediately. We need to
# do this because otherwise the task manager would report a dangling
# task if we don't remove it.

9
src/pipecat/services/minimax/tts.py
View File

@@ -227,7 +227,8 @@ class MiniMaxHttpTTSService(TTSService):
# Process the streaming response
buffer = bytearray()
CHUNK_SIZE = 1024
CHUNK_SIZE = self.chunk_size
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
if not chunk:
@@ -279,10 +280,8 @@ class MiniMaxHttpTTSService(TTSService):
await self.stop_ttfb_metrics()
yield TTSAudioRawFrame(
audio=audio_chunk,
sample_rate=self._settings["audio_setting"][
"sample_rate"
],
num_channels=self._settings["audio_setting"]["channel"],
sample_rate=self.sample_rate,
num_channels=1,
)
except ValueError as e:
logger.error(f"Error converting hex to binary: {e}")

29
src/pipecat/services/openai/base_llm.py
View File

@@ -34,14 +34,10 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.utils.tracing.service_decorators import traced_llm
class OpenAIUnhandledFunctionException(Exception):
pass
class BaseOpenAILLMService(LLMService):
"""This is the base for all services that use the AsyncOpenAI client.
@@ -260,23 +256,22 @@ class BaseOpenAILLMService(LLMService):
arguments_list.append(arguments)
tool_id_list.append(tool_call_id)
for index, (function_name, arguments, tool_id) in enumerate(
zip(functions_list, arguments_list, tool_id_list), start=1
function_calls = []
for function_name, arguments, tool_id in zip(
functions_list, arguments_list, tool_id_list
):
if self.has_function(function_name):
run_llm = False
arguments = json.loads(arguments)
await self.call_function(
arguments = json.loads(arguments)
function_calls.append(
FunctionCallFromLLM(
context=context,
tool_call_id=tool_id,
function_name=function_name,
arguments=arguments,
tool_call_id=tool_id,
run_llm=run_llm,
)
else:
raise OpenAIUnhandledFunctionException(
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
)
)
await self.run_function_calls(function_calls)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)

2
src/pipecat/services/openai/tts.py
View File

@@ -125,7 +125,7 @@ class OpenAITTSService(TTSService):
await self.start_tts_usage_metrics(text)
CHUNK_SIZE = 1024
CHUNK_SIZE = self.chunk_size
yield TTSStartedFrame()
async for chunk in r.iter_bytes(CHUNK_SIZE):

53
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -48,9 +48,11 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import LLMService
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai.llm import OpenAIContextAggregatorPair
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt, traced_tts
from . import events
from .context import (
@@ -76,10 +78,6 @@ class CurrentAudioResponse:
total_size: int = 0
class OpenAIUnhandledFunctionException(Exception):
pass
class OpenAIRealtimeBetaLLMService(LLMService):
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
adapter_class = OpenAIRealtimeLLMAdapter
@@ -100,6 +98,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self.api_key = api_key
self.base_url = full_url
self.set_model_name(model)
self._session_properties: events.SessionProperties = (
session_properties or events.SessionProperties()
@@ -402,6 +401,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
# errors are fatal, so exit the receive loop
return
@traced_openai_realtime(operation="llm_setup")
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message
# to configure the session properties.
@@ -464,17 +464,25 @@ class OpenAIRealtimeBetaLLMService(LLMService):
if self._send_transcription_frames:
await self.push_frame(
# no way to get a language code?
InterimTranscriptionFrame(evt.delta, "", time_now_iso8601())
InterimTranscriptionFrame(evt.delta, "", time_now_iso8601(), result=evt)
)
@traced_stt
async def _handle_user_transcription(
self, transcript: str, is_final: bool, language: Optional[Language] = None
):
"""Handle a transcription result with tracing."""
pass
async def handle_evt_input_audio_transcription_completed(self, evt):
await self._call_event_handler("on_conversation_item_updated", evt.item_id, None)
if self._send_transcription_frames:
await self.push_frame(
# no way to get a language code?
TranscriptionFrame(evt.transcript, "", time_now_iso8601())
TranscriptionFrame(evt.transcript, "", time_now_iso8601(), result=evt)
)
await self._handle_user_transcription(evt.transcript, True, Language.EN)
pair = self._user_and_response_message_tuple
if pair:
user, assistant = pair
@@ -493,6 +501,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
for future in futures:
future.set_result(evt.item)
@traced_openai_realtime(operation="llm_response")
async def _handle_evt_response_done(self, evt):
# todo: figure out whether there's anything we need to do for "cancelled" events
# usage metrics
@@ -574,25 +583,18 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._handle_function_call_items(function_calls)
async def _handle_function_call_items(self, items):
total_items = len(items)
for index, item in enumerate(items):
function_name = item.name
tool_id = item.call_id
arguments = json.loads(item.arguments)
if self.has_function(function_name):
run_llm = index == total_items - 1
if function_name in self._functions.keys() or None in self._functions.keys():
await self.call_function(
context=self._context,
tool_call_id=tool_id,
function_name=function_name,
arguments=arguments,
run_llm=run_llm,
)
else:
raise OpenAIUnhandledFunctionException(
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
function_calls = []
for item in items:
args = json.loads(item.arguments)
function_calls.append(
FunctionCallFromLLM(
context=self._context,
tool_call_id=item.call_id,
function_name=item.name,
arguments=args,
)
)
await self.run_function_calls(function_calls)
#
# state and client events for the current conversation
@@ -609,6 +611,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self._context.llm_needs_initial_messages = True
await self._connect()
@traced_openai_realtime(operation="llm_request")
async def _create_response(self):
if not self._api_session_ready:
self._run_llm_when_api_session_ready = True

9
src/pipecat/services/piper/tts.py
View File

@@ -74,19 +74,18 @@ class PiperTTSService(TTSService):
async with self._session.post(self._base_url, data=text, headers=headers) as response:
if response.status != 200:
eror = await response.text()
error = await response.text()
logger.error(
f"{self} error getting audio (status: {response.status}, error: {eror})"
f"{self} error getting audio (status: {response.status}, error: {error})"
)
yield ErrorFrame(
f"Error getting audio (status: {response.status}, error: {eror})"
f"Error getting audio (status: {response.status}, error: {error})"
)
return
await self.start_tts_usage_metrics(text)
# Process the streaming response
CHUNK_SIZE = 1024
CHUNK_SIZE = self.chunk_size
yield TTSStartedFrame()
async for chunk in response.content.iter_chunked(CHUNK_SIZE):

14
src/pipecat/services/rime/tts.py
View File

@@ -26,6 +26,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.transcriptions import language
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
@@ -49,6 +50,8 @@ def language_to_rime_language(language: Language) -> str:
str: Three-letter language code used by Rime (e.g., 'eng' for English).
"""
LANGUAGE_MAP = {
Language.DE: "ger",
Language.FR: "fra",
Language.EN: "eng",
Language.ES: "spa",
}
@@ -352,6 +355,7 @@ class RimeTTSService(AudioContextWordTTSService):
class RimeHttpTTSService(TTSService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
pause_between_brackets: Optional[bool] = False
phonemize_between_brackets: Optional[bool] = False
inline_speed_alpha: Optional[str] = None
@@ -377,6 +381,9 @@ class RimeHttpTTSService(TTSService):
self._session = aiohttp_session
self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = {
"lang": self.language_to_service_language(params.language)
if params.language
else "eng",
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": params.pause_between_brackets,
@@ -391,6 +398,10 @@ class RimeHttpTTSService(TTSService):
def can_generate_metrics(self) -> bool:
return True
def language_to_service_language(self, language: Language) -> str | None:
"""Convert pipecat language to Rime language code."""
return language_to_rime_language(language)
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"{self}: Generating TTS [{text}]")
@@ -430,8 +441,7 @@ class RimeHttpTTSService(TTSService):
yield TTSStartedFrame()
# Process the streaming response
CHUNK_SIZE = 1024
CHUNK_SIZE = self.chunk_size
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
if need_to_strip_wav_header and chunk.startswith(b"RIFF"):

14
src/pipecat/services/riva/stt.py
View File

@@ -256,7 +256,13 @@ class RivaSTTService(STTService):
if result.is_final:
await self.stop_processing_metrics()
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), self._language_code)
TranscriptionFrame(
transcript,
"",
time_now_iso8601(),
self._language_code,
result=result,
)
)
await self._handle_transcription(
transcript=transcript,
@@ -266,7 +272,11 @@ class RivaSTTService(STTService):
else:
await self.push_frame(
InterimTranscriptionFrame(
transcript, "", time_now_iso8601(), self._language_code
transcript,
"",
time_now_iso8601(),
self._language_code,
result=result,
)
)

13
src/pipecat/services/tts_service.py
View File

@@ -106,6 +106,19 @@ class TTSService(AIService):
def sample_rate(self) -> int:
return self._sample_rate
@property
def chunk_size(self) -> int:
"""This property indicates how much audio we download (from TTS services
that require chunking) before we start pushing the first audio
frame. This will make sure we download the rest of the audio while audio
is being played without causing audio glitches (specially at the
beginning). Of course, this will also depend on how fast the TTS service
generates bytes.
"""
CHUNK_SECONDS = 0.5
return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample
async def set_model(self, model: str):
self.set_model_name(model)

2
src/pipecat/services/xtts/tts.py
View File

@@ -152,7 +152,7 @@ class XTTSService(TTSService):
yield TTSStartedFrame()
CHUNK_SIZE = 1024
CHUNK_SIZE = self.chunk_size
buffer = bytearray()
async for chunk in r.content.iter_chunked(CHUNK_SIZE):

83
src/pipecat/transports/base_input.py
View File

@@ -17,6 +17,8 @@ from pipecat.audio.turn.base_turn_analyzer import (
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
from pipecat.frames.frames import (
BotInterruptionFrame,
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
CancelFrame,
EmulateUserStartedSpeakingFrame,
EmulateUserStoppedSpeakingFrame,
@@ -24,9 +26,11 @@ from pipecat.frames.frames import (
FilterUpdateSettingsFrame,
Frame,
InputAudioRawFrame,
InputImageRawFrame,
MetricsFrame,
StartFrame,
StartInterruptionFrame,
StopFrame,
StopInterruptionFrame,
SystemFrame,
UserStartedSpeakingFrame,
@@ -49,6 +53,9 @@ class BaseInputTransport(FrameProcessor):
# Input sample rate. It will be initialized on StartFrame.
self._sample_rate = 0
# Track bot speaking state for interruption logic
self._bot_speaking = False
# We read audio from a single queue one at a time and we then run VAD in
# a thread. Therefore, only one thread should be necessary.
self._executor = ThreadPoolExecutor(max_workers=1)
@@ -57,6 +64,11 @@ class BaseInputTransport(FrameProcessor):
# if passthrough is enabled.
self._audio_task = None
# If the transport is stopped with `StopFrame` we might still be
# receiving frames from the transport but we really don't want to push
# them downstream until we get another `StartFrame`.
self._paused = False
if self._params.vad_enabled:
import warnings
@@ -117,6 +129,8 @@ class BaseInputTransport(FrameProcessor):
return self._params.turn_analyzer
async def start(self, frame: StartFrame):
self._paused = False
self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
# Configure VAD analyzer.
@@ -133,28 +147,33 @@ class BaseInputTransport(FrameProcessor):
async def stop(self, frame: EndFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_task)
self._audio_task = None
await self._cancel_audio_task()
# Stop audio filter.
if self._params.audio_in_filter:
await self._params.audio_in_filter.stop()
async def pause(self, frame: StopFrame):
self._paused = True
# Cancel task so we clear the queue
await self._cancel_audio_task()
# Retart the task
self._create_audio_task()
async def cancel(self, frame: CancelFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_task)
self._audio_task = None
await self._cancel_audio_task()
async def set_transport_ready(self, frame: StartFrame):
"""To be called when the transport is ready to stream."""
# Create audio input queue and task if needed.
if not self._audio_task and self._params.audio_in_enabled:
self._audio_in_queue = asyncio.Queue()
self._audio_task = self.create_task(self._audio_task_handler())
self._create_audio_task()
async def push_video_frame(self, frame: InputImageRawFrame):
if self._params.video_in_enabled and not self._paused:
await self.push_frame(frame)
async def push_audio_frame(self, frame: InputAudioRawFrame):
if self._params.audio_in_enabled:
if self._params.audio_in_enabled and not self._paused:
await self._audio_in_queue.put(frame)
#
@@ -175,6 +194,12 @@ class BaseInputTransport(FrameProcessor):
await self.push_frame(frame, direction)
elif isinstance(frame, BotInterruptionFrame):
await self._handle_bot_interruption(frame)
elif isinstance(frame, BotStartedSpeakingFrame):
await self._handle_bot_started_speaking(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, BotStoppedSpeakingFrame):
await self._handle_bot_stopped_speaking(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, EmulateUserStartedSpeakingFrame):
logger.debug("Emulating user started speaking")
await self._handle_user_interruption(UserStartedSpeakingFrame(emulated=True))
@@ -190,6 +215,9 @@ class BaseInputTransport(FrameProcessor):
# finish and the task finishes when EndFrame is processed.
await self.push_frame(frame, direction)
await self.stop(frame)
elif isinstance(frame, StopFrame):
await self.push_frame(frame, direction)
await self.pause(frame)
elif isinstance(frame, VADParamsUpdateFrame):
if self.vad_analyzer:
self.vad_analyzer.set_params(frame.params)
@@ -213,13 +241,26 @@ class BaseInputTransport(FrameProcessor):
if isinstance(frame, UserStartedSpeakingFrame):
logger.debug("User started speaking")
await self.push_frame(frame)
# Only push StartInterruptionFrame if:
# 1. No interruption config is set, OR
# 2. Interruption config is set but bot is not speaking
should_push_immediate_interruption = (
not self.interruption_strategies or not self._bot_speaking
)
# Make sure we notify about interruptions quickly out-of-band.
if self.interruptions_allowed:
if should_push_immediate_interruption and self.interruptions_allowed:
await self._start_interruption()
# Push an out-of-band frame (i.e. not using the ordered push
# frame task) to stop everything, specially at the output
# transport.
await self.push_frame(StartInterruptionFrame())
elif self.interruption_strategies and self._bot_speaking:
logger.debug(
"User started speaking while bot is speaking with interruption config - "
"deferring interruption to aggregator"
)
elif isinstance(frame, UserStoppedSpeakingFrame):
logger.debug("User stopped speaking")
await self.push_frame(frame)
@@ -227,10 +268,30 @@ class BaseInputTransport(FrameProcessor):
await self._stop_interruption()
await self.push_frame(StopInterruptionFrame())
#
# Handle bot speaking state
#
async def _handle_bot_started_speaking(self, frame: BotStartedSpeakingFrame):
self._bot_speaking = True
async def _handle_bot_stopped_speaking(self, frame: BotStoppedSpeakingFrame):
self._bot_speaking = False
#
# Audio input
#
def _create_audio_task(self):
if not self._audio_task and self._params.audio_in_enabled:
self._audio_in_queue = asyncio.Queue()
self._audio_task = self.create_task(self._audio_task_handler())
async def _cancel_audio_task(self):
if self._audio_task:
await self.cancel_task(self._audio_task)
self._audio_task = None
async def _vad_analyze(self, audio_frame: InputAudioRawFrame) -> VADState:
state = VADState.QUIET
if self.vad_analyzer:

21
src/pipecat/transports/base_output.py
View File

@@ -25,6 +25,8 @@ from pipecat.frames.frames import (
Frame,
MixerControlFrame,
OutputAudioRawFrame,
OutputDTMFFrame,
OutputDTMFUrgentFrame,
OutputImageRawFrame,
SpriteFrame,
StartFrame,
@@ -132,12 +134,13 @@ class BaseOutputTransport(FrameProcessor):
async def register_audio_destination(self, destination: str):
pass
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
async def write_video_frame(self, frame: OutputImageRawFrame):
pass
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
pass
async def write_dtmf(self, frame: OutputDTMFFrame | OutputDTMFUrgentFrame):
pass
async def send_audio(self, frame: OutputAudioRawFrame):
@@ -171,6 +174,8 @@ class BaseOutputTransport(FrameProcessor):
await self._handle_frame(frame)
elif isinstance(frame, TransportMessageUrgentFrame):
await self.send_message(frame)
elif isinstance(frame, OutputDTMFUrgentFrame):
await self.write_dtmf(frame)
elif isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
# Control frames.
@@ -346,6 +351,7 @@ class BaseOutputTransport(FrameProcessor):
sample_rate=self._sample_rate,
num_channels=frame.num_channels,
)
chunk.transport_destination = self._destination
await self._audio_queue.put(chunk)
self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
@@ -425,6 +431,8 @@ class BaseOutputTransport(FrameProcessor):
await self._set_video_images(frame.images)
elif isinstance(frame, TransportMessageFrame):
await self._transport.send_message(frame)
elif isinstance(frame, OutputDTMFFrame):
await self._transport.write_dtmf(frame)
def _next_frame(self) -> AsyncGenerator[Frame, None]:
async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
@@ -498,7 +506,7 @@ class BaseOutputTransport(FrameProcessor):
# Send audio.
if isinstance(frame, OutputAudioRawFrame):
await self._transport.write_raw_audio_frames(frame.audio, self._destination)
await self._transport.write_audio_frame(frame)
#
# Video handling
@@ -581,8 +589,7 @@ class BaseOutputTransport(FrameProcessor):
frame = await self._transport.get_event_loop().run_in_executor(
self._executor, resize_frame, frame
)
await self._transport.write_raw_video_frame(frame, self._destination)
await self._transport.write_video_frame(frame)
#
# Clock handling

6
src/pipecat/transports/local/audio.py
View File

@@ -10,7 +10,7 @@ from typing import Optional
from loguru import logger
from pipecat.frames.frames import InputAudioRawFrame, StartFrame
from pipecat.frames.frames import InputAudioRawFrame, OutputAudioRawFrame, StartFrame
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
@@ -122,10 +122,10 @@ class LocalAudioOutputTransport(BaseOutputTransport):
self._out_stream.close()
self._out_stream = None
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
if self._out_stream:
await self.get_event_loop().run_in_executor(
self._executor, self._out_stream.write, frames
self._executor, self._out_stream.write, frame.audio
)

15
src/pipecat/transports/local/tk.py
View File

@@ -12,7 +12,12 @@ from typing import Optional
import numpy as np
from loguru import logger
from pipecat.frames.frames import InputAudioRawFrame, OutputImageRawFrame, StartFrame
from pipecat.frames.frames import (
InputAudioRawFrame,
OutputAudioRawFrame,
OutputImageRawFrame,
StartFrame,
)
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -135,15 +140,13 @@ class TkOutputTransport(BaseOutputTransport):
self._out_stream.close()
self._out_stream = None
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
if self._out_stream:
await self.get_event_loop().run_in_executor(
self._executor, self._out_stream.write, frames
self._executor, self._out_stream.write, frame.audio
)
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
async def write_video_frame(self, frame: OutputImageRawFrame):
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
def _write_frame_to_tk(self, frame: OutputImageRawFrame):

48
src/pipecat/transports/network/fastapi_websocket.py
View File

@@ -26,7 +26,7 @@ from pipecat.frames.frames import (
TransportMessageFrame,
TransportMessageUrgentFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
@@ -45,7 +45,7 @@ except ModuleNotFoundError as e:
class FastAPIWebsocketParams(TransportParams):
add_wav_header: bool = False
serializer: FrameSerializer
serializer: Optional[FrameSerializer] = None
session_timeout: Optional[int] = None
@@ -92,7 +92,7 @@ class FastAPIWebsocketClient:
async def trigger_client_connected(self):
await self._callbacks.on_client_connected(self._websocket)
async def trigger_client_timout(self):
async def trigger_client_timeout(self):
await self._callbacks.on_session_timeout(self._websocket)
def _can_send(self):
@@ -122,10 +122,20 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
self._receive_task = None
self._monitor_websocket_task = None
# Whether we have seen a StartFrame already.
self._initialized = False
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.setup(frame)
await self._params.serializer.setup(frame)
if self._params.serializer:
await self._params.serializer.setup(frame)
if not self._monitor_websocket_task and self._params.session_timeout:
self._monitor_websocket_task = self.create_task(self._monitor_websocket())
await self._client.trigger_client_connected()
@@ -158,6 +168,9 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
async def _receive_messages(self):
try:
async for message in self._client.receive():
if not self._params.serializer:
continue
frame = await self._params.serializer.deserialize(message)
if not frame:
@@ -175,7 +188,7 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
async def _monitor_websocket(self):
"""Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
await asyncio.sleep(self._params.session_timeout)
await self._client.trigger_client_timout()
await self._client.trigger_client_timeout()
class FastAPIWebsocketOutputTransport(BaseOutputTransport):
@@ -192,7 +205,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
self._client = client
self._params = params
# write_raw_audio_frames() is called quickly, as soon as we get audio
# write_audio_frame() is called quickly, as soon as we get audio
# (e.g. from the TTS), and since this is just a network connection we
# would be sending it to quickly. Instead, we want to block to emulate
# an audio device, this is what the send interval is. It will be
@@ -200,10 +213,20 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
self._send_interval = 0
self._next_send_time = 0
# Whether we have seen a StartFrame already.
self._initialized = False
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.setup(frame)
await self._params.serializer.setup(frame)
if self._params.serializer:
await self._params.serializer.setup(frame)
self._send_interval = (self.audio_chunk_size / self.sample_rate) / 2
await self.set_transport_ready(frame)
@@ -231,7 +254,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._write_frame(frame)
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
if self._client.is_closing:
return
@@ -241,7 +264,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
return
frame = OutputAudioRawFrame(
audio=frames,
audio=frame.audio,
sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
@@ -266,6 +289,9 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
if not self._params.serializer:
return
try:
payload = await self._params.serializer.serialize(frame)
if payload:
@@ -302,7 +328,9 @@ class FastAPIWebsocketTransport(BaseTransport):
on_session_timeout=self._on_session_timeout,
)
is_binary = self._params.serializer.type == FrameSerializerType.BINARY
is_binary = False
if self._params.serializer:
is_binary = self._params.serializer.type == FrameSerializerType.BINARY
self._client = FastAPIWebsocketClient(websocket, is_binary, self._callbacks)
self._input = FastAPIWebsocketInputTransport(

59
src/pipecat/transports/network/small_webrtc.py
View File

@@ -19,7 +19,6 @@ from pipecat.frames.frames import (
EndFrame,
Frame,
InputAudioRawFrame,
InputImageRawFrame,
OutputAudioRawFrame,
OutputImageRawFrame,
SpriteFrame,
@@ -50,7 +49,6 @@ class SmallWebRTCCallbacks(BaseModel):
on_app_message: Callable[[Any], Awaitable[None]]
on_client_connected: Callable[[SmallWebRTCConnection], Awaitable[None]]
on_client_disconnected: Callable[[SmallWebRTCConnection], Awaitable[None]]
on_client_closed: Callable[[SmallWebRTCConnection], Awaitable[None]]
class RawAudioTrack(AudioStreamTrack):
@@ -169,7 +167,7 @@ class SmallWebRTCClient:
@self._webrtc_connection.event_handler("disconnected")
async def on_disconnected(connection: SmallWebRTCConnection):
logger.debug("Peer connection lost.")
await self._handle_client_disconnected()
await self._handle_peer_disconnected()
@self._webrtc_connection.event_handler("closed")
async def on_closed(connection: SmallWebRTCConnection):
@@ -233,7 +231,8 @@ class SmallWebRTCClient:
frame_array = frame.to_ndarray(format=format_name)
frame_rgb = self._convert_frame(frame_array, format_name)
image_frame = InputImageRawFrame(
image_frame = UserImageRawFrame(
user_id=self._webrtc_connection.pc_id,
image=frame_rgb.tobytes(),
size=(frame.width, frame.height),
format="RGB",
@@ -284,13 +283,11 @@ class SmallWebRTCClient:
)
yield audio_frame
async def write_raw_audio_frames(self, data: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
if self._can_send() and self._audio_output_track:
await self._audio_output_track.add_audio_bytes(data)
await self._audio_output_track.add_audio_bytes(frame.audio)
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
async def write_video_frame(self, frame: OutputImageRawFrame):
if self._can_send() and self._video_output_track:
self._video_output_track.add_video_frame(frame)
@@ -313,7 +310,7 @@ class SmallWebRTCClient:
logger.info(f"Disconnecting to Small WebRTC")
self._closing = True
await self._webrtc_connection.disconnect()
await self._handle_client_disconnected()
await self._handle_peer_disconnected()
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
if self._can_send():
@@ -338,19 +335,18 @@ class SmallWebRTCClient:
await self._callbacks.on_client_connected(self._webrtc_connection)
async def _handle_client_disconnected(self):
async def _handle_peer_disconnected(self):
self._audio_input_track = None
self._video_input_track = None
self._audio_output_track = None
self._video_output_track = None
await self._callbacks.on_client_disconnected(self._webrtc_connection)
async def _handle_client_closed(self):
self._audio_input_track = None
self._video_input_track = None
self._audio_output_track = None
self._video_output_track = None
await self._callbacks.on_client_closed(self._webrtc_connection)
await self._callbacks.on_client_disconnected(self._webrtc_connection)
async def _handle_app_message(self, message: Any):
await self._callbacks.on_app_message(message)
@@ -381,6 +377,9 @@ class SmallWebRTCInputTransport(BaseInputTransport):
self._receive_video_task = None
self._image_requests = {}
# Whether we have seen a StartFrame already.
self._initialized = False
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -389,6 +388,12 @@ class SmallWebRTCInputTransport(BaseInputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.setup(self._params, frame)
await self._client.connect()
if not self._receive_audio_task and self._params.audio_in_enabled:
@@ -428,7 +433,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
try:
async for video_frame in self._client.read_video_frame():
if video_frame:
await self.push_frame(video_frame)
await self.push_video_frame(video_frame)
# Check if there are any pending image requests and create UserImageRawFrame
if self._image_requests:
@@ -442,7 +447,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
format=video_frame.format,
)
# Push the frame to the pipeline
await self.push_frame(image_frame)
await self.push_video_frame(image_frame)
# Remove from pending requests
del self._image_requests[req_id]
@@ -484,8 +489,17 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
self._client = client
self._params = params
# Whether we have seen a StartFrame already.
self._initialized = False
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.setup(self._params, frame)
await self._client.connect()
await self.set_transport_ready(frame)
@@ -501,13 +515,11 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._client.send_message(frame)
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
await self._client.write_raw_audio_frames(frames)
async def write_audio_frame(self, frame: OutputAudioRawFrame):
await self._client.write_audio_frame(frame)
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
await self._client.write_raw_video_frame(frame)
async def write_video_frame(self, frame: OutputImageRawFrame):
await self._client.write_video_frame(frame)
class SmallWebRTCTransport(BaseTransport):
@@ -525,7 +537,6 @@ class SmallWebRTCTransport(BaseTransport):
on_app_message=self._on_app_message,
on_client_connected=self._on_client_connected,
on_client_disconnected=self._on_client_disconnected,
on_client_closed=self._on_client_closed,
)
self._client = SmallWebRTCClient(webrtc_connection, self._callbacks)
@@ -538,7 +549,6 @@ class SmallWebRTCTransport(BaseTransport):
self._register_event_handler("on_app_message")
self._register_event_handler("on_client_connected")
self._register_event_handler("on_client_disconnected")
self._register_event_handler("on_client_closed")
def input(self) -> SmallWebRTCInputTransport:
if not self._input:
@@ -572,6 +582,3 @@ class SmallWebRTCTransport(BaseTransport):
async def _on_client_disconnected(self, webrtc_connection):
await self._call_event_handler("on_client_disconnected", webrtc_connection)
async def _on_client_closed(self, webrtc_connection):
await self._call_event_handler("on_client_closed", webrtc_connection)

52
src/pipecat/transports/network/websocket_client.py
View File

@@ -24,6 +24,7 @@ from pipecat.frames.frames import (
TransportMessageFrame,
TransportMessageUrgentFrame,
)
from pipecat.processors.frame_processor import FrameProcessorSetup
from pipecat.serializers.base_serializer import FrameSerializer
from pipecat.serializers.protobuf import ProtobufFrameSerializer
from pipecat.transports.base_input import BaseInputTransport
@@ -34,7 +35,7 @@ from pipecat.utils.asyncio import BaseTaskManager
class WebsocketClientParams(TransportParams):
add_wav_header: bool = True
serializer: FrameSerializer = ProtobufFrameSerializer()
serializer: Optional[FrameSerializer] = None
class WebsocketClientCallbacks(BaseModel):
@@ -68,10 +69,10 @@ class WebsocketClientSession:
)
return self._task_manager
async def setup(self, frame: StartFrame):
async def setup(self, task_manager: BaseTaskManager):
self._leave_counter += 1
if not self._task_manager:
self._task_manager = frame.task_manager
self._task_manager = task_manager
async def connect(self):
if self._websocket:
@@ -131,10 +132,23 @@ class WebsocketClientInputTransport(BaseInputTransport):
self._session = session
self._params = params
# Whether we have seen a StartFrame already.
self._initialized = False
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._session.setup(setup.task_manager)
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
await self._session.setup(frame)
if self._initialized:
return
self._initialized = True
if self._params.serializer:
await self._params.serializer.setup(frame)
await self._session.connect()
await self.set_transport_ready(frame)
@@ -151,6 +165,8 @@ class WebsocketClientInputTransport(BaseInputTransport):
await self._transport.cleanup()
async def on_message(self, websocket, message):
if not self._params.serializer:
return
frame = await self._params.serializer.deserialize(message)
if not frame:
return
@@ -173,7 +189,7 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
self._session = session
self._params = params
# write_raw_audio_frames() is called quickly, as soon as we get audio
# write_audio_frame() is called quickly, as soon as we get audio
# (e.g. from the TTS), and since this is just a network connection we
# would be sending it to quickly. Instead, we want to block to emulate
# an audio device, this is what the send interval is. It will be
@@ -181,11 +197,24 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
self._send_interval = 0
self._next_send_time = 0
# Whether we have seen a StartFrame already.
self._initialized = False
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._session.setup(setup.task_manager)
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
self._send_interval = (self.audio_chunk_size / self.sample_rate) / 2
await self._params.serializer.setup(frame)
await self._session.setup(frame)
if self._params.serializer:
await self._params.serializer.setup(frame)
await self._session.connect()
await self.set_transport_ready(frame)
@@ -204,9 +233,9 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._write_frame(frame)
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
frame = OutputAudioRawFrame(
audio=frames,
audio=frame.audio,
sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
@@ -231,6 +260,8 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
if not self._params.serializer:
return
payload = await self._params.serializer.serialize(frame)
if payload:
await self._session.send(payload)
@@ -255,6 +286,7 @@ class WebsocketClientTransport(BaseTransport):
super().__init__()
self._params = params or WebsocketClientParams()
self._params.serializer = self._params.serializer or ProtobufFrameSerializer()
callbacks = WebsocketClientCallbacks(
on_connected=self._on_connected,

38
src/pipecat/transports/network/websocket_server.py
View File

@@ -40,7 +40,7 @@ except ModuleNotFoundError as e:
class WebsocketServerParams(TransportParams):
add_wav_header: bool = False
serializer: FrameSerializer
serializer: Optional[FrameSerializer] = None
session_timeout: Optional[int] = None
@@ -78,9 +78,19 @@ class WebsocketServerInputTransport(BaseInputTransport):
self._stop_server_event = asyncio.Event()
# Whether we have seen a StartFrame already.
self._initialized = False
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
if self._initialized:
return
self._initialized = True
if self._params.serializer:
await self._params.serializer.setup(frame)
if not self._server_task:
self._server_task = self.create_task(self._server_task_handler())
await self.set_transport_ready(frame)
@@ -134,6 +144,9 @@ class WebsocketServerInputTransport(BaseInputTransport):
# Handle incoming messages
try:
async for message in websocket:
if not self._params.serializer:
continue
frame = await self._params.serializer.deserialize(message)
if not frame:
@@ -178,7 +191,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
self._websocket: Optional[websockets.WebSocketServerProtocol] = None
# write_raw_audio_frames() is called quickly, as soon as we get audio
# write_audio_frame() is called quickly, as soon as we get audio
# (e.g. from the TTS), and since this is just a network connection we
# would be sending it to quickly. Instead, we want to block to emulate
# an audio device, this is what the send interval is. It will be
@@ -186,6 +199,9 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
self._send_interval = 0
self._next_send_time = 0
# Whether we have seen a StartFrame already.
self._initialized = False
async def set_client_connection(self, websocket: Optional[websockets.WebSocketServerProtocol]):
if self._websocket:
await self._websocket.close()
@@ -194,7 +210,14 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
if self._initialized:
return
self._initialized = True
if self._params.serializer:
await self._params.serializer.setup(frame)
self._send_interval = (self.audio_chunk_size / self.sample_rate) / 2
await self.set_transport_ready(frame)
@@ -220,14 +243,14 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._write_frame(frame)
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
if not self._websocket:
# Simulate audio playback with a sleep.
await self._write_audio_sleep()
return
frame = OutputAudioRawFrame(
audio=frames,
audio=frame.audio,
sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
@@ -252,6 +275,9 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
if not self._params.serializer:
return
try:
payload = await self._params.serializer.serialize(frame)
if payload and self._websocket:

163
src/pipecat/transports/services/daily.py
View File

@@ -22,6 +22,8 @@ from pipecat.frames.frames import (
Frame,
InterimTranscriptionFrame,
OutputAudioRawFrame,
OutputDTMFFrame,
OutputDTMFUrgentFrame,
OutputImageRawFrame,
SpriteFrame,
StartFrame,
@@ -370,9 +372,10 @@ class DailyTransportClient(EventHandler):
self._custom_audio_tracks[destination] = await self.add_custom_audio_track(destination)
self._client.update_publishing({"customAudio": {destination: True}})
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
future = self._get_event_loop().create_future()
destination = frame.transport_destination
audio_source: Optional[CustomAudioSource] = None
if not destination and self._microphone_track:
audio_source = self._microphone_track.source
@@ -381,17 +384,15 @@ class DailyTransportClient(EventHandler):
audio_source = track.source
if audio_source:
audio_source.write_frames(frames, completion=completion_callback(future))
audio_source.write_frames(frame.audio, completion=completion_callback(future))
else:
logger.warning(f"{self} unable to write audio frames to destination [{destination}]")
future.set_result(None)
await future
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
if not destination and self._camera:
async def write_video_frame(self, frame: OutputImageRawFrame):
if not frame.transport_destination and self._camera:
self._camera.write_frame(frame.image)
async def setup(self, setup: FrameProcessorSetup):
@@ -476,7 +477,7 @@ class DailyTransportClient(EventHandler):
logger.info(f"Joined {self._room_url}")
if self._params.transcription_enabled:
await self._start_transcription()
await self.start_transcription(self._params.transcription_settings)
await self._callbacks.on_joined(data)
@@ -491,23 +492,6 @@ class DailyTransportClient(EventHandler):
self._joining = False
await self._callbacks.on_error(error_msg)
async def _start_transcription(self):
if not self._token:
logger.warning("Transcription can't be started without a room token")
return
logger.info(f"Enabling transcription with settings {self._params.transcription_settings}")
future = self._get_event_loop().create_future()
self._client.start_transcription(
settings=self._params.transcription_settings.model_dump(exclude_none=True),
completion=completion_callback(future),
)
error = await future
if error:
logger.error(f"Unable to start transcription: {error}")
return
async def _join(self):
future = self._get_event_loop().create_future()
@@ -577,7 +561,7 @@ class DailyTransportClient(EventHandler):
logger.info(f"Leaving {self._room_url}")
if self._params.transcription_enabled:
await self._stop_transcription()
await self.stop_transcription()
# Remove any custom tracks, if any.
for track_name, _ in self._custom_audio_tracks.items():
@@ -597,15 +581,6 @@ class DailyTransportClient(EventHandler):
logger.error(error_msg)
await self._callbacks.on_error(error_msg)
async def _stop_transcription(self):
if not self._token:
return
future = self._get_event_loop().create_future()
self._client.stop_transcription(completion=completion_callback(future))
error = await future
if error:
logger.error(f"Unable to stop transcription: {error}")
async def _leave(self):
future = self._get_event_loop().create_future()
self._client.leave(completion=completion_callback(future))
@@ -623,14 +598,22 @@ class DailyTransportClient(EventHandler):
return self._client.participant_counts()
async def start_dialout(self, settings):
logger.debug(f"Starting dialout: settings={settings}")
future = self._get_event_loop().create_future()
self._client.start_dialout(settings, completion=completion_callback(future))
await future
error = await future
if error:
logger.error(f"Unable to start dialout: {error}")
async def stop_dialout(self, participant_id):
logger.debug(f"Stopping dialout: participant_id={participant_id}")
future = self._get_event_loop().create_future()
self._client.stop_dialout(participant_id, completion=completion_callback(future))
await future
error = await future
if error:
logger.error(f"Unable to stop dialout: {error}")
async def send_dtmf(self, settings):
future = self._get_event_loop().create_future()
@@ -648,16 +631,54 @@ class DailyTransportClient(EventHandler):
await future
async def start_recording(self, streaming_settings, stream_id, force_new):
logger.debug(
f"Starting recording: stream_id={stream_id} force_new={force_new} settings={streaming_settings}"
)
future = self._get_event_loop().create_future()
self._client.start_recording(
streaming_settings, stream_id, force_new, completion=completion_callback(future)
)
await future
error = await future
if error:
logger.error(f"Unable to start recording: {error}")
async def stop_recording(self, stream_id):
logger.debug(f"Stopping recording: stream_id={stream_id}")
future = self._get_event_loop().create_future()
self._client.stop_recording(stream_id, completion=completion_callback(future))
await future
error = await future
if error:
logger.error(f"Unable to stop recording: {error}")
async def start_transcription(self, settings):
if not self._token:
logger.warning("Transcription can't be started without a room token")
return
logger.debug(f"Starting transcription: settings={settings}")
future = self._get_event_loop().create_future()
self._client.start_transcription(
settings=self._params.transcription_settings.model_dump(exclude_none=True),
completion=completion_callback(future),
)
error = await future
if error:
logger.error(f"Unable to start transcription: {error}")
async def stop_transcription(self):
if not self._token:
return
logger.debug(f"Stopping transcription")
future = self._get_event_loop().create_future()
self._client.stop_transcription(completion=completion_callback(future))
error = await future
if error:
logger.error(f"Unable to stop transcription: {error}")
async def send_prebuilt_chat_message(self, message: str, user_name: Optional[str] = None):
if not self._joined:
@@ -695,7 +716,9 @@ class DailyTransportClient(EventHandler):
self._audio_renderers.setdefault(participant_id, {})[audio_source] = callback
logger.info(f"Starting to capture [{audio_source}] audio from participant {participant_id}")
logger.debug(
f"Starting to capture [{audio_source}] audio from participant {participant_id}"
)
self._client.set_audio_renderer(
participant_id,
@@ -723,6 +746,10 @@ class DailyTransportClient(EventHandler):
self._video_renderers.setdefault(participant_id, {})[video_source] = callback
logger.debug(
f"Starting to capture [{video_source}] video from participant {participant_id}"
)
self._client.set_video_renderer(
participant_id,
self._video_frame_received,
@@ -979,14 +1006,14 @@ class DailyInputTransport(BaseInputTransport):
await self._transport.cleanup()
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
if self._initialized:
return
self._initialized = True
# Parent start.
await super().start(frame)
# Setup client.
await self._client.start(frame)
@@ -1106,7 +1133,7 @@ class DailyInputTransport(BaseInputTransport):
next_time = prev_time + 1 / framerate
render_frame = (next_time - curr_time) < 0.1
elif self._video_renderers[participant_id][video_source]["render_next_frame"]:
if self._video_renderers[participant_id][video_source]["render_next_frame"]:
request_frame = self._video_renderers[participant_id][video_source][
"render_next_frame"
].pop(0)
@@ -1121,7 +1148,7 @@ class DailyInputTransport(BaseInputTransport):
format=video_frame.color_format,
)
frame.transport_source = video_source
await self.push_frame(frame)
await self.push_video_frame(frame)
self._video_renderers[participant_id][video_source]["timestamp"] = curr_time
@@ -1156,14 +1183,14 @@ class DailyOutputTransport(BaseOutputTransport):
await self._transport.cleanup()
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
if self._initialized:
return
self._initialized = True
# Parent start.
await super().start(frame)
# Setup client.
await self._client.start(frame)
@@ -1194,13 +1221,19 @@ class DailyOutputTransport(BaseOutputTransport):
async def register_audio_destination(self, destination: str):
await self._client.register_audio_destination(destination)
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
await self._client.write_raw_audio_frames(frames, destination)
async def write_dtmf(self, frame: OutputDTMFFrame | OutputDTMFUrgentFrame):
await self._client.send_dtmf(
{
"sessionId": frame.transport_destination,
"tones": frame.button.value,
}
)
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
await self._client.write_raw_video_frame(frame, destination)
async def write_audio_frame(self, frame: OutputAudioRawFrame):
await self._client.write_audio_frame(frame)
async def write_video_frame(self, frame: OutputImageRawFrame):
await self._client.write_video_frame(frame)
class DailyTransport(BaseTransport):
@@ -1346,6 +1379,14 @@ class DailyTransport(BaseTransport):
await self._client.stop_dialout(participant_id)
async def send_dtmf(self, settings):
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"`DailyTransport.send_dtmf()` is deprecated, push an `OutputDTMFFrame` or an `OutputDTMFUrgentFrame` instead.",
DeprecationWarning,
)
await self._client.send_dtmf(settings)
async def sip_call_transfer(self, settings):
@@ -1360,6 +1401,12 @@ class DailyTransport(BaseTransport):
async def stop_recording(self, stream_id=None):
await self._client.stop_recording(stream_id)
async def start_transcription(self, settings=None):
await self._client.start_transcription(settings)
async def stop_transcription(self):
await self._client.stop_transcription()
async def send_prebuilt_chat_message(self, message: str, user_name: Optional[str] = None):
"""Sends a chat message to Daily's Prebuilt main room.
@@ -1551,10 +1598,16 @@ class DailyTransport(BaseTransport):
except KeyError:
language = None
if is_final:
frame = TranscriptionFrame(text, participant_id, timestamp, language)
frame = TranscriptionFrame(text, participant_id, timestamp, language, result=message)
logger.debug(f"Transcription (from: {participant_id}): [{text}]")
else:
frame = InterimTranscriptionFrame(text, participant_id, timestamp, language)
frame = InterimTranscriptionFrame(
text,
participant_id,
timestamp,
language,
result=message,
)
if self._input:
await self._input.push_transcription_frame(frame)

22
src/pipecat/transports/services/livekit.py
View File

@@ -364,12 +364,21 @@ class LiveKitInputTransport(BaseInputTransport):
self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer
self._resampler = create_default_resampler()
# Whether we have seen a StartFrame already.
self._initialized = False
@property
def vad_analyzer(self) -> Optional[VADAnalyzer]:
return self._vad_analyzer
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.start(frame)
await self._client.connect()
if not self._audio_in_task and self._params.audio_in_enabled:
@@ -447,8 +456,17 @@ class LiveKitOutputTransport(BaseOutputTransport):
self._transport = transport
self._client = client
# Whether we have seen a StartFrame already.
self._initialized = False
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.start(frame)
await self._client.connect()
await self.set_transport_ready(frame)
@@ -477,8 +495,8 @@ class LiveKitOutputTransport(BaseOutputTransport):
else:
await self._client.send_data(frame.message.encode())
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
livekit_audio = self._convert_pipecat_audio_to_livekit(frames)
async def write_audio_frame(self, frame: OutputAudioRawFrame):
livekit_audio = self._convert_pipecat_audio_to_livekit(frame.audio)
await self._client.publish_audio(livekit_audio)
def _convert_pipecat_audio_to_livekit(self, pipecat_audio: bytes) -> rtc.AudioFrame:

43
src/pipecat/transports/services/tavus.py
View File

@@ -17,7 +17,7 @@ from pipecat.frames.frames import (
EndFrame,
Frame,
InputAudioRawFrame,
OutputImageRawFrame,
OutputAudioRawFrame,
StartFrame,
StartInterruptionFrame,
TransportMessageFrame,
@@ -290,12 +290,18 @@ class TavusTransportClient:
await self.send_message(transport_frame)
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
if not self._client:
return
await self._client.update_subscriptions(
participant_settings=participant_settings, profile_settings=profile_settings
)
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
await self._client.write_raw_audio_frames(frames, destination)
async def write_audio_frame(self, frame: OutputAudioRawFrame):
if not self._client:
return
await self._client.write_audio_frame(frame)
class TavusInputTransport(BaseInputTransport):
@@ -310,6 +316,9 @@ class TavusInputTransport(BaseInputTransport):
self._params = params
self._resampler = create_default_resampler()
# Whether we have seen a StartFrame already.
self._initialized = False
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._client.setup(setup)
@@ -320,6 +329,12 @@ class TavusInputTransport(BaseInputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.start(frame)
await self.set_transport_ready(frame)
@@ -368,6 +383,9 @@ class TavusOutputTransport(BaseOutputTransport):
self._start_time = None
self._current_idx_str: Optional[str] = None
# Whether we have seen a StartFrame already.
self._initialized = False
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await self._client.setup(setup)
@@ -378,6 +396,12 @@ class TavusOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
if self._initialized:
return
self._initialized = True
await self._client.start(frame)
await self.set_transport_ready(frame)
@@ -418,26 +442,21 @@ class TavusOutputTransport(BaseOutputTransport):
async def _handle_interruptions(self):
await self._client.send_interrupt_message()
async def write_raw_audio_frames(self, frames: bytes, destination: Optional[str] = None):
async def write_audio_frame(self, frame: OutputAudioRawFrame):
# Compute wait time for synchronization
wait = self._start_time + (self._samples_sent / self.sample_rate) - time.time()
if wait > 0:
logger.trace(f"TavusOutputTransport write_raw_audio_frames wait: {wait}")
logger.trace(f"TavusOutputTransport write_audio_frame wait: {wait}")
await asyncio.sleep(wait)
if self._current_idx_str is None:
logger.warning("TavusOutputTransport self._current_idx_str not defined yet!")
return
await self._client.encode_audio_and_send(frames, False, self._current_idx_str)
await self._client.encode_audio_and_send(frame.audio, False, self._current_idx_str)
# Update timestamp based on number of samples sent
self._samples_sent += len(frames) // 2 # 2 bytes per sample (16-bit)
async def write_raw_video_frame(
self, frame: OutputImageRawFrame, destination: Optional[str] = None
):
pass
self._samples_sent += len(frame.audio) // 2 # 2 bytes per sample (16-bit)
class TavusTransport(BaseTransport):

14
src/pipecat/utils/base_object.py
View File

@@ -49,14 +49,16 @@ class BaseObject(ABC):
return decorator
def add_event_handler(self, event_name: str, handler):
if event_name not in self._event_handlers:
raise Exception(f"Event handler {event_name} not registered")
self._event_handlers[event_name].append(handler)
if event_name in self._event_handlers:
self._event_handlers[event_name].append(handler)
else:
logger.warning(f"Event handler {event_name} not registered")
def _register_event_handler(self, event_name: str):
if event_name in self._event_handlers:
raise Exception(f"Event handler {event_name} already registered")
self._event_handlers[event_name] = []
if event_name not in self._event_handlers:
self._event_handlers[event_name] = []
else:
logger.warning(f"Event handler {event_name} not registered")
async def _call_event_handler(self, event_name: str, *args, **kwargs):
# If we haven't registered an event handler, we don't need to do

230
src/pipecat/utils/tracing/service_attributes.py
View File

@@ -6,7 +6,7 @@
"""Functions for adding attributes to OpenTelemetry spans."""
from typing import TYPE_CHECKING, Any, Dict, Optional
from typing import TYPE_CHECKING, Any, Dict, List, Optional
# Import for type checking only
if TYPE_CHECKING:
@@ -60,7 +60,7 @@ def add_tts_span_attributes(
settings: Optional[Dict[str, Any]] = None,
character_count: Optional[int] = None,
operation_name: str = "tts",
ttfb_ms: Optional[float] = None,
ttfb: Optional[float] = None,
**kwargs,
) -> None:
"""Add TTS-specific attributes to a span.
@@ -74,7 +74,7 @@ def add_tts_span_attributes(
settings: Service configuration settings
character_count: Number of characters in the text
operation_name: Name of the operation (default: "tts")
ttfb_ms: Time to first byte in milliseconds
ttfb: Time to first byte in seconds
**kwargs: Additional attributes to add
"""
# Add standard attributes
@@ -91,8 +91,8 @@ def add_tts_span_attributes(
if character_count is not None:
span.set_attribute("metrics.character_count", character_count)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
if ttfb is not None:
span.set_attribute("metrics.ttfb", ttfb)
# Add settings if provided
if settings:
@@ -116,7 +116,7 @@ def add_stt_span_attributes(
language: Optional[str] = None,
settings: Optional[Dict[str, Any]] = None,
vad_enabled: bool = False,
ttfb_ms: Optional[float] = None,
ttfb: Optional[float] = None,
**kwargs,
) -> None:
"""Add STT-specific attributes to a span.
@@ -131,7 +131,7 @@ def add_stt_span_attributes(
language: Detected or configured language
settings: Service configuration settings
vad_enabled: Whether voice activity detection is enabled
ttfb_ms: Time to first byte in milliseconds
ttfb: Time to first byte in seconds
**kwargs: Additional attributes to add
"""
# Add standard attributes
@@ -150,8 +150,8 @@ def add_stt_span_attributes(
if language:
span.set_attribute("language", language)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
if ttfb is not None:
span.set_attribute("metrics.ttfb", ttfb)
# Add settings if provided
if settings:
@@ -178,7 +178,7 @@ def add_llm_span_attributes(
system: Optional[str] = None,
parameters: Optional[Dict[str, Any]] = None,
extra_parameters: Optional[Dict[str, Any]] = None,
ttfb_ms: Optional[float] = None,
ttfb: Optional[float] = None,
**kwargs,
) -> None:
"""Add LLM-specific attributes to a span.
@@ -196,7 +196,7 @@ def add_llm_span_attributes(
system: System message
parameters: Service parameters
extra_parameters: Additional parameters
ttfb_ms: Time to first byte in milliseconds
ttfb: Time to first byte in seconds
**kwargs: Additional attributes to add
"""
# Add standard attributes
@@ -225,8 +225,8 @@ def add_llm_span_attributes(
if system:
span.set_attribute("system", system)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
if ttfb is not None:
span.set_attribute("metrics.ttfb", ttfb)
# Add parameters if provided
if parameters:
@@ -256,3 +256,207 @@ def add_llm_span_attributes(
for key, value in kwargs.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(key, value)
def add_gemini_live_span_attributes(
span: "Span",
service_name: str,
model: str,
operation_name: str,
voice_id: Optional[str] = None,
language: Optional[str] = None,
modalities: Optional[str] = None,
settings: Optional[Dict[str, Any]] = None,
tools: Optional[List[Dict]] = None,
tools_serialized: Optional[str] = None,
transcript: Optional[str] = None,
is_input: Optional[bool] = None,
text_output: Optional[str] = None,
audio_data_size: Optional[int] = None,
**kwargs,
) -> None:
"""Add Gemini Live specific attributes to a span.
Args:
span: The span to add attributes to
service_name: Name of the service
model: Model name/identifier
operation_name: Name of the operation (setup, model_turn, tool_call, etc.)
voice_id: Voice identifier used for output
language: Language code for the session
modalities: Supported modalities (e.g., "AUDIO", "TEXT")
settings: Service configuration settings
tools: Available tools/functions list
tools_serialized: JSON-serialized tools for detailed inspection
transcript: Transcription text
is_input: Whether transcript is input (True) or output (False)
text_output: Text output from model
audio_data_size: Size of audio data in bytes
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("gen_ai.system", "gcp.gemini")
span.set_attribute("gen_ai.request.model", model)
span.set_attribute("gen_ai.operation.name", operation_name)
span.set_attribute("service.operation", operation_name)
# Add optional attributes
if voice_id:
span.set_attribute("voice_id", voice_id)
if language:
span.set_attribute("language", language)
if modalities:
span.set_attribute("modalities", modalities)
if transcript:
span.set_attribute("transcript", transcript)
if is_input is not None:
span.set_attribute("transcript.is_input", is_input)
if text_output:
span.set_attribute("text_output", text_output)
if audio_data_size is not None:
span.set_attribute("audio.data_size_bytes", audio_data_size)
if tools:
span.set_attribute("tools.count", len(tools))
span.set_attribute("tools.available", True)
# Add individual tool names for easier filtering
tool_names = []
for tool in tools:
if isinstance(tool, dict) and "name" in tool:
tool_names.append(tool["name"])
elif hasattr(tool, "name"):
tool_name = getattr(tool, "name", None)
if tool_name is not None:
tool_names.append(tool_name)
if tool_names:
span.set_attribute("tools.names", ",".join(tool_names))
if tools_serialized:
span.set_attribute("tools.definitions", tools_serialized)
# Add settings if provided
if settings:
for key, value in settings.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(f"settings.{key}", value)
elif key == "vad" and value:
# Handle VAD settings specially
if hasattr(value, "disabled") and value.disabled is not None:
span.set_attribute("settings.vad.disabled", value.disabled)
if hasattr(value, "start_sensitivity") and value.start_sensitivity:
span.set_attribute(
"settings.vad.start_sensitivity", value.start_sensitivity.value
)
if hasattr(value, "end_sensitivity") and value.end_sensitivity:
span.set_attribute("settings.vad.end_sensitivity", value.end_sensitivity.value)
# Add any additional keyword arguments as attributes
for key, value in kwargs.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(key, value)
def add_openai_realtime_span_attributes(
span: "Span",
service_name: str,
model: str,
operation_name: str,
session_properties: Optional[Dict[str, Any]] = None,
transcript: Optional[str] = None,
is_input: Optional[bool] = None,
context_messages: Optional[str] = None,
function_calls: Optional[List[Dict]] = None,
tools: Optional[List[Dict]] = None,
tools_serialized: Optional[str] = None,
audio_data_size: Optional[int] = None,
**kwargs,
) -> None:
"""Add OpenAI Realtime specific attributes to a span.
Args:
span: The span to add attributes to
service_name: Name of the service
model: Model name/identifier
operation_name: Name of the operation (setup, transcription, response, etc.)
session_properties: Session configuration properties
transcript: Transcription text
is_input: Whether transcript is input (True) or output (False)
context_messages: JSON-serialized context messages
function_calls: Function calls being made
tools: Available tools/functions list
tools_serialized: JSON-serialized tools for detailed inspection
audio_data_size: Size of audio data in bytes
**kwargs: Additional attributes to add
"""
# Add standard attributes
span.set_attribute("gen_ai.system", "openai")
span.set_attribute("gen_ai.request.model", model)
span.set_attribute("gen_ai.operation.name", operation_name)
span.set_attribute("service.operation", operation_name)
# Add optional attributes
if transcript:
span.set_attribute("transcript", transcript)
if is_input is not None:
span.set_attribute("transcript.is_input", is_input)
if context_messages:
span.set_attribute("input", context_messages)
if audio_data_size is not None:
span.set_attribute("audio.data_size_bytes", audio_data_size)
if tools:
span.set_attribute("tools.count", len(tools))
span.set_attribute("tools.available", True)
# Add individual tool names for easier filtering
tool_names = []
for tool in tools:
if isinstance(tool, dict) and "name" in tool:
tool_names.append(tool["name"])
elif hasattr(tool, "name"):
tool_names.append(tool.name)
elif isinstance(tool, dict) and "function" in tool and "name" in tool["function"]:
tool_names.append(tool["function"]["name"])
if tool_names:
span.set_attribute("tools.names", ",".join(tool_names))
if tools_serialized:
span.set_attribute("tools.definitions", tools_serialized)
if function_calls:
span.set_attribute("function_calls.count", len(function_calls))
if function_calls:
call = function_calls[0]
if hasattr(call, "name"):
span.set_attribute("function_calls.first_name", call.name)
elif isinstance(call, dict) and "name" in call:
span.set_attribute("function_calls.first_name", call["name"])
# Add session properties if provided
if session_properties:
for key, value in session_properties.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(f"session.{key}", value)
elif key == "turn_detection" and value is not None:
if isinstance(value, bool):
span.set_attribute("session.turn_detection.enabled", value)
elif isinstance(value, dict):
span.set_attribute("session.turn_detection.enabled", True)
for td_key, td_value in value.items():
if isinstance(td_value, (str, int, float, bool)):
span.set_attribute(f"session.turn_detection.{td_key}", td_value)
# Add any additional keyword arguments as attributes
for key, value in kwargs.items():
if isinstance(value, (str, int, float, bool)):
span.set_attribute(key, value)

544
src/pipecat/utils/tracing/service_decorators.py
View File

@@ -24,7 +24,9 @@ if TYPE_CHECKING:
from opentelemetry import trace
from pipecat.utils.tracing.service_attributes import (
add_gemini_live_span_attributes,
add_llm_span_attributes,
add_openai_realtime_span_attributes,
add_stt_span_attributes,
add_tts_span_attributes,
)
@@ -152,9 +154,9 @@ def traced_tts(func: Optional[Callable] = None, *, name: Optional[str] = None) -
raise
finally:
# Update TTFB metric at the end
ttfb_ms = getattr(getattr(self, "_metrics", None), "ttfb_ms", None)
if ttfb_ms is not None:
span.set_attribute("metrics.ttfb_ms", ttfb_ms)
ttfb: Optional[float] = getattr(getattr(self, "_metrics", None), "ttfb", None)
if ttfb is not None:
span.set_attribute("metrics.ttfb", ttfb)
if is_async_generator:
@@ -238,7 +240,9 @@ def traced_stt(func: Optional[Callable] = None, *, name: Optional[str] = None) -
) as current_span:
try:
# Get TTFB metric if available
ttfb_ms = getattr(getattr(self, "_metrics", None), "ttfb_ms", None)
ttfb: Optional[float] = getattr(
getattr(self, "_metrics", None), "ttfb", None
)
# Use settings from the service if available
settings = getattr(self, "_settings", {})
@@ -252,7 +256,7 @@ def traced_stt(func: Optional[Callable] = None, *, name: Optional[str] = None) -
language=str(language) if language else None,
vad_enabled=getattr(self, "vad_enabled", False),
settings=settings,
ttfb_ms=ttfb_ms,
ttfb=ttfb,
)
# Call the original function
@@ -460,9 +464,11 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) -
self.start_llm_usage_metrics = original_start_llm_usage_metrics
# Update TTFB metric
ttfb_ms = getattr(getattr(self, "_metrics", None), "ttfb_ms", None)
if ttfb_ms is not None:
current_span.set_attribute("metrics.ttfb_ms", ttfb_ms)
ttfb: Optional[float] = getattr(
getattr(self, "_metrics", None), "ttfb", None
)
if ttfb is not None:
current_span.set_attribute("metrics.ttfb", ttfb)
except Exception as e:
logging.error(f"Error in LLM tracing (continuing without tracing): {e}")
# If tracing fails, fall back to the original function
@@ -473,3 +479,525 @@ def traced_llm(func: Optional[Callable] = None, *, name: Optional[str] = None) -
if func is not None:
return decorator(func)
return decorator
def traced_gemini_live(operation: str) -> Callable:
"""Traces Gemini Live service methods with operation-specific attributes.
This decorator automatically captures relevant information based on the operation type:
- llm_setup: Configuration, tools definitions, and system instructions
- llm_tool_call: Function call information
- llm_tool_result: Function execution results
- llm_response: Complete LLM response with usage and output
Args:
operation: The operation name (matches the event type being handled)
Returns:
Wrapped method with Gemini Live specific tracing.
"""
if not is_tracing_available():
return _noop_decorator
def decorator(func):
@functools.wraps(func)
async def wrapper(self, *args, **kwargs):
try:
if not is_tracing_available():
return await func(self, *args, **kwargs)
service_class_name = self.__class__.__name__
span_name = f"{operation}"
# Get the parent context - turn context if available, otherwise service context
turn_context = get_current_turn_context()
parent_context = turn_context or _get_parent_service_context(self)
# Create a new span as child of the turn span or service span
tracer = trace.get_tracer("pipecat")
with tracer.start_as_current_span(
span_name, context=parent_context
) as current_span:
try:
# Base service attributes
model_name = getattr(
self, "model_name", getattr(self, "_model_name", "unknown")
)
voice_id = getattr(self, "_voice_id", None)
language_code = getattr(self, "_language_code", None)
settings = getattr(self, "_settings", {})
# Get modalities if available
modalities = None
if hasattr(self, "_settings") and "modalities" in self._settings:
modality_obj = self._settings["modalities"]
if hasattr(modality_obj, "value"):
modalities = modality_obj.value
else:
modalities = str(modality_obj)
# Operation-specific attribute collection
operation_attrs = {}
if operation == "llm_setup":
# Capture detailed tool information
tools = getattr(self, "_tools", None)
if tools:
# Handle different tool formats
tools_list = []
tools_serialized = None
try:
if hasattr(tools, "standard_tools"):
# ToolsSchema object
tools_list = tools.standard_tools
# Serialize the tools for detailed inspection
tools_serialized = json.dumps(
[
{
"name": tool.name
if hasattr(tool, "name")
else tool.get("name", "unknown"),
"description": tool.description
if hasattr(tool, "description")
else tool.get("description", ""),
"properties": tool.properties
if hasattr(tool, "properties")
else tool.get("properties", {}),
"required": tool.required
if hasattr(tool, "required")
else tool.get("required", []),
}
for tool in tools_list
]
)
elif isinstance(tools, list):
# List of tool dictionaries or objects
tools_list = tools
tools_serialized = json.dumps(
[
{
"name": tool.get("name", "unknown")
if isinstance(tool, dict)
else getattr(tool, "name", "unknown"),
"description": tool.get("description", "")
if isinstance(tool, dict)
else getattr(tool, "description", ""),
"properties": tool.get("properties", {})
if isinstance(tool, dict)
else getattr(tool, "properties", {}),
"required": tool.get("required", [])
if isinstance(tool, dict)
else getattr(tool, "required", []),
}
for tool in tools_list
]
)
if tools_list:
operation_attrs["tools"] = tools_list
operation_attrs["tools_serialized"] = tools_serialized
except Exception as e:
logging.warning(f"Error serializing tools for tracing: {e}")
# Fallback to basic tool count
if tools_list:
operation_attrs["tools"] = tools_list
# Capture system instruction information
system_instruction = getattr(self, "_system_instruction", None)
if system_instruction:
operation_attrs["system_instruction"] = system_instruction[
:500
] # Truncate if very long
# Capture context system instructions if available
if hasattr(self, "_context") and self._context:
try:
context_system = self._context.extract_system_instructions()
if context_system:
operation_attrs["context_system_instruction"] = (
context_system[:500]
) # Truncate if very long
except Exception as e:
logging.warning(
f"Error extracting context system instructions: {e}"
)
elif operation == "llm_tool_call" and args:
# Extract tool call information
evt = args[0] if args else None
if evt and hasattr(evt, "toolCall") and evt.toolCall.functionCalls:
function_calls = evt.toolCall.functionCalls
if function_calls:
# Add information about the first function call
call = function_calls[0]
operation_attrs["tool.function_name"] = call.name
operation_attrs["tool.call_id"] = call.id
operation_attrs["tool.calls_count"] = len(function_calls)
# Add all function names being called
all_function_names = [c.name for c in function_calls]
operation_attrs["tool.all_function_names"] = ",".join(
all_function_names
)
# Add arguments for the first call (truncated if too long)
try:
args_str = json.dumps(call.args) if call.args else "{}"
if len(args_str) > 1000:
args_str = args_str[:1000] + "..."
operation_attrs["tool.arguments"] = args_str
except Exception:
operation_attrs["tool.arguments"] = str(call.args)[:1000]
elif operation == "llm_tool_result" and args:
# Extract tool result information
tool_result_message = args[0] if args else None
if tool_result_message and isinstance(tool_result_message, dict):
# Extract the tool call information
tool_call_id = tool_result_message.get("tool_call_id")
tool_call_name = tool_result_message.get("tool_call_name")
result_content = tool_result_message.get("content")
if tool_call_id:
operation_attrs["tool.call_id"] = tool_call_id
if tool_call_name:
operation_attrs["tool.function_name"] = tool_call_name
# Parse and capture the result
if result_content:
try:
result = json.loads(result_content)
# Serialize the result, truncating if too long
result_str = json.dumps(result)
if len(result_str) > 2000: # Larger limit for results
result_str = result_str[:2000] + "..."
operation_attrs["tool.result"] = result_str
# Add result status/success indicator if present
if isinstance(result, dict):
if "error" in result:
operation_attrs["tool.result_status"] = "error"
elif "success" in result:
operation_attrs["tool.result_status"] = "success"
else:
operation_attrs["tool.result_status"] = "completed"
except json.JSONDecodeError as e:
operation_attrs["tool.result"] = (
f"Invalid JSON: {str(result_content)[:500]}"
)
operation_attrs["tool.result_status"] = "parse_error"
except Exception as e:
operation_attrs["tool.result"] = (
f"Error processing result: {str(e)}"
)
operation_attrs["tool.result_status"] = "processing_error"
elif operation == "llm_response" and args:
# Extract usage and response metadata from turn complete event
evt = args[0] if args else None
if evt and hasattr(evt, "usageMetadata") and evt.usageMetadata:
usage = evt.usageMetadata
# Token usage - basic attributes for span visibility
if hasattr(usage, "promptTokenCount"):
operation_attrs["tokens.prompt"] = usage.promptTokenCount or 0
if hasattr(usage, "responseTokenCount"):
operation_attrs["tokens.completion"] = (
usage.responseTokenCount or 0
)
if hasattr(usage, "totalTokenCount"):
operation_attrs["tokens.total"] = usage.totalTokenCount or 0
# Get output text and modality from service state
text = getattr(self, "_bot_text_buffer", "")
audio_text = getattr(self, "_llm_output_buffer", "")
if text:
# TEXT modality
operation_attrs["output"] = text
operation_attrs["output_modality"] = "TEXT"
elif audio_text:
# AUDIO modality
operation_attrs["output"] = audio_text
operation_attrs["output_modality"] = "AUDIO"
# Add turn completion status
if (
evt
and hasattr(evt, "serverContent")
and evt.serverContent.turnComplete
):
operation_attrs["turn_complete"] = True
# Add all attributes to the span
add_gemini_live_span_attributes(
span=current_span,
service_name=service_class_name,
model=model_name,
operation_name=operation,
voice_id=voice_id,
language=language_code,
modalities=modalities,
settings=settings,
**operation_attrs,
)
# For llm_response operation, also handle token usage metrics
if operation == "llm_response" and hasattr(self, "start_llm_usage_metrics"):
evt = args[0] if args else None
if evt and hasattr(evt, "usageMetadata") and evt.usageMetadata:
usage = evt.usageMetadata
# Create LLMTokenUsage object
from pipecat.metrics.metrics import LLMTokenUsage
tokens = LLMTokenUsage(
prompt_tokens=usage.promptTokenCount or 0,
completion_tokens=usage.responseTokenCount or 0,
total_tokens=usage.totalTokenCount or 0,
)
_add_token_usage_to_span(current_span, tokens)
# Capture TTFB metric if available
ttfb = getattr(getattr(self, "_metrics", None), "ttfb", None)
if ttfb is not None:
current_span.set_attribute("metrics.ttfb", ttfb)
# Run the original function
result = await func(self, *args, **kwargs)
return result
except Exception as e:
current_span.record_exception(e)
current_span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
raise
except Exception as e:
logging.error(f"Error in Gemini Live tracing (continuing without tracing): {e}")
# If tracing fails, fall back to the original function
return await func(self, *args, **kwargs)
return wrapper
return decorator
def traced_openai_realtime(operation: str) -> Callable:
"""Traces OpenAI Realtime service methods with operation-specific attributes.
This decorator automatically captures relevant information based on the operation type:
- llm_setup: Session configuration and tools
- llm_request: Context and input messages
- llm_response: Usage metadata, output, and function calls
Args:
operation: The operation name (matches the event type being handled)
Returns:
Wrapped method with OpenAI Realtime specific tracing.
"""
if not is_tracing_available():
return _noop_decorator
def decorator(func):
@functools.wraps(func)
async def wrapper(self, *args, **kwargs):
try:
if not is_tracing_available():
return await func(self, *args, **kwargs)
service_class_name = self.__class__.__name__
span_name = f"{operation}"
# Get the parent context - turn context if available, otherwise service context
turn_context = get_current_turn_context()
parent_context = turn_context or _get_parent_service_context(self)
# Create a new span as child of the turn span or service span
tracer = trace.get_tracer("pipecat")
with tracer.start_as_current_span(
span_name, context=parent_context
) as current_span:
try:
# Base service attributes
model_name = getattr(
self, "model_name", getattr(self, "_model_name", "unknown")
)
# Operation-specific attribute collection
operation_attrs = {}
if operation == "llm_setup":
# Capture session properties and tools
session_properties = getattr(self, "_session_properties", None)
if session_properties:
try:
# Convert to dict for easier processing
if hasattr(session_properties, "model_dump"):
props_dict = session_properties.model_dump()
elif hasattr(session_properties, "__dict__"):
props_dict = session_properties.__dict__
else:
props_dict = {}
operation_attrs["session_properties"] = props_dict
# Extract tools if available
tools = props_dict.get("tools")
if tools:
operation_attrs["tools"] = tools
try:
operation_attrs["tools_serialized"] = json.dumps(tools)
except Exception as e:
logging.warning(f"Error serializing OpenAI tools: {e}")
# Extract instructions
instructions = props_dict.get("instructions")
if instructions:
operation_attrs["instructions"] = instructions[:500]
except Exception as e:
logging.warning(f"Error processing session properties: {e}")
# Also check context for tools
if hasattr(self, "_context") and self._context:
try:
context_tools = getattr(self._context, "tools", None)
if context_tools and not operation_attrs.get("tools"):
operation_attrs["tools"] = context_tools
operation_attrs["tools_serialized"] = json.dumps(
context_tools
)
except Exception as e:
logging.warning(f"Error extracting context tools: {e}")
elif operation == "llm_request":
# Capture context messages being sent
if hasattr(self, "_context") and self._context:
try:
messages = self._context.get_messages_for_logging()
if messages:
operation_attrs["context_messages"] = json.dumps(messages)
except Exception as e:
logging.warning(f"Error getting context messages: {e}")
elif operation == "llm_response" and args:
# Extract usage and response metadata
evt = args[0] if args else None
if evt and hasattr(evt, "response"):
response = evt.response
# Token usage - basic attributes for span visibility
if hasattr(response, "usage"):
usage = response.usage
if hasattr(usage, "input_tokens"):
operation_attrs["tokens.prompt"] = usage.input_tokens
if hasattr(usage, "output_tokens"):
operation_attrs["tokens.completion"] = usage.output_tokens
if hasattr(usage, "total_tokens"):
operation_attrs["tokens.total"] = usage.total_tokens
# Response status and metadata
if hasattr(response, "status"):
operation_attrs["response.status"] = response.status
if hasattr(response, "id"):
operation_attrs["response.id"] = response.id
# Output items and extract transcript for output field
if hasattr(response, "output") and response.output:
operation_attrs["response.output_items"] = len(response.output)
# Extract assistant transcript and function calls
assistant_transcript = ""
function_calls = []
for item in response.output:
if (
hasattr(item, "content")
and item.content
and hasattr(item, "role")
and item.role == "assistant"
):
for content in item.content:
if (
hasattr(content, "transcript")
and content.transcript
):
assistant_transcript += content.transcript + " "
elif hasattr(item, "type") and item.type == "function_call":
function_call_info = {
"name": getattr(item, "name", "unknown"),
"call_id": getattr(item, "call_id", "unknown"),
}
if hasattr(item, "arguments"):
args_str = item.arguments
if len(args_str) > 500:
args_str = args_str[:500] + "..."
function_call_info["arguments"] = args_str
function_calls.append(function_call_info)
if assistant_transcript.strip():
operation_attrs["output"] = assistant_transcript.strip()
if function_calls:
operation_attrs["function_calls"] = function_calls
operation_attrs["function_calls.count"] = len(
function_calls
)
all_names = [call["name"] for call in function_calls]
operation_attrs["function_calls.all_names"] = ",".join(
all_names
)
# Add all attributes to the span
add_openai_realtime_span_attributes(
span=current_span,
service_name=service_class_name,
model=model_name,
operation_name=operation,
**operation_attrs,
)
# For llm_response operation, also handle token usage metrics
if operation == "llm_response" and hasattr(self, "start_llm_usage_metrics"):
evt = args[0] if args else None
if evt and hasattr(evt, "response") and hasattr(evt.response, "usage"):
usage = evt.response.usage
# Create LLMTokenUsage object
from pipecat.metrics.metrics import LLMTokenUsage
tokens = LLMTokenUsage(
prompt_tokens=getattr(usage, "input_tokens", 0),
completion_tokens=getattr(usage, "output_tokens", 0),
total_tokens=getattr(usage, "total_tokens", 0),
)
_add_token_usage_to_span(current_span, tokens)
# Capture TTFB metric if available
ttfb = getattr(getattr(self, "_metrics", None), "ttfb", None)
if ttfb is not None:
current_span.set_attribute("metrics.ttfb", ttfb)
# Run the original function
result = await func(self, *args, **kwargs)
return result
except Exception as e:
current_span.record_exception(e)
current_span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
raise
except Exception as e:
logging.error(f"Error in OpenAI Realtime tracing (continuing without tracing): {e}")
# If tracing fails, fall back to the original function
return await func(self, *args, **kwargs)
return wrapper
return decorator

10
src/pipecat/utils/tracing/turn_trace_observer.py
View File

@@ -35,7 +35,11 @@ class TurnTraceObserver(BaseObserver):
"""
def __init__(
self, turn_tracker: TurnTrackingObserver, conversation_id: Optional[str] = None, **kwargs
self,
turn_tracker: TurnTrackingObserver,
conversation_id: Optional[str] = None,
additional_span_attributes: Optional[dict] = None,
**kwargs,
):
super().__init__(**kwargs)
self._turn_tracker = turn_tracker
@@ -47,6 +51,7 @@ class TurnTraceObserver(BaseObserver):
# Conversation tracking properties
self._conversation_span: Optional["Span"] = None
self._conversation_id = conversation_id
self._additional_span_attributes = additional_span_attributes or {}
if turn_tracker:
@@ -89,6 +94,9 @@ class TurnTraceObserver(BaseObserver):
# Set span attributes
self._conversation_span.set_attribute("conversation.id", conversation_id)
self._conversation_span.set_attribute("conversation.type", "voice")
# Set custom otel attributes if provided
for k, v in (self._additional_span_attributes or {}).items():
self._conversation_span.set_attribute(k, v)
# Update the conversation context provider
context_provider.set_current_conversation_context(