wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into kyle/fix-ultravox-performance

Browse Source
This commit is contained in:
Kyle Gani
2025-04-25 13:16:48 +02:00
committed by GitHub
parent 49fbcc86ac d20c3307b9
commit 5489ac5a73
415 changed files with 33852 additions and 13833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/pipecat/adapters/schemas/function_schema.py
View File

@@ -8,16 +8,22 @@ from typing import Any, Dict, List
class FunctionSchema:
"""Standardized function schema representation for tool definition.
Provides a structured way to define function tools used with AI models like OpenAI.
This schema defines the function's name, description, parameter properties, and
required parameters, following specifications required by AI service providers.
Args:
name: Name of the function to be called.
description: Description of what the function does.
properties: Dictionary defining parameter types, descriptions, and constraints.
required: List of property names that are required parameters.
"""
def __init__(
self, name: str, description: str, properties: Dict[str, Any], required: List[str]
) -> None:
"""Standardized function schema representation.
:param name: Name of the function.
:param description: Description of the function.
:param properties: Dictionary defining properties types and descriptions.
:param required: List of required parameters.
"""
self._name = name
self._description = description
self._properties = properties
@@ -26,7 +32,8 @@ class FunctionSchema:
def to_default_dict(self) -> Dict[str, Any]:
"""Converts the function schema to a dictionary.
:return: Dictionary representation of the function schema.
Returns:
Dictionary representation of the function schema.
"""
return {
"name": self._name,
@@ -40,16 +47,36 @@ class FunctionSchema:
@property
def name(self) -> str:
"""Get the function name.
Returns:
The function name.
"""
return self._name
@property
def description(self) -> str:
"""Get the function description.
Returns:
The function description.
"""
return self._description
@property
def properties(self) -> Dict[str, Any]:
"""Get the function properties.
Returns:
Dictionary of parameter specifications.
"""
return self._properties
@property
def required(self) -> List[str]:
"""Get the required parameters.
Returns:
List of required parameter names.
"""
return self._required

4
src/pipecat/audio/mixers/soundfile_mixer.py
View File

@@ -38,9 +38,11 @@ class SoundfileMixer(BaseAudioMixer):
def __init__(
self,
*,
sound_files: Mapping[str, str],
default_sound: str,
volume: float = 0.4,
mixing: bool = True,
loop: bool = True,
**kwargs,
):
@@ -52,7 +54,7 @@ class SoundfileMixer(BaseAudioMixer):
self._sound_pos = 0
self._sounds: Dict[str, Any] = {}
self._current_sound = default_sound
self._mixing = True
self._mixing = mixing
self._loop = loop
async def start(self, sample_rate: int):

0
src/pipecat/audio/turn/__init__.py Normal file
View File

80
src/pipecat/audio/turn/base_turn_analyzer.py Normal file
View File

@@ -0,0 +1,80 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from abc import ABC, abstractmethod
from enum import Enum
from typing import Optional, Tuple
from pipecat.metrics.metrics import MetricsData
class EndOfTurnState(Enum):
COMPLETE = 1
INCOMPLETE = 2
class BaseTurnAnalyzer(ABC):
"""Abstract base class for analyzing user end of turn.
This class inherits from BaseObject to leverage its event handling system
while still defining an abstract interface through abstract methods.
"""
def __init__(self, *, sample_rate: Optional[int] = None):
self._init_sample_rate = sample_rate
self._sample_rate = 0
@property
def sample_rate(self) -> int:
"""Returns the current sample rate.
Returns:
int: The effective sample rate for audio processing.
"""
return self._sample_rate
def set_sample_rate(self, sample_rate: int):
"""Sets the sample rate for audio processing.
If the initial sample rate was provided, it will use that; otherwise, it sets to
the provided sample rate.
Args:
sample_rate (int): The sample rate to set.
"""
self._sample_rate = self._init_sample_rate or sample_rate
@property
@abstractmethod
def speech_triggered(self) -> bool:
"""Determines if speech has been detected.
Returns:
bool: True if speech is triggered, otherwise False.
"""
pass
@abstractmethod
def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
"""Appends audio data for analysis.
Args:
buffer (bytes): The audio data to append.
is_speech (bool): Indicates whether the appended audio is speech or not.
Returns:
EndOfTurnState: The resulting state after appending the audio.
"""
pass
@abstractmethod
async def analyze_end_of_turn(self) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
"""Analyzes if an end of turn has occurred based on the audio input.
Returns:
EndOfTurnState: The result of the end of turn analysis.
"""
pass

0
src/pipecat/audio/turn/smart_turn/__init__.py Normal file
View File

198
src/pipecat/audio/turn/smart_turn/base_smart_turn.py Normal file
View File

@@ -0,0 +1,198 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import time
from abc import abstractmethod
from typing import Any, Dict, Optional, Tuple
import numpy as np
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, EndOfTurnState
from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData
# Default timing parameters
STOP_SECS = 3
PRE_SPEECH_MS = 0
MAX_DURATION_SECONDS = 8 # Max allowed segment duration
USE_ONLY_LAST_VAD_SEGMENT = True
class SmartTurnParams(BaseModel):
stop_secs: float = STOP_SECS
pre_speech_ms: float = PRE_SPEECH_MS
max_duration_secs: float = MAX_DURATION_SECONDS
# not exposing this for now yet until the model can handle it.
# use_only_last_vad_segment: bool = USE_ONLY_LAST_VAD_SEGMENT
class SmartTurnTimeoutException(Exception):
pass
class BaseSmartTurn(BaseTurnAnalyzer):
def __init__(
self, *, sample_rate: Optional[int] = None, params: SmartTurnParams = SmartTurnParams()
):
super().__init__(sample_rate=sample_rate)
self._params = params
# Configuration
self._stop_ms = self._params.stop_secs * 1000 # silence threshold in ms
# Inference state
self._audio_buffer = []
self._speech_triggered = False
self._silence_ms = 0
self._speech_start_time = 0
@property
def speech_triggered(self) -> bool:
return self._speech_triggered
def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
# Convert raw audio to float32 format and append to the buffer
audio_int16 = np.frombuffer(buffer, dtype=np.int16)
audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
self._audio_buffer.append((time.time(), audio_float32))
state = EndOfTurnState.INCOMPLETE
if is_speech:
# Reset silence tracking on speech
self._silence_ms = 0
self._speech_triggered = True
if self._speech_start_time == 0:
self._speech_start_time = time.time()
else:
if self._speech_triggered:
chunk_duration_ms = len(audio_int16) / (self._sample_rate / 1000)
self._silence_ms += chunk_duration_ms
# If silence exceeds threshold, mark end of turn
if self._silence_ms >= self._stop_ms:
logger.debug(
f"End of Turn complete due to stop_secs. Silence in ms: {self._silence_ms}"
)
state = EndOfTurnState.COMPLETE
self._clear(state)
else:
# Trim buffer to prevent unbounded growth before speech
max_buffer_time = (
(self._params.pre_speech_ms / 1000)
+ self._params.stop_secs
+ self._params.max_duration_secs
)
while (
self._audio_buffer and self._audio_buffer[0][0] < time.time() - max_buffer_time
):
self._audio_buffer.pop(0)
return state
async def analyze_end_of_turn(self) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
state, result = await self._process_speech_segment(self._audio_buffer)
if state == EndOfTurnState.COMPLETE or USE_ONLY_LAST_VAD_SEGMENT:
self._clear(state)
logger.debug(f"End of Turn result: {state}")
return state, result
def _clear(self, turn_state: EndOfTurnState):
# If the state is still incomplete, keep the _speech_triggered as True
self._speech_triggered = turn_state == EndOfTurnState.INCOMPLETE
self._audio_buffer = []
self._speech_start_time = 0
self._silence_ms = 0
async def _process_speech_segment(
self, audio_buffer
) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
state = EndOfTurnState.INCOMPLETE
if not audio_buffer:
return state, None
# Extract recent audio segment for prediction
start_time = self._speech_start_time - (self._params.pre_speech_ms / 1000)
start_index = 0
for i, (t, _) in enumerate(audio_buffer):
if t >= start_time:
start_index = i
break
end_index = len(audio_buffer) - 1
# Extract the audio segment
segment_audio_chunks = [chunk for _, chunk in audio_buffer[start_index : end_index + 1]]
segment_audio = np.concatenate(segment_audio_chunks)
# Limit maximum duration
max_samples = int(self._params.max_duration_secs * self.sample_rate)
if len(segment_audio) > max_samples:
# slices the array to keep the last max_samples samples, discarding the earlier part.
segment_audio = segment_audio[-max_samples:]
result_data = None
if len(segment_audio) > 0:
start_time = time.perf_counter()
try:
result = await self._predict_endpoint(segment_audio)
state = (
EndOfTurnState.COMPLETE
if result["prediction"] == 1
else EndOfTurnState.INCOMPLETE
)
end_time = time.perf_counter()
# Calculate processing time
e2e_processing_time_ms = (end_time - start_time) * 1000
# Extract metrics from the nested structure
metrics = result.get("metrics", {})
inference_time = metrics.get("inference_time", 0)
total_time = metrics.get("total_time", 0)
# Prepare the result data
result_data = SmartTurnMetricsData(
processor="BaseSmartTurn",
is_complete=result["prediction"] == 1,
probability=result["probability"],
inference_time_ms=inference_time * 1000,
server_total_time_ms=total_time * 1000,
e2e_processing_time_ms=e2e_processing_time_ms,
)
logger.trace(
f"Prediction: {'Complete' if result_data.is_complete else 'Incomplete'}"
)
logger.trace(f"Probability of complete: {result_data.probability:.4f}")
logger.trace(f"Inference time: {result_data.inference_time_ms:.2f}ms")
logger.trace(f"Server total time: {result_data.server_total_time_ms:.2f}ms")
logger.trace(f"E2E processing time: {result_data.e2e_processing_time_ms:.2f}ms")
except SmartTurnTimeoutException:
logger.debug(
f"End of Turn complete due to stop_secs. Silence in ms: {self._silence_ms}"
)
state = EndOfTurnState.COMPLETE
else:
logger.trace(f"params: {self._params}, stop_ms: {self._stop_ms}")
logger.trace("Captured empty audio segment, skipping prediction.")
return state, result_data
@abstractmethod
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
"""Abstract method to predict if a turn has ended based on audio.
Args:
audio_array: Float32 numpy array of audio samples at 16kHz.
Returns:
Dictionary with:
- prediction: 1 if turn is complete, else 0
- probability: Confidence of the prediction
"""
pass

26
src/pipecat/audio/turn/smart_turn/fal_smart_turn.py Normal file
View File

@@ -0,0 +1,26 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Optional
import aiohttp
from pipecat.audio.turn.smart_turn.http_smart_turn import HttpSmartTurnAnalyzer
class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
def __init__(
self,
*,
aiohttp_session: aiohttp.ClientSession,
url: str = "https://fal.run/fal-ai/smart-turn/raw",
api_key: Optional[str] = None,
**kwargs,
):
headers = {}
if api_key:
headers = {"Authorization": f"Key {api_key}"}
super().__init__(url=url, aiohttp_session=aiohttp_session, headers=headers, **kwargs)

80
src/pipecat/audio/turn/smart_turn/http_smart_turn.py Normal file
View File

@@ -0,0 +1,80 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import io
from typing import Any, Dict
import aiohttp
import numpy as np
from loguru import logger
from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn, SmartTurnTimeoutException
class HttpSmartTurnAnalyzer(BaseSmartTurn):
def __init__(
self,
*,
url: str,
aiohttp_session: aiohttp.ClientSession,
headers: Dict[str, str] = {},
**kwargs,
):
super().__init__(**kwargs)
self._url = url
self._headers = headers
self._aiohttp_session = aiohttp_session
def _serialize_array(self, audio_array: np.ndarray) -> bytes:
logger.trace("Serializing NumPy array to bytes...")
buffer = io.BytesIO()
np.save(buffer, audio_array)
serialized_bytes = buffer.getvalue()
logger.trace(f"Serialized size: {len(serialized_bytes)} bytes")
return serialized_bytes
async def _send_raw_request(self, data_bytes: bytes) -> Dict[str, Any]:
headers = {"Content-Type": "application/octet-stream"}
headers.update(self._headers)
logger.trace(f"Sending {len(data_bytes)} bytes as raw body to {self._url}...")
try:
timeout = aiohttp.ClientTimeout(total=self._params.stop_secs)
async with self._aiohttp_session.post(
self._url, data=data_bytes, headers=headers, timeout=timeout
) as response:
logger.trace("\n--- Response ---")
logger.trace(f"Status Code: {response.status}")
if response.status == 200:
try:
json_data = await response.json()
logger.trace("Response JSON:")
logger.trace(json_data)
return json_data
except aiohttp.ContentTypeError:
# Non-JSON response
text = await response.text()
logger.trace("Response Content (non-JSON):")
logger.trace(text)
raise Exception(f"Non-JSON response: {text}")
else:
error_text = await response.text()
logger.trace("Response Content (Error):")
logger.trace(error_text)
response.raise_for_status()
except asyncio.TimeoutError:
logger.error(f"Request timed out after {self._params.stop_secs} seconds")
raise SmartTurnTimeoutException(f"Request exceeded {self._params.stop_secs} seconds.")
except aiohttp.ClientError as e:
logger.error(f"Failed to send raw request to Daily Smart Turn: {e}")
raise Exception("Failed to send raw request to Daily Smart Turn.")
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
serialized_array = self._serialize_array(audio_array)
return await self._send_raw_request(serialized_array)

64
src/pipecat/audio/turn/smart_turn/local_coreml_smart_turn.py Normal file
View File

@@ -0,0 +1,64 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, Dict
import numpy as np
from loguru import logger
from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn
try:
import coremltools as ct
import torch
from transformers import AutoFeatureExtractor
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`."
)
raise Exception(f"Missing module: {e}")
class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
def __init__(self, *, smart_turn_model_path: str, **kwargs):
super().__init__(**kwargs)
if not smart_turn_model_path:
logger.error("smart_turn_model_path is not set.")
raise Exception("smart_turn_model_path must be provided.")
core_ml_model_path = f"{smart_turn_model_path}/coreml/smart_turn_classifier.mlpackage"
logger.debug("Loading Local Smart Turn model...")
# Only load the processor, not the torch model
self._turn_processor = AutoFeatureExtractor.from_pretrained(smart_turn_model_path)
self._turn_model = ct.models.MLModel(core_ml_model_path)
logger.debug("Loaded Local Smart Turn")
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
inputs = self._turn_processor(
audio_array,
sampling_rate=16000,
padding="max_length",
truncation=True,
max_length=800, # Maximum length as specified in training
return_attention_mask=True,
return_tensors="pt",
)
output = self._turn_model.predict(dict(inputs))
logits = output["logits"] # Core ML returns numpy array
logits_tensor = torch.tensor(logits)
probabilities = torch.nn.functional.softmax(logits_tensor, dim=1)
completion_prob = probabilities[0, 1].item() # Probability of class 1 (Complete)
prediction = 1 if completion_prob > 0.5 else 0
return {
"prediction": prediction,
"probability": completion_prob,
}

38
src/pipecat/frames/frames.py
View File

@@ -377,25 +377,6 @@ class LLMEnablePromptCachingFrame(DataFrame):
enable: bool
@dataclass
class FunctionCallResultProperties:
"""Properties for a function call result frame."""
run_llm: Optional[bool] = None
on_context_updated: Optional[Callable[[], Awaitable[None]]] = None
@dataclass
class FunctionCallResultFrame(DataFrame):
"""A frame containing the result of an LLM function (tool) call."""
function_name: str
tool_call_id: str
arguments: Any
result: Any
properties: Optional[FunctionCallResultProperties] = None
@dataclass
class TTSSpeakFrame(DataFrame):
"""A frame that contains a text that should be spoken by the TTS in the
@@ -652,6 +633,25 @@ class FunctionCallCancelFrame(SystemFrame):
tool_call_id: str
@dataclass
class FunctionCallResultProperties:
"""Properties for a function call result frame."""
run_llm: Optional[bool] = None
on_context_updated: Optional[Callable[[], Awaitable[None]]] = None
@dataclass
class FunctionCallResultFrame(SystemFrame):
"""A frame containing the result of an LLM function (tool) call."""
function_name: str
tool_call_id: str
arguments: Any
result: Any
properties: Optional[FunctionCallResultProperties] = None
@dataclass
class STTMuteFrame(SystemFrame):
"""System frame to mute/unmute the STT service."""

10
src/pipecat/metrics/metrics.py
View File

@@ -30,3 +30,13 @@ class LLMUsageMetricsData(MetricsData):
class TTSUsageMetricsData(MetricsData):
value: int
class SmartTurnMetricsData(MetricsData):
"""Metrics data for smart turn predictions."""
is_complete: bool
probability: float
inference_time_ms: float
server_total_time_ms: float
e2e_processing_time_ms: float

2
src/pipecat/observers/loggers/llm_log_observer.py
View File

@@ -18,7 +18,7 @@ from pipecat.frames.frames import (
from pipecat.observers.base_observer import BaseObserver
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.ai_services import LLMService
from pipecat.services.llm_service import LLMService
class LLMLogObserver(BaseObserver):

2
src/pipecat/observers/loggers/transcription_log_observer.py
View File

@@ -13,7 +13,7 @@ from pipecat.frames.frames import (
)
from pipecat.observers.base_observer import BaseObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.ai_services import STTService
from pipecat.services.stt_service import STTService
class TranscriptionLogObserver(BaseObserver):

79
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -6,6 +6,7 @@
import asyncio
from abc import abstractmethod
from dataclasses import dataclass
from typing import Dict, List, Literal, Set
from loguru import logger
@@ -46,6 +47,16 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.time import time_now_iso8601
@dataclass
class LLMUserAggregatorParams:
aggregation_timeout: float = 1.0
@dataclass
class LLMAssistantAggregatorParams:
expect_stripped_words: bool = True
class LLMFullResponseAggregator(FrameProcessor):
"""This is an LLM aggregator that aggregates a full LLM completion. It
aggregates LLM text frames (tokens) received between
@@ -149,7 +160,8 @@ class BaseLLMResponseAggregator(FrameProcessor):
@abstractmethod
def reset(self):
"""Reset the internals of this aggregator. This should not modify the
internal messages."""
internal messages.
"""
pass
@abstractmethod
@@ -229,11 +241,23 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
def __init__(
self,
context: OpenAILLMContext,
aggregation_timeout: float = 1.0,
*,
params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
**kwargs,
):
super().__init__(context=context, role="user", **kwargs)
self._aggregation_timeout = aggregation_timeout
self._params = params
if "aggregation_timeout" in kwargs:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'aggregation_timeout' is deprecated, use 'params' instead.",
DeprecationWarning,
)
self._params.aggregation_timeout = kwargs["aggregation_timeout"]
self._seen_interim_results = False
self._user_speaking = False
@@ -356,7 +380,9 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
async def _aggregation_task_handler(self):
while True:
try:
await asyncio.wait_for(self._aggregation_event.wait(), self._aggregation_timeout)
await asyncio.wait_for(
self._aggregation_event.wait(), self._params.aggregation_timeout
)
await self._maybe_push_bot_interruption()
except asyncio.TimeoutError:
if not self._user_speaking:
@@ -393,9 +419,27 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
"""
def __init__(self, context: OpenAILLMContext, *, expect_stripped_words: bool = True, **kwargs):
def __init__(
self,
context: OpenAILLMContext,
*,
params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
**kwargs,
):
super().__init__(context=context, role="assistant", **kwargs)
self._expect_stripped_words = expect_stripped_words
self._params = params
if "expect_stripped_words" in kwargs:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'expect_stripped_words' is deprecated, use 'params' instead.",
DeprecationWarning,
)
self._params.expect_stripped_words = kwargs["expect_stripped_words"]
self._started = 0
self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {}
@@ -446,6 +490,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
await self._handle_user_image_frame(frame)
elif isinstance(frame, BotStoppedSpeakingFrame):
await self.push_aggregation()
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
@@ -556,7 +601,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
if not self._started:
return
if self._expect_stripped_words:
if self._params.expect_stripped_words:
self._aggregation += f" {frame.text}" if self._aggregation else frame.text
else:
self._aggregation += frame.text
@@ -570,8 +615,14 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
class LLMUserResponseAggregator(LLMUserContextAggregator):
def __init__(self, messages: List[dict] = [], **kwargs):
super().__init__(context=OpenAILLMContext(messages), **kwargs)
def __init__(
self,
messages: List[dict] = [],
*,
params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
**kwargs,
):
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)
async def push_aggregation(self):
if len(self._aggregation) > 0:
@@ -586,8 +637,14 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
def __init__(self, messages: List[dict] = [], **kwargs):
super().__init__(context=OpenAILLMContext(messages), **kwargs)
def __init__(
self,
messages: List[dict] = [],
*,
params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
**kwargs,
):
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)
async def push_aggregation(self):
if len(self._aggregation) > 0:

65
src/pipecat/processors/consumer_processor.py Normal file
View File

@@ -0,0 +1,65 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from typing import Awaitable, Callable, Optional
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.producer_processor import ProducerProcessor, identity_transformer
class ConsumerProcessor(FrameProcessor):
"""This class passes-through frames and also consumes frames from a
producer's queue. When a frame from a producer queue is received it will be
pushed to the specified direction. The frames can be transformed into a
different type of frame before being pushed.
"""
def __init__(
self,
*,
producer: ProducerProcessor,
transformer: Callable[[Frame], Awaitable[Frame]] = identity_transformer,
direction: FrameDirection = FrameDirection.DOWNSTREAM,
**kwargs,
):
super().__init__(**kwargs)
self._transformer = transformer
self._direction = direction
self._queue: asyncio.Queue = producer.add_consumer()
self._consumer_task: Optional[asyncio.Task] = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
await self._start(frame)
elif isinstance(frame, EndFrame):
await self._stop(frame)
elif isinstance(frame, CancelFrame):
await self._cancel(frame)
await self.push_frame(frame, direction)
async def _start(self, _: StartFrame):
if not self._consumer_task:
self._consumer_task = self.create_task(self._consumer_task_handler())
async def _stop(self, _: EndFrame):
if self._consumer_task:
await self.cancel_task(self._consumer_task)
async def _cancel(self, _: CancelFrame):
if self._consumer_task:
await self.cancel_task(self._consumer_task)
async def _consumer_task_handler(self):
while True:
frame = await self._queue.get()
new_frame = await self._transformer(frame)
await self.push_frame(new_frame, self._direction)

2
src/pipecat/processors/filters/stt_mute_filter.py
View File

@@ -108,7 +108,7 @@ class STTMuteFilter(FrameProcessor):
async def _handle_mute_state(self, should_mute: bool):
"""Handles both STT muting and interruption control."""
if should_mute != self.is_muted:
logger.debug(f"STT {'muting' if should_mute else 'unmuting'}")
logger.debug(f"STTMuteFilter {'muting' if should_mute else 'unmuting'}")
self._is_muted = should_mute
await self.push_frame(STTMuteFrame(mute=should_mute))

73
src/pipecat/processors/producer_processor.py Normal file
View File

@@ -0,0 +1,73 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from typing import Awaitable, Callable, List
from pipecat.frames.frames import Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
async def identity_transformer(frame: Frame):
return frame
class ProducerProcessor(FrameProcessor):
"""This class optionally passes-through received frames and decides if those
frames should be sent to consumers based on a user-defined filter. The
frames can be transformed into a different type of frame before being
sending them to the consumers. More than one consumer can be added.
"""
def __init__(
self,
*,
filter: Callable[[Frame], Awaitable[bool]],
transformer: Callable[[Frame], Awaitable[Frame]] = identity_transformer,
passthrough: bool = True,
):
super().__init__()
self._filter = filter
self._transformer = transformer
self._passthrough = passthrough
self._consumers: List[asyncio.Queue] = []
def add_consumer(self):
"""
Adds a new consumer and returns its associated queue.
Returns:
asyncio.Queue: The queue for the newly added consumer.
"""
queue = asyncio.Queue()
self._consumers.append(queue)
return queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""
Processes an incoming frame and determines whether to produce it as a ProducerItem.
If the frame meets the produce criteria, it will be added to the consumer queues.
If passthrough is enabled, the frame will also be sent to consumers.
Args:
frame (Frame): The frame to process.
direction (FrameDirection): The direction of the frame.
"""
await super().process_frame(frame, direction)
if await self._filter(frame):
await self._produce(frame)
if self._passthrough:
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
async def _produce(self, frame: Frame):
for consumer in self._consumers:
new_frame = await self._transformer(frame)
await consumer.put(new_frame)

24
src/pipecat/processors/transcript_processor.py
View File

@@ -175,22 +175,28 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
"""
await super().process_frame(frame, direction)
if isinstance(frame, TTSTextFrame):
if isinstance(frame, (StartInterruptionFrame, CancelFrame)):
# Push frame first otherwise our emitted transcription update frame
# might get cleaned up.
await self.push_frame(frame, direction)
# Emit accumulated text with interruptions
await self._emit_aggregated_text()
elif isinstance(frame, TTSTextFrame):
# Start timestamp on first text part
if not self._aggregation_start_time:
self._aggregation_start_time = time_now_iso8601()
self._current_text_parts.append(frame.text)
elif isinstance(frame, (BotStoppedSpeakingFrame, StartInterruptionFrame, CancelFrame)):
# Emit accumulated text when bot finishes speaking or is interrupted
# Push frame.
await self.push_frame(frame, direction)
elif isinstance(frame, (BotStoppedSpeakingFrame, EndFrame)):
# Emit accumulated text when bot finishes speaking or pipeline ends.
await self._emit_aggregated_text()
elif isinstance(frame, EndFrame):
# Emit any remaining text when pipeline ends
await self._emit_aggregated_text()
await self.push_frame(frame, direction)
# Push frame.
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
class TranscriptProcessor:

7
src/pipecat/processors/user_idle_processor.py
View File

@@ -127,9 +127,10 @@ class UserIdleProcessor(FrameProcessor):
# Check for end frames before processing
if isinstance(frame, (EndFrame, CancelFrame)):
await self.push_frame(frame, direction) # Push the frame down the pipeline
if self._idle_task:
await self._stop() # Stop the idle task, if it exists
# Stop the idle task, if it exists
await self._stop()
# Push the frame down the pipeline
await self.push_frame(frame, direction)
return
await self.push_frame(frame, direction)

0
src/pipecat/py.typed Normal file
View File

145
src/pipecat/serializers/telnyx.py
View File

@@ -8,6 +8,8 @@ import base64
import json
from typing import Optional
import aiohttp
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import (
@@ -19,6 +21,8 @@ from pipecat.audio.utils import (
)
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
InputDTMFFrame,
@@ -30,38 +34,120 @@ from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializer
class TelnyxFrameSerializer(FrameSerializer):
"""Serializer for Telnyx WebSocket protocol.
This serializer handles converting between Pipecat frames and Telnyx's WebSocket
media streams protocol. It supports audio conversion, DTMF events, and automatic
call termination.
When auto_hang_up is enabled (default), the serializer will automatically terminate
the Telnyx call when an EndFrame or CancelFrame is processed, but requires Telnyx
credentials to be provided.
Attributes:
_stream_id: The Telnyx Stream ID.
_call_control_id: The associated Telnyx Call Control ID.
_api_key: Telnyx API key for API access.
_params: Configuration parameters.
_telnyx_sample_rate: Sample rate used by Telnyx (typically 8kHz).
_sample_rate: Input sample rate for the pipeline.
_resampler: Audio resampler for format conversion.
_hangup_attempted: Flag to track if hang-up has been attempted.
"""
class InputParams(BaseModel):
telnyx_sample_rate: int = 8000 # Default Telnyx rate (8kHz)
sample_rate: Optional[int] = None # Pipeline input rate
"""Configuration parameters for TelnyxFrameSerializer.
Attributes:
telnyx_sample_rate: Sample rate used by Telnyx, defaults to 8000 Hz.
sample_rate: Optional override for pipeline input sample rate.
inbound_encoding: Audio encoding for data sent to Telnyx (e.g., "PCMU").
outbound_encoding: Audio encoding for data received from Telnyx (e.g., "PCMU").
auto_hang_up: Whether to automatically terminate call on EndFrame.
"""
telnyx_sample_rate: int = 8000
sample_rate: Optional[int] = None
inbound_encoding: str = "PCMU"
outbound_encoding: str = "PCMU"
auto_hang_up: bool = True
def __init__(
self,
stream_id: str,
outbound_encoding: str,
inbound_encoding: str,
call_control_id: Optional[str] = None,
api_key: Optional[str] = None,
params: InputParams = InputParams(),
):
"""Initialize the TelnyxFrameSerializer.
Args:
stream_id: The Stream ID for Telnyx.
outbound_encoding: The encoding type for outbound audio (e.g., "PCMU").
inbound_encoding: The encoding type for inbound audio (e.g., "PCMU").
call_control_id: The Call Control ID for the Telnyx call (optional, but required for auto hang-up).
api_key: Your Telnyx API key (required for auto hang-up).
params: Configuration parameters.
"""
self._stream_id = stream_id
params.outbound_encoding = outbound_encoding
params.inbound_encoding = inbound_encoding
self._call_control_id = call_control_id
self._api_key = api_key
self._params = params
self._telnyx_sample_rate = self._params.telnyx_sample_rate
self._sample_rate = 0 # Pipeline input rate
self._resampler = create_default_resampler()
self._hangup_attempted = False
@property
def type(self) -> FrameSerializerType:
"""Gets the serializer type.
Returns:
The serializer type, either TEXT or BINARY.
"""
return FrameSerializerType.TEXT
async def setup(self, frame: StartFrame):
"""Sets up the serializer with pipeline configuration.
Args:
frame: The StartFrame containing pipeline configuration.
"""
self._sample_rate = self._params.sample_rate or frame.audio_in_sample_rate
async def serialize(self, frame: Frame) -> str | bytes | None:
if isinstance(frame, AudioRawFrame):
"""Serializes a Pipecat frame to Telnyx WebSocket format.
Handles conversion of various frame types to Telnyx WebSocket messages.
For EndFrames and CancelFrames, initiates call termination if auto_hang_up is enabled.
Args:
frame: The Pipecat frame to serialize.
Returns:
Serialized data as string or bytes, or None if the frame isn't handled.
Raises:
ValueError: If an unsupported encoding is specified.
"""
if (
self._params.auto_hang_up
and not self._hangup_attempted
and isinstance(frame, (EndFrame, CancelFrame))
):
self._hangup_attempted = True
await self._hang_up_call()
return None
elif isinstance(frame, StartInterruptionFrame):
answer = {"event": "clear"}
return json.dumps(answer)
elif isinstance(frame, AudioRawFrame):
data = frame.audio
# Output: Convert PCM at frame's rate to 8kHz encoded for Telnyx
@@ -84,11 +170,58 @@ class TelnyxFrameSerializer(FrameSerializer):
return json.dumps(answer)
if isinstance(frame, StartInterruptionFrame):
answer = {"event": "clear"}
return json.dumps(answer)
# Return None for unhandled frames
return None
async def _hang_up_call(self):
"""Hang up the Telnyx call using Telnyx's REST API."""
try:
call_control_id = self._call_control_id
api_key = self._api_key
if not call_control_id or not api_key:
logger.warning(
"Cannot hang up Telnyx call: call_control_id and api_key must be provided"
)
return
# Telnyx API endpoint for hanging up a call
endpoint = f"https://api.telnyx.com/v2/calls/{call_control_id}/actions/hangup"
# Set headers with API key
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
# Make the POST request to hang up the call
async with aiohttp.ClientSession() as session:
async with session.post(endpoint, headers=headers) as response:
if response.status == 200:
logger.info(f"Successfully terminated Telnyx call {call_control_id}")
else:
# Get the error details for better debugging
error_text = await response.text()
logger.error(
f"Failed to terminate Telnyx call {call_control_id}: "
f"Status {response.status}, Response: {error_text}"
)
except Exception as e:
logger.exception(f"Failed to hang up Telnyx call: {e}")
async def deserialize(self, data: str | bytes) -> Frame | None:
"""Deserializes Telnyx WebSocket data to Pipecat frames.
Handles conversion of Telnyx media events to appropriate Pipecat frames,
including audio data and DTMF keypresses.
Args:
data: The raw WebSocket data from Telnyx.
Returns:
A Pipecat frame corresponding to the Telnyx event, or None if unhandled.
Raises:
ValueError: If an unsupported encoding is specified.
"""
message = json.loads(data)
if message["event"] == "media":

155
src/pipecat/serializers/twilio.py
View File

@@ -8,11 +8,14 @@ import base64
import json
from typing import Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import create_default_resampler, pcm_to_ulaw, ulaw_to_pcm
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
InputDTMFFrame,
@@ -26,28 +29,107 @@ from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializer
class TwilioFrameSerializer(FrameSerializer):
class InputParams(BaseModel):
twilio_sample_rate: int = 8000 # Default Twilio rate (8kHz)
sample_rate: Optional[int] = None # Pipeline input rate
"""Serializer for Twilio Media Streams WebSocket protocol.
def __init__(self, stream_sid: str, params: InputParams = InputParams()):
This serializer handles converting between Pipecat frames and Twilio's WebSocket
media streams protocol. It supports audio conversion, DTMF events, and automatic
call termination.
When auto_hang_up is enabled (default), the serializer will automatically terminate
the Twilio call when an EndFrame or CancelFrame is processed, but requires Twilio
credentials to be provided.
Attributes:
_stream_sid: The Twilio Media Stream SID.
_call_sid: The associated Twilio Call SID.
_account_sid: Twilio account SID for API access.
_auth_token: Twilio authentication token for API access.
_params: Configuration parameters.
_twilio_sample_rate: Sample rate used by Twilio (typically 8kHz).
_sample_rate: Input sample rate for the pipeline.
_resampler: Audio resampler for format conversion.
"""
class InputParams(BaseModel):
"""Configuration parameters for TwilioFrameSerializer.
Attributes:
twilio_sample_rate: Sample rate used by Twilio, defaults to 8000 Hz.
sample_rate: Optional override for pipeline input sample rate.
auto_hang_up: Whether to automatically terminate call on EndFrame.
"""
twilio_sample_rate: int = 8000
sample_rate: Optional[int] = None
auto_hang_up: bool = True
def __init__(
self,
stream_sid: str,
call_sid: Optional[str] = None,
account_sid: Optional[str] = None,
auth_token: Optional[str] = None,
params: InputParams = InputParams(),
):
"""Initialize the TwilioFrameSerializer.
Args:
stream_sid: The Twilio Media Stream SID.
call_sid: The associated Twilio Call SID (optional, but required for auto hang-up).
account_sid: Twilio account SID (required for auto hang-up).
auth_token: Twilio auth token (required for auto hang-up).
params: Configuration parameters.
"""
self._stream_sid = stream_sid
self._call_sid = call_sid
self._account_sid = account_sid
self._auth_token = auth_token
self._params = params
self._twilio_sample_rate = self._params.twilio_sample_rate
self._sample_rate = 0 # Pipeline input rate
self._resampler = create_default_resampler()
self._hangup_attempted = False
@property
def type(self) -> FrameSerializerType:
"""Gets the serializer type.
Returns:
The serializer type, either TEXT or BINARY.
"""
return FrameSerializerType.TEXT
async def setup(self, frame: StartFrame):
"""Sets up the serializer with pipeline configuration.
Args:
frame: The StartFrame containing pipeline configuration.
"""
self._sample_rate = self._params.sample_rate or frame.audio_in_sample_rate
async def serialize(self, frame: Frame) -> str | bytes | None:
if isinstance(frame, StartInterruptionFrame):
"""Serializes a Pipecat frame to Twilio WebSocket format.
Handles conversion of various frame types to Twilio WebSocket messages.
For EndFrames, initiates call termination if auto_hang_up is enabled.
Args:
frame: The Pipecat frame to serialize.
Returns:
Serialized data as string or bytes, or None if the frame isn't handled.
"""
if (
self._params.auto_hang_up
and not self._hangup_attempted
and isinstance(frame, (EndFrame, CancelFrame))
):
self._hangup_attempted = True
await self._hang_up_call()
return None
elif isinstance(frame, StartInterruptionFrame):
answer = {"event": "clear", "streamSid": self._stream_sid}
return json.dumps(answer)
elif isinstance(frame, AudioRawFrame):
@@ -68,7 +150,70 @@ class TwilioFrameSerializer(FrameSerializer):
elif isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
return json.dumps(frame.message)
# Return None for unhandled frames
return None
async def _hang_up_call(self):
"""Hang up the Twilio call using Twilio's REST API."""
try:
import aiohttp
account_sid = self._account_sid
auth_token = self._auth_token
call_sid = self._call_sid
if not call_sid or not account_sid or not auth_token:
missing = []
if not call_sid:
missing.append("call_sid")
if not account_sid:
missing.append("account_sid")
if not auth_token:
missing.append("auth_token")
logger.warning(
f"Cannot hang up Twilio call: missing required parameters: {', '.join(missing)}"
)
return
# Twilio API endpoint for updating calls
endpoint = (
f"https://api.twilio.com/2010-04-01/Accounts/{account_sid}/Calls/{call_sid}.json"
)
# Create basic auth from account_sid and auth_token
auth = aiohttp.BasicAuth(account_sid, auth_token)
# Parameters to set the call status to "completed" (hang up)
params = {"Status": "completed"}
# Make the POST request to update the call
async with aiohttp.ClientSession() as session:
async with session.post(endpoint, auth=auth, data=params) as response:
if response.status == 200:
logger.info(f"Successfully terminated Twilio call {call_sid}")
else:
# Get the error details for better debugging
error_text = await response.text()
logger.error(
f"Failed to terminate Twilio call {call_sid}: "
f"Status {response.status}, Response: {error_text}"
)
except Exception as e:
logger.exception(f"Failed to hang up Twilio call: {e}")
async def deserialize(self, data: str | bytes) -> Frame | None:
"""Deserializes Twilio WebSocket data to Pipecat frames.
Handles conversion of Twilio media events to appropriate Pipecat frames.
Args:
data: The raw WebSocket data from Twilio.
Returns:
A Pipecat frame corresponding to the Twilio event, or None if unhandled.
"""
message = json.loads(data)
if message["event"] == "media":

2
src/pipecat/services/__init__.py
View File

@@ -37,4 +37,4 @@ class DeprecatedModuleProxy:
def __getattr__(self, attr):
if attr in self._globals:
return _warn_deprecated_access(self._globals, attr, self._old, self._new)
raise AttributeError(f"module 'pipecat.{self._old}' has no attribute '{attr}'")
raise AttributeError(f"module 'pipecat.services.{self._old}' has no attribute '{attr}'")

105
src/pipecat/services/ai_service.py Normal file
View File

@@ -0,0 +1,105 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, AsyncGenerator, Dict, Mapping
from loguru import logger
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
StartFrame,
)
from pipecat.metrics.metrics import MetricsData
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class AIService(FrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._model_name: str = ""
self._settings: Dict[str, Any] = {}
self._session_properties: Dict[str, Any] = {}
@property
def model_name(self) -> str:
return self._model_name
def set_model_name(self, model: str):
self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
async def start(self, frame: StartFrame):
pass
async def stop(self, frame: EndFrame):
pass
async def cancel(self, frame: CancelFrame):
pass
async def _update_settings(self, settings: Mapping[str, Any]):
from pipecat.services.openai_realtime_beta.events import (
SessionProperties,
)
for key, value in settings.items():
logger.debug("Update request for:", key, value)
if key in self._settings:
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._settings[key] = value
elif key in SessionProperties.model_fields:
logger.debug("Attempting to update", key, value)
try:
from pipecat.services.openai_realtime_beta.events import (
TurnDetection,
)
if isinstance(self._session_properties, SessionProperties):
current_properties = self._session_properties
else:
current_properties = SessionProperties(**self._session_properties)
if key == "turn_detection" and isinstance(value, dict):
turn_detection = TurnDetection(**value)
setattr(current_properties, key, turn_detection)
else:
setattr(current_properties, key, value)
validated_properties = SessionProperties.model_validate(
current_properties.model_dump()
)
logger.info(f"Updating LLM setting {key} to: [{value}]")
self._session_properties = validated_properties.model_dump()
except Exception as e:
logger.warning(f"Unexpected error updating session property {key}: {e}")
elif key == "model":
logger.info(f"Updating LLM setting {key} to: [{value}]")
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for {self.name} service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
await self.start(frame)
elif isinstance(frame, CancelFrame):
await self.cancel(frame)
elif isinstance(frame, EndFrame):
await self.stop(frame)
async def process_generator(self, generator: AsyncGenerator[Frame | None, None]):
async for f in generator:
if f:
if isinstance(f, ErrorFrame):
await self.push_error(f)
else:
await self.push_frame(f)

1129
src/pipecat/services/ai_services.py
View File

File diff suppressed because it is too large Load Diff

29
src/pipecat/services/anthropic/llm.py
View File

@@ -11,7 +11,7 @@ import io
import json
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Mapping, Optional, Union
from typing import Any, Dict, List, Optional, Union
import httpx
from loguru import logger
@@ -35,7 +35,9 @@ from pipecat.frames.frames import (
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMAssistantContextAggregator,
LLMUserAggregatorParams,
LLMUserContextAggregator,
)
from pipecat.processors.aggregators.openai_llm_context import (
@@ -43,16 +45,13 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.services.llm_service import LLMService
try:
from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Anthropic, you need to `pip install pipecat-ai[anthropic]`. "
+ "Also, set `ANTHROPIC_API_KEY` environment variable."
)
logger.error("In order to use Anthropic, you need to `pip install pipecat-ai[anthropic]`.")
raise Exception(f"Missing module: {e}")
@@ -120,8 +119,8 @@ class AnthropicLLMService(LLMService):
self,
context: OpenAILLMContext,
*,
user_kwargs: Mapping[str, Any] = {},
assistant_kwargs: Mapping[str, Any] = {},
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> AnthropicContextAggregatorPair:
"""Create an instance of AnthropicContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
@@ -129,12 +128,10 @@ class AnthropicLLMService(LLMService):
Args:
context (OpenAILLMContext): The LLM context.
user_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the user context aggregator constructor. Defaults
to an empty mapping.
assistant_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the assistant context aggregator
constructor. Defaults to an empty mapping.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
Returns:
AnthropicContextAggregatorPair: A pair of context aggregators, one
@@ -146,8 +143,8 @@ class AnthropicLLMService(LLMService):
if isinstance(context, OpenAILLMContext):
context = AnthropicLLMContext.from_openai_context(context)
user = AnthropicUserContextAggregator(context, **user_kwargs)
assistant = AnthropicAssistantContextAggregator(context, **assistant_kwargs)
user = AnthropicUserContextAggregator(context, params=user_params)
assistant = AnthropicAssistantContextAggregator(context, params=assistant_params)
return AnthropicContextAggregatorPair(_user=user, _assistant=assistant)
async def _process_context(self, context: OpenAILLMContext):

2
src/pipecat/services/assemblyai/stt.py
View File

@@ -18,7 +18,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.ai_services import STTService
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601

8
src/pipecat/services/aws/tts.py
View File

@@ -18,7 +18,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
try:
@@ -231,9 +231,9 @@ class PollyTTSService(TTSService):
yield TTSStartedFrame()
chunk_size = 8192
for i in range(0, len(audio_data), chunk_size):
chunk = audio_data[i : i + chunk_size]
CHUNK_SIZE = 1024
for i in range(0, len(audio_data), CHUNK_SIZE):
chunk = audio_data[i : i + CHUNK_SIZE]
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)

2
src/pipecat/services/azure/image.py
View File

@@ -13,7 +13,7 @@ from loguru import logger
from PIL import Image
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.ai_services import ImageGenService
from pipecat.services.image_service import ImageGenService
class AzureImageGenServiceREST(ImageGenService):

2
src/pipecat/services/azure/stt.py
View File

@@ -16,8 +16,8 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.ai_services import STTService
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601

6
src/pipecat/services/azure/tts.py
View File

@@ -18,8 +18,8 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.azure.common import language_to_azure_language
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
try:
@@ -159,8 +159,8 @@ class AzureTTSService(AzureBaseTTSService):
self._speech_config = SpeechConfig(
subscription=self._api_key,
region=self._region,
speech_recognition_language=self._settings["language"],
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)
@@ -254,8 +254,8 @@ class AzureHttpTTSService(AzureBaseTTSService):
self._speech_config = SpeechConfig(
subscription=self._api_key,
region=self._region,
speech_recognition_language=self._settings["language"],
)
self._speech_config.speech_synthesis_language = self._settings["language"]
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)

2
src/pipecat/services/canonical/metrics.py
View File

@@ -18,7 +18,7 @@ from pipecat.frames.frames import CancelFrame, EndFrame, Frame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AIService
from pipecat.services.ai_service import AIService
try:
import aiofiles

20
src/pipecat/services/cartesia/tts.py
View File

@@ -24,7 +24,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AudioContextWordTTSService, TTSService
from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
@@ -158,7 +158,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
msg = {
"transcript": text or " ", # Text must contain at least one character
"transcript": text,
"continue": continue_transcript,
"context_id": self._context_id,
"model_id": self.model_name,
@@ -166,6 +166,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"add_timestamps": add_timestamps,
"use_original_timestamps": True,
}
return json.dumps(msg)
@@ -184,7 +185,8 @@ class CartesiaTTSService(AudioContextWordTTSService):
async def _connect(self):
await self._connect_websocket()
if not self._receive_task:
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self):
@@ -196,7 +198,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
async def _connect_websocket(self):
try:
if self._websocket:
if self._websocket and self._websocket.open:
return
logger.debug("Connecting to Cartesia")
self._websocket = await websockets.connect(
@@ -214,11 +216,11 @@ class CartesiaTTSService(AudioContextWordTTSService):
if self._websocket:
logger.debug("Disconnecting from Cartesia")
await self._websocket.close()
self._websocket = None
self._context_id = None
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
finally:
self._context_id = None
self._websocket = None
def _get_websocket(self):
if self._websocket:
@@ -278,7 +280,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
if not self._websocket:
if not self._websocket or self._websocket.closed:
await self._connect()
if not self._context_id:
@@ -287,7 +289,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
self._context_id = str(uuid.uuid4())
await self.create_audio_context(self._context_id)
msg = self._build_msg(text=text or " ") # Text must contain at least one character
msg = self._build_msg(text=text)
try:
await self._get_websocket().send(msg)

16
src/pipecat/services/deepgram/stt.py
View File

@@ -19,7 +19,7 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import STTService
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
@@ -45,6 +45,7 @@ class DeepgramSTTService(STTService):
*,
api_key: str,
url: str = "",
base_url: str = "",
sample_rate: Optional[int] = None,
live_options: Optional[LiveOptions] = None,
addons: Optional[Dict] = None,
@@ -53,6 +54,17 @@ class DeepgramSTTService(STTService):
sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, **kwargs)
if url:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'url' is deprecated, use 'base_url' instead.",
DeprecationWarning,
)
base_url = url
default_options = LiveOptions(
encoding="linear16",
language=Language.EN,
@@ -81,7 +93,7 @@ class DeepgramSTTService(STTService):
self._client = DeepgramClient(
api_key,
config=DeepgramClientOptions(
url=url,
url=base_url,
options={"keepalive": "true"}, # verbose=logging.DEBUG
),
)

14
src/pipecat/services/deepgram/tts.py
View File

@@ -4,7 +4,6 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -16,10 +15,10 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
try:
from deepgram import DeepgramClient, SpeakOptions
from deepgram import DeepgramClient, DeepgramClientOptions, SpeakOptions
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Deepgram, you need to `pip install pipecat-ai[deepgram]`.")
@@ -32,6 +31,7 @@ class DeepgramTTSService(TTSService):
*,
api_key: str,
voice: str = "aura-helios-en",
base_url: str = "",
sample_rate: Optional[int] = None,
encoding: str = "linear16",
**kwargs,
@@ -42,7 +42,9 @@ class DeepgramTTSService(TTSService):
"encoding": encoding,
}
self.set_voice(voice)
self._deepgram_client = DeepgramClient(api_key=api_key)
client_options = DeepgramClientOptions(url=base_url)
self._deepgram_client = DeepgramClient(api_key, config=client_options)
def can_generate_metrics(self) -> bool:
return True
@@ -60,8 +62,8 @@ class DeepgramTTSService(TTSService):
try:
await self.start_ttfb_metrics()
response = await asyncio.to_thread(
self._deepgram_client.speak.v("1").stream, {"text": text}, options
response = await self._deepgram_client.speak.asyncrest.v("1").stream_memory(
{"text": text}, options
)
await self.start_tts_usage_metrics(text)

237
src/pipecat/services/elevenlabs/tts.py
View File

@@ -18,6 +18,7 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
LLMFullResponseEndFrame,
StartFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
@@ -25,7 +26,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import InterruptibleWordTTSService, TTSService
from pipecat.services.tts_service import InterruptibleWordTTSService, WordTTSService
from pipecat.transcriptions.language import Language
# See .env.example for ElevenLabs configuration needed
@@ -125,31 +126,14 @@ def build_elevenlabs_voice_settings(
settings: Dictionary containing voice settings parameters
Returns:
Dictionary of voice settings or None if required parameters are missing
Dictionary of voice settings or None if no valid settings are provided
"""
voice_setting_keys = ["stability", "similarity_boost", "style", "use_speaker_boost", "speed"]
voice_settings = {}
if settings["stability"] is not None and settings["similarity_boost"] is not None:
voice_settings["stability"] = settings["stability"]
voice_settings["similarity_boost"] = settings["similarity_boost"]
if settings["style"] is not None:
voice_settings["style"] = settings["style"]
if settings["use_speaker_boost"] is not None:
voice_settings["use_speaker_boost"] = settings["use_speaker_boost"]
if settings["speed"] is not None:
voice_settings["speed"] = settings["speed"]
else:
if settings["style"] is not None:
logger.warning(
"'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
)
if settings["use_speaker_boost"] is not None:
logger.warning(
"'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
)
if settings["speed"] is not None:
logger.warning(
"'speed' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
)
for key in voice_setting_keys:
if key in settings and settings[key] is not None:
voice_settings[key] = settings[key]
return voice_settings or None
@@ -308,10 +292,10 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
async def _connect(self):
await self._connect_websocket()
if not self._receive_task:
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
if not self._keepalive_task:
if self._websocket and not self._keepalive_task:
self._keepalive_task = self.create_task(self._keepalive_task_handler())
async def _disconnect(self):
@@ -327,7 +311,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
async def _connect_websocket(self):
try:
if self._websocket:
if self._websocket and self._websocket.open:
return
logger.debug("Connecting to ElevenLabs")
@@ -374,11 +358,11 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
logger.debug("Disconnecting from ElevenLabs")
await self._websocket.send(json.dumps({"text": ""}))
await self._websocket.close()
self._websocket = None
self._started = False
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
finally:
self._started = False
self._websocket = None
def _get_websocket(self):
if self._websocket:
@@ -418,7 +402,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
if not self._websocket:
if not self._websocket or self._websocket.closed:
await self._connect()
try:
@@ -441,8 +425,8 @@ class ElevenLabsTTSService(InterruptibleWordTTSService):
logger.error(f"{self} exception: {e}")
class ElevenLabsHttpTTSService(TTSService):
"""ElevenLabs Text-to-Speech service using HTTP streaming.
class ElevenLabsHttpTTSService(WordTTSService):
"""ElevenLabs Text-to-Speech service using HTTP streaming with word timestamps.
Args:
api_key: ElevenLabs API key
@@ -475,7 +459,13 @@ class ElevenLabsHttpTTSService(TTSService):
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
super().__init__(
aggregate_sentences=True,
push_text_frames=False,
push_stop_frames=True,
sample_rate=sample_rate,
**kwargs,
)
self._api_key = api_key
self._base_url = base_url
@@ -498,34 +488,136 @@ class ElevenLabsHttpTTSService(TTSService):
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
# Track cumulative time to properly sequence word timestamps across utterances
self._cumulative_time = 0
self._started = False
# Store previous text for context within a turn
self._previous_text = ""
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language to ElevenLabs language code."""
return language_to_elevenlabs_language(language)
def can_generate_metrics(self) -> bool:
"""Indicate that this service can generate usage metrics."""
return True
def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings)
def _reset_state(self):
"""Reset internal state variables."""
self._cumulative_time = 0
self._started = False
self._previous_text = ""
logger.debug(f"{self}: Reset internal state")
async def start(self, frame: StartFrame):
"""Initialize the service upon receiving a StartFrame."""
await super().start(frame)
self._output_format = output_format_from_sample_rate(self.sample_rate)
self._reset_state()
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using ElevenLabs streaming API.
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
await super().push_frame(frame, direction)
if isinstance(frame, (StartInterruptionFrame, TTSStoppedFrame)):
# Reset timing on interruption or stop
self._reset_state()
if isinstance(frame, TTSStoppedFrame):
await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)])
elif isinstance(frame, LLMFullResponseEndFrame):
# End of turn - reset previous text
self._previous_text = ""
def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]:
"""Calculate word timing from character alignment data.
Example input data:
{
"characters": [" ", "H", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d"],
"character_start_times_seconds": [0.0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
"character_end_times_seconds": [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
}
Would produce word times (with cumulative_time=0):
[("Hello", 0.1), ("world", 0.5)]
Args:
text: The text to convert to speech
alignment_info: Character timing data from ElevenLabs
Returns:
List of (word, timestamp) pairs
"""
chars = alignment_info.get("characters", [])
char_start_times = alignment_info.get("character_start_times_seconds", [])
if not chars or not char_start_times or len(chars) != len(char_start_times):
logger.warning(
f"Invalid alignment data: chars={len(chars)}, times={len(char_start_times)}"
)
return []
# Build the words and find their start times
words = []
word_start_times = []
current_word = ""
first_char_idx = -1
for i, char in enumerate(chars):
if char == " ":
if current_word: # Only add non-empty words
words.append(current_word)
# Use time of the first character of the word, offset by cumulative time
word_start_times.append(
self._cumulative_time + char_start_times[first_char_idx]
)
current_word = ""
first_char_idx = -1
else:
if not current_word: # This is the first character of a new word
first_char_idx = i
current_word += char
# Don't forget the last word if there's no trailing space
if current_word and first_char_idx >= 0:
words.append(current_word)
word_start_times.append(self._cumulative_time + char_start_times[first_char_idx])
# Create word-time pairs
word_times = list(zip(words, word_start_times))
return word_times
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using ElevenLabs streaming API with timestamps.
Makes a request to the ElevenLabs API to generate audio and timing data.
Tracks the duration of each utterance to ensure correct sequencing.
Includes previous text as context for better prosody continuity.
Args:
text: Text to convert to speech
Yields:
Frames containing audio data and status information
Audio and control frames
"""
logger.debug(f"{self}: Generating TTS [{text}]")
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream"
# Use the with-timestamps endpoint
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream/with-timestamps"
payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = {
"text": text,
"model_id": self._model_name,
}
# Include previous text as context if available
if self._previous_text:
payload["previous_text"] = self._previous_text
if self._voice_settings:
payload["voice_settings"] = self._voice_settings
@@ -550,8 +642,6 @@ class ElevenLabsHttpTTSService(TTSService):
if self._settings["optimize_streaming_latency"] is not None:
params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"]
logger.debug(f"ElevenLabs request - payload: {payload}, params: {params}")
try:
await self.start_ttfb_metrics()
@@ -566,17 +656,66 @@ class ElevenLabsHttpTTSService(TTSService):
await self.start_tts_usage_metrics(text)
# Process the streaming response
CHUNK_SIZE = 1024
# Start TTS sequence if not already started
if not self._started:
self.start_word_timestamps()
yield TTSStartedFrame()
self._started = True
# Track the duration of this utterance based on the last character's end time
utterance_duration = 0
async for line in response.content:
line_str = line.decode("utf-8").strip()
if not line_str:
continue
try:
# Parse the JSON object
data = json.loads(line_str)
# Process audio if present
if data and "audio_base64" in data:
await self.stop_ttfb_metrics()
audio = base64.b64decode(data["audio_base64"])
yield TTSAudioRawFrame(audio, self.sample_rate, 1)
# Process alignment if present
if data and "alignment" in data:
alignment = data["alignment"]
if alignment: # Ensure alignment is not None
# Get end time of the last character in this chunk
char_end_times = alignment.get("character_end_times_seconds", [])
if char_end_times:
chunk_end_time = char_end_times[-1]
# Update to the longest end time seen so far
utterance_duration = max(utterance_duration, chunk_end_time)
# Calculate word timestamps
word_times = self.calculate_word_times(alignment)
if word_times:
await self.add_word_timestamps(word_times)
except json.JSONDecodeError as e:
logger.warning(f"Failed to parse JSON from stream: {e}")
continue
except Exception as e:
logger.error(f"Error processing response: {e}", exc_info=True)
continue
# After processing all chunks, add the total utterance duration
# to the cumulative time to ensure next utterance starts after this one
if utterance_duration > 0:
self._cumulative_time += utterance_duration
# Append the current text to previous_text for context continuity
# Only add a space if there's already text
if self._previous_text:
self._previous_text += " " + text
else:
self._previous_text = text
yield TTSStartedFrame()
async for chunk in response.content.iter_chunked(CHUNK_SIZE):
if len(chunk) > 0:
await self.stop_ttfb_metrics()
yield TTSAudioRawFrame(chunk, self.sample_rate, 1)
except Exception as e:
logger.error(f"Error in run_tts: {e}")
yield ErrorFrame(error=str(e))
finally:
await self.stop_ttfb_metrics()
yield TTSStoppedFrame()
# Let the parent class handle TTSStoppedFrame

2
src/pipecat/services/fal/image.py
View File

@@ -15,7 +15,7 @@ from PIL import Image
from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.ai_services import ImageGenService
from pipecat.services.image_service import ImageGenService
try:
import fal_client

2
src/pipecat/services/fal/stt.py
View File

@@ -11,7 +11,7 @@ from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.ai_services import SegmentedSTTService
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601

16
src/pipecat/services/fish/tts.py
View File

@@ -22,7 +22,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import InterruptibleTTSService
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language
try:
@@ -104,7 +104,8 @@ class FishAudioTTSService(InterruptibleTTSService):
async def _connect(self):
await self._connect_websocket()
if not self._receive_task:
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self):
@@ -116,7 +117,7 @@ class FishAudioTTSService(InterruptibleTTSService):
async def _connect_websocket(self):
try:
if self._websocket:
if self._websocket and self._websocket.open:
return
logger.debug("Connecting to Fish Audio")
@@ -141,16 +142,17 @@ class FishAudioTTSService(InterruptibleTTSService):
stop_message = {"event": "stop"}
await self._websocket.send(ormsgpack.packb(stop_message))
await self._websocket.close()
self._websocket = None
self._request_id = None
self._started = False
except Exception as e:
logger.error(f"Error closing websocket: {e}")
finally:
self._request_id = None
self._started = False
self._websocket = None
async def flush_audio(self):
"""Flush any buffered audio by sending a flush event to Fish Audio."""
logger.trace(f"{self}: Flushing audio buffers")
if not self._websocket:
if not self._websocket or self._websocket.closed:
return
flush_message = {"event": "flush"}
await self._get_websocket().send(ormsgpack.packb(flush_message))

44
src/pipecat/services/gemini_multimodal_live/events.py
View File

@@ -8,6 +8,7 @@
import base64
import io
import json
from enum import Enum
from typing import List, Literal, Optional
from PIL import Image
@@ -35,6 +36,38 @@ class Turn(BaseModel):
parts: List[ContentPart]
class StartSensitivity(str, Enum):
"""Determines how start of speech is detected."""
UNSPECIFIED = "START_SENSITIVITY_UNSPECIFIED" # Default is HIGH
HIGH = "START_SENSITIVITY_HIGH" # Detect start of speech more often
LOW = "START_SENSITIVITY_LOW" # Detect start of speech less often
class EndSensitivity(str, Enum):
"""Determines how end of speech is detected."""
UNSPECIFIED = "END_SENSITIVITY_UNSPECIFIED" # Default is HIGH
HIGH = "END_SENSITIVITY_HIGH" # End speech more often
LOW = "END_SENSITIVITY_LOW" # End speech less often
class AutomaticActivityDetection(BaseModel):
"""Configures automatic detection of activity."""
disabled: Optional[bool] = None
start_of_speech_sensitivity: Optional[StartSensitivity] = None
prefix_padding_ms: Optional[int] = None
end_of_speech_sensitivity: Optional[EndSensitivity] = None
silence_duration_ms: Optional[int] = None
class RealtimeInputConfig(BaseModel):
"""Configures the realtime input behavior."""
automatic_activity_detection: Optional[AutomaticActivityDetection] = None
class RealtimeInput(BaseModel):
mediaChunks: List[MediaChunk]
@@ -78,11 +111,17 @@ class SystemInstruction(BaseModel):
parts: List[ContentPart]
class AudioTranscriptionConfig(BaseModel):
pass
class Setup(BaseModel):
model: str
system_instruction: Optional[SystemInstruction] = None
tools: Optional[List[dict]] = None
generation_config: Optional[dict] = None
output_audio_transcription: Optional[AudioTranscriptionConfig] = None
realtime_input_config: Optional[RealtimeInputConfig] = None
class Config(BaseModel):
@@ -120,10 +159,15 @@ class ServerContentTurnComplete(BaseModel):
turnComplete: bool
class BidiGenerateContentTranscription(BaseModel):
text: str
class ServerContent(BaseModel):
modelTurn: Optional[ModelTurn] = None
interrupted: Optional[bool] = None
turnComplete: Optional[bool] = None
outputTranscription: Optional[BidiGenerateContentTranscription] = None
class FunctionCall(BaseModel):

314
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -10,9 +10,8 @@ import json
import time
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Mapping, Optional, Union
from typing import Any, Dict, List, Optional, Union
import websockets
from loguru import logger
from pydantic import BaseModel, Field
@@ -45,21 +44,125 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMUserAggregatorParams,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.services.llm_service import LLMService
from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from . import events
from .audio_transcriber import AudioTranscriber
try:
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
raise Exception(f"Missing module: {e}")
def language_to_gemini_language(language: Language) -> Optional[str]:
"""Maps a Language enum value to a Gemini Live supported language code.
Source:
https://ai.google.dev/api/generate-content#MediaResolution
Returns None if the language is not supported by Gemini Live.
"""
language_map = {
# Arabic
Language.AR: "ar-XA",
# Bengali
Language.BN_IN: "bn-IN",
# Chinese (Mandarin)
Language.CMN: "cmn-CN",
Language.CMN_CN: "cmn-CN",
Language.ZH: "cmn-CN", # Map general Chinese to Mandarin for Gemini
Language.ZH_CN: "cmn-CN", # Map Simplified Chinese to Mandarin for Gemini
# German
Language.DE: "de-DE",
Language.DE_DE: "de-DE",
# English
Language.EN: "en-US", # Default to US English (though not explicitly listed in supported codes)
Language.EN_US: "en-US",
Language.EN_AU: "en-AU",
Language.EN_GB: "en-GB",
Language.EN_IN: "en-IN",
# Spanish
Language.ES: "es-ES", # Default to Spain Spanish
Language.ES_ES: "es-ES",
Language.ES_US: "es-US",
# French
Language.FR: "fr-FR", # Default to France French
Language.FR_FR: "fr-FR",
Language.FR_CA: "fr-CA",
# Gujarati
Language.GU: "gu-IN",
Language.GU_IN: "gu-IN",
# Hindi
Language.HI: "hi-IN",
Language.HI_IN: "hi-IN",
# Indonesian
Language.ID: "id-ID",
Language.ID_ID: "id-ID",
# Italian
Language.IT: "it-IT",
Language.IT_IT: "it-IT",
# Japanese
Language.JA: "ja-JP",
Language.JA_JP: "ja-JP",
# Kannada
Language.KN: "kn-IN",
Language.KN_IN: "kn-IN",
# Korean
Language.KO: "ko-KR",
Language.KO_KR: "ko-KR",
# Malayalam
Language.ML: "ml-IN",
Language.ML_IN: "ml-IN",
# Marathi
Language.MR: "mr-IN",
Language.MR_IN: "mr-IN",
# Dutch
Language.NL: "nl-NL",
Language.NL_NL: "nl-NL",
# Polish
Language.PL: "pl-PL",
Language.PL_PL: "pl-PL",
# Portuguese (Brazil)
Language.PT_BR: "pt-BR",
# Russian
Language.RU: "ru-RU",
Language.RU_RU: "ru-RU",
# Tamil
Language.TA: "ta-IN",
Language.TA_IN: "ta-IN",
# Telugu
Language.TE: "te-IN",
Language.TE_IN: "te-IN",
# Thai
Language.TH: "th-TH",
Language.TH_TH: "th-TH",
# Turkish
Language.TR: "tr-TR",
Language.TR_TR: "tr-TR",
# Vietnamese
Language.VI: "vi-VN",
Language.VI_VN: "vi-VN",
}
return language_map.get(language)
class GeminiMultimodalLiveContext(OpenAILLMContext):
@staticmethod
@@ -143,6 +246,25 @@ class GeminiMultimodalModalities(Enum):
AUDIO = "AUDIO"
class GeminiMediaResolution(str, Enum):
"""Media resolution options for Gemini Multimodal Live."""
UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default
LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens
MEDIUM = "MEDIA_RESOLUTION_MEDIUM" # 256 tokens
HIGH = "MEDIA_RESOLUTION_HIGH" # Zoomed reframing with 256 tokens
class GeminiVADParams(BaseModel):
"""Voice Activity Detection parameters."""
disabled: Optional[bool] = Field(default=None)
start_sensitivity: Optional[events.StartSensitivity] = Field(default=None)
end_sensitivity: Optional[events.EndSensitivity] = Field(default=None)
prefix_padding_ms: Optional[int] = Field(default=None)
silence_duration_ms: Optional[int] = Field(default=None)
class InputParams(BaseModel):
frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0)
max_tokens: Optional[int] = Field(default=4096, ge=1)
@@ -153,6 +275,11 @@ class InputParams(BaseModel):
modalities: Optional[GeminiMultimodalModalities] = Field(
default=GeminiMultimodalModalities.AUDIO
)
language: Optional[Language] = Field(default=Language.EN_US)
media_resolution: Optional[GeminiMediaResolution] = Field(
default=GeminiMediaResolution.UNSPECIFIED
)
vad: Optional[GeminiVADParams] = Field(default=None)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
@@ -164,25 +291,25 @@ class GeminiMultimodalLiveLLMService(LLMService):
self,
*,
api_key: str,
base_url="generativelanguage.googleapis.com",
model="models/gemini-2.0-flash-exp",
base_url: str = "generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent",
model="models/gemini-2.0-flash-live-001",
voice_id: str = "Charon",
start_audio_paused: bool = False,
start_video_paused: bool = False,
system_instruction: Optional[str] = None,
tools: Optional[Union[List[dict], ToolsSchema]] = None,
transcribe_user_audio: bool = False,
transcribe_model_audio: bool = False,
params: InputParams = InputParams(),
inference_on_context_initialization: bool = True,
**kwargs,
):
super().__init__(base_url=base_url, **kwargs)
self._last_sent_time = 0
self.api_key = api_key
self.base_url = base_url
self._api_key = api_key
self._base_url = base_url
self.set_model_name(model)
self._voice_id = voice_id
self._language_code = params.language
self._system_instruction = system_instruction
self._tools = tools
@@ -195,9 +322,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._websocket = None
self._receive_task = None
self._transcribe_audio_task = None
self._transcribe_model_audio_task = None
self._transcribe_audio_queue = asyncio.Queue()
self._transcribe_model_audio_queue = asyncio.Queue()
self._disconnecting = False
self._api_session_ready = False
@@ -205,7 +330,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._transcriber = AudioTranscriber(api_key)
self._transcribe_user_audio = transcribe_user_audio
self._transcribe_model_audio = transcribe_model_audio
self._user_is_speaking = False
self._bot_is_speaking = False
self._user_audio_buffer = bytearray()
@@ -214,6 +338,12 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._sample_rate = 24000
self._language = params.language
self._language_code = (
language_to_gemini_language(params.language) if params.language else "en-US"
)
self._vad_params = params.vad
self._settings = {
"frequency_penalty": params.frequency_penalty,
"max_tokens": params.max_tokens,
@@ -222,6 +352,9 @@ class GeminiMultimodalLiveLLMService(LLMService):
"top_k": params.top_k,
"top_p": params.top_p,
"modalities": params.modalities,
"language": self._language_code,
"media_resolution": params.media_resolution,
"vad": params.vad,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
@@ -237,6 +370,13 @@ class GeminiMultimodalLiveLLMService(LLMService):
def set_model_modalities(self, modalities: GeminiMultimodalModalities):
self._settings["modalities"] = modalities
def set_language(self, language: Language):
"""Set the language for generation."""
self._language = language
self._language_code = language_to_gemini_language(language) or "en-US"
self._settings["language"] = self._language_code
logger.info(f"Set Gemini language to: {self._language_code}")
async def set_context(self, context: OpenAILLMContext):
"""Set the context explicitly from outside the pipeline.
@@ -303,22 +443,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
TranscriptionFrame(text=text, user_id="user", timestamp=time_now_iso8601())
)
async def _handle_transcribe_model_audio(self, audio, context):
# Early return if modalities are not set to audio.
if self._settings["modalities"] != GeminiMultimodalModalities.AUDIO:
return
text = await self._transcribe_audio(audio, context)
logger.debug(f"[Transcription:model] {text}")
# We add user messages directly to the context. We don't do that for assistant messages,
# because we assume the frames we emit will work normally in this downstream case. This
# definitely feels like a hack. Need to revisit when the API evolves.
# context.add_message({"role": "assistant", "content": [{"type": "text", "text": text}]})
await self.push_frame(LLMFullResponseStartFrame())
await self.push_frame(LLMTextFrame(text=text))
await self.push_frame(TTSTextFrame(text=text))
await self.push_frame(LLMFullResponseEndFrame())
async def _transcribe_audio(self, audio, context):
(text, prompt_tokens, completion_tokens, total_tokens) = await self._transcriber.transcribe(
audio, context
@@ -407,36 +531,66 @@ class GeminiMultimodalLiveLLMService(LLMService):
logger.info("Connecting to Gemini service")
try:
uri = f"wss://{self.base_url}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={self.api_key}"
logger.info(f"Connecting to {uri}")
logger.info(f"Connecting to wss://{self._base_url}")
uri = f"wss://{self._base_url}?key={self._api_key}"
self._websocket = await websockets.connect(uri=uri)
self._receive_task = self.create_task(self._receive_task_handler())
self._transcribe_audio_task = self.create_task(self._transcribe_audio_handler())
self._transcribe_model_audio_task = self.create_task(
self._transcribe_model_audio_handler()
)
config = events.Config.model_validate(
{
"setup": {
"model": self._model_name,
"generation_config": {
"frequency_penalty": self._settings["frequency_penalty"],
"max_output_tokens": self._settings["max_tokens"], # Not supported yet
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"response_modalities": self._settings["modalities"].value,
"speech_config": {
"voice_config": {
"prebuilt_voice_config": {"voice_name": self._voice_id}
},
},
},
},
}
)
# Create the basic configuration
config_data = {
"setup": {
"model": self._model_name,
"generation_config": {
"frequency_penalty": self._settings["frequency_penalty"],
"max_output_tokens": self._settings["max_tokens"],
"presence_penalty": self._settings["presence_penalty"],
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
"response_modalities": self._settings["modalities"].value,
"speech_config": {
"voice_config": {
"prebuilt_voice_config": {"voice_name": self._voice_id}
},
"language_code": self._settings["language"],
},
"media_resolution": self._settings["media_resolution"].value,
},
"output_audio_transcription": {},
}
}
# Add VAD configuration if provided
if self._settings.get("vad"):
vad_config = {}
vad_params = self._settings["vad"]
# Only add parameters that are explicitly set
if vad_params.disabled is not None:
vad_config["disabled"] = vad_params.disabled
if vad_params.start_sensitivity:
vad_config["start_of_speech_sensitivity"] = vad_params.start_sensitivity.value
if vad_params.end_sensitivity:
vad_config["end_of_speech_sensitivity"] = vad_params.end_sensitivity.value
if vad_params.prefix_padding_ms is not None:
vad_config["prefix_padding_ms"] = vad_params.prefix_padding_ms
if vad_params.silence_duration_ms is not None:
vad_config["silence_duration_ms"] = vad_params.silence_duration_ms
# Only add automatic_activity_detection if we have VAD settings
if vad_config:
realtime_config = {"automatic_activity_detection": vad_config}
config_data["setup"]["realtime_input_config"] = realtime_config
config = events.Config.model_validate(config_data)
# Add system instruction if available
system_instruction = self._system_instruction or ""
if self._context and hasattr(self._context, "extract_system_instructions"):
system_instruction += "\n" + self._context.extract_system_instructions()
@@ -445,9 +599,13 @@ class GeminiMultimodalLiveLLMService(LLMService):
config.setup.system_instruction = events.SystemInstruction(
parts=[events.ContentPart(text=system_instruction)]
)
# Add tools if available
if self._tools:
logger.debug(f"Gemini is configuring to use tools{self._tools}")
config.setup.tools = self.get_llm_adapter().from_standard_tools(self._tools)
# Send the configuration
await self.send_client_event(config)
except Exception as e:
@@ -469,9 +627,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
if self._transcribe_audio_task:
await self.cancel_task(self._transcribe_audio_task)
self._transcribe_audio_task = None
if self._transcribe_model_audio_task:
await self.cancel_task(self._transcribe_model_audio_task)
self._transcribe_model_audio_task = None
self._disconnecting = False
except Exception as e:
logger.error(f"{self} error disconnecting: {e}")
@@ -508,6 +663,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._handle_evt_model_turn(evt)
elif evt.serverContent and evt.serverContent.turnComplete:
await self._handle_evt_turn_complete(evt)
elif evt.serverContent and evt.serverContent.outputTranscription:
await self._handle_evt_output_transcription(evt)
elif evt.toolCall:
await self._handle_evt_tool_call(evt)
elif False: # !!! todo: error events?
@@ -522,11 +679,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
audio = await self._transcribe_audio_queue.get()
await self._handle_transcribe_user_audio(audio, self._context)
async def _transcribe_model_audio_handler(self):
while True:
audio = await self._transcribe_model_audio_queue.get()
await self._handle_transcribe_model_audio(audio, self._context)
#
#
#
@@ -706,24 +858,31 @@ class GeminiMultimodalLiveLLMService(LLMService):
async def _handle_evt_turn_complete(self, evt):
self._bot_is_speaking = False
audio = self._bot_audio_buffer
text = self._bot_text_buffer
self._bot_audio_buffer = bytearray()
self._bot_text_buffer = ""
if audio and self._transcribe_model_audio and self._context:
await self._transcribe_model_audio_queue.put(audio)
elif text:
if text:
await self.push_frame(LLMFullResponseEndFrame())
await self.push_frame(TTSStoppedFrame())
async def _handle_evt_output_transcription(self, evt):
if not evt.serverContent.outputTranscription:
return
text = evt.serverContent.outputTranscription.text
if text:
await self.push_frame(LLMFullResponseStartFrame())
await self.push_frame(LLMTextFrame(text=text))
await self.push_frame(TTSTextFrame(text=text))
await self.push_frame(LLMFullResponseEndFrame())
def create_context_aggregator(
self,
context: OpenAILLMContext,
*,
user_kwargs: Mapping[str, Any] = {},
assistant_kwargs: Mapping[str, Any] = {},
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> GeminiMultimodalLiveContextAggregatorPair:
"""Create an instance of GeminiMultimodalLiveContextAggregatorPair from
an OpenAILLMContext. Constructor keyword arguments for both the user and
@@ -731,12 +890,10 @@ class GeminiMultimodalLiveLLMService(LLMService):
Args:
context (OpenAILLMContext): The LLM context.
user_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the user context aggregator constructor. Defaults
to an empty mapping.
assistant_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the assistant context aggregator
constructor. Defaults to an empty mapping.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
Returns:
GeminiMultimodalLiveContextAggregatorPair: A pair of context
@@ -747,11 +904,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
context.set_llm_adapter(self.get_llm_adapter())
GeminiMultimodalLiveContext.upgrade(context)
user = GeminiMultimodalLiveUserContextAggregator(context, **user_kwargs)
user = GeminiMultimodalLiveUserContextAggregator(context, params=user_params)
default_assistant_kwargs = {"expect_stripped_words": False}
default_assistant_kwargs.update(assistant_kwargs)
assistant = GeminiMultimodalLiveAssistantContextAggregator(
context, **default_assistant_kwargs
)
assistant_params.expect_stripped_words = True
assistant = GeminiMultimodalLiveAssistantContextAggregator(context, params=assistant_params)
return GeminiMultimodalLiveContextAggregatorPair(_user=user, _assistant=assistant)

253
src/pipecat/services/gladia.py
View File

@@ -1,253 +0,0 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import base64
import json
from typing import AsyncGenerator, Optional
import aiohttp
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
TranscriptionFrame,
)
from pipecat.services.ai_services import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
try:
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Gladia, you need to `pip install pipecat-ai[gladia]`. Also, set `GLADIA_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
def language_to_gladia_language(language: Language) -> Optional[str]:
BASE_LANGUAGES = {
Language.AF: "af",
Language.AM: "am",
Language.AR: "ar",
Language.AS: "as",
Language.AZ: "az",
Language.BG: "bg",
Language.BN: "bn",
Language.BS: "bs",
Language.CA: "ca",
Language.CS: "cs",
Language.CY: "cy",
Language.DA: "da",
Language.DE: "de",
Language.EL: "el",
Language.EN: "en",
Language.ES: "es",
Language.ET: "et",
Language.EU: "eu",
Language.FA: "fa",
Language.FI: "fi",
Language.FR: "fr",
Language.GA: "ga",
Language.GL: "gl",
Language.GU: "gu",
Language.HE: "he",
Language.HI: "hi",
Language.HR: "hr",
Language.HU: "hu",
Language.HY: "hy",
Language.ID: "id",
Language.IS: "is",
Language.IT: "it",
Language.JA: "ja",
Language.JV: "jv",
Language.KA: "ka",
Language.KK: "kk",
Language.KM: "km",
Language.KN: "kn",
Language.KO: "ko",
Language.LO: "lo",
Language.LT: "lt",
Language.LV: "lv",
Language.MK: "mk",
Language.ML: "ml",
Language.MN: "mn",
Language.MR: "mr",
Language.MS: "ms",
Language.MT: "mt",
Language.MY: "my",
Language.NE: "ne",
Language.NL: "nl",
Language.NO: "no",
Language.OR: "or",
Language.PA: "pa",
Language.PL: "pl",
Language.PS: "ps",
Language.PT: "pt",
Language.RO: "ro",
Language.RU: "ru",
Language.SI: "si",
Language.SK: "sk",
Language.SL: "sl",
Language.SO: "so",
Language.SQ: "sq",
Language.SR: "sr",
Language.SU: "su",
Language.SV: "sv",
Language.SW: "sw",
Language.TA: "ta",
Language.TE: "te",
Language.TH: "th",
Language.TR: "tr",
Language.UK: "uk",
Language.UR: "ur",
Language.UZ: "uz",
Language.VI: "vi",
Language.ZH: "zh",
Language.ZU: "zu",
}
result = BASE_LANGUAGES.get(language)
# If not found in base languages, try to find the base language from a variant
if not result:
# Convert enum value to string and get the base language part (e.g. es-ES -> es)
lang_str = str(language.value)
base_code = lang_str.split("-")[0].lower()
# Look up the base code in our supported languages
result = base_code if base_code in BASE_LANGUAGES.values() else None
return result
class GladiaSTTService(STTService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
endpointing: Optional[float] = 0.2
maximum_duration_without_endpointing: Optional[int] = 10
audio_enhancer: Optional[bool] = None
words_accurate_timestamps: Optional[bool] = None
speech_threshold: Optional[float] = 0.99
def __init__(
self,
*,
api_key: str,
url: str = "https://api.gladia.io/v2/live",
confidence: float = 0.5,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._url = url
self._settings = {
"encoding": "wav/pcm",
"bit_depth": 16,
"sample_rate": 0,
"channels": 1,
"language_config": {
"languages": [self.language_to_service_language(params.language)]
if params.language
else [],
"code_switching": False,
},
"endpointing": params.endpointing,
"maximum_duration_without_endpointing": params.maximum_duration_without_endpointing,
"pre_processing": {
"audio_enhancer": params.audio_enhancer,
"speech_threshold": params.speech_threshold,
},
"realtime_processing": {
"words_accurate_timestamps": params.words_accurate_timestamps,
},
}
self._confidence = confidence
self._websocket = None
self._receive_task = None
def language_to_service_language(self, language: Language) -> Optional[str]:
return language_to_gladia_language(language)
async def start(self, frame: StartFrame):
await super().start(frame)
if self._websocket:
return
self._settings["sample_rate"] = self.sample_rate
response = await self._setup_gladia()
self._websocket = await websockets.connect(response["url"])
if not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._send_stop_recording()
if self._websocket:
await self._websocket.close()
self._websocket = None
if self._receive_task:
await self.wait_for_task(self._receive_task)
self._receive_task = None
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._websocket.close()
if self._receive_task:
await self.cancel_task(self._receive_task)
self._receive_task = None
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
await self._send_audio(audio)
await self.stop_processing_metrics()
yield None
async def _setup_gladia(self):
async with aiohttp.ClientSession() as session:
async with session.post(
self._url,
headers={"X-Gladia-Key": self._api_key, "Content-Type": "application/json"},
json=self._settings,
) as response:
if response.ok:
return await response.json()
else:
logger.error(
f"Gladia error: {response.status}: {response.text or response.reason}"
)
raise Exception(f"Failed to initialize Gladia session: {response.status}")
async def _send_audio(self, audio: bytes):
data = base64.b64encode(audio).decode("utf-8")
message = {"type": "audio_chunk", "data": {"chunk": data}}
await self._websocket.send(json.dumps(message))
async def _send_stop_recording(self):
await self._websocket.send(json.dumps({"type": "stop_recording"}))
async def _receive_task_handler(self):
async for message in self._websocket:
content = json.loads(message)
if content["type"] == "transcript":
utterance = content["data"]["utterance"]
confidence = utterance.get("confidence", 0)
transcript = utterance["text"]
if confidence >= self._confidence:
if content["data"]["is_final"]:
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601())
)
else:
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601())
)

13
src/pipecat/services/gladia/__init__.py Normal file
View File

@@ -0,0 +1,13 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import sys
from pipecat.services import DeprecatedModuleProxy
from .stt import *
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "gladia", "gladia.stt")

163
src/pipecat/services/gladia/config.py Normal file
View File

@@ -0,0 +1,163 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, Dict, List, Optional, Union
from pydantic import BaseModel
from pipecat.transcriptions.language import Language
class LanguageConfig(BaseModel):
"""Configuration for language detection and handling.
Attributes:
languages: List of language codes to use for transcription
code_switching: Whether to auto-detect language changes during transcription
"""
languages: Optional[List[str]] = None
code_switching: Optional[bool] = None
class PreProcessingConfig(BaseModel):
"""Configuration for audio pre-processing options.
Attributes:
speech_threshold: Sensitivity for speech detection (0-1)
"""
speech_threshold: Optional[float] = None
class CustomVocabularyItem(BaseModel):
"""Represents a custom vocabulary item with an intensity value.
Attributes:
value: The vocabulary word or phrase
intensity: The bias intensity for this vocabulary item (0-1)
"""
value: str
intensity: float
class CustomVocabularyConfig(BaseModel):
"""Configuration for custom vocabulary.
Attributes:
vocabulary: List of words/phrases or CustomVocabularyItem objects
default_intensity: Default intensity for simple string vocabulary items
"""
vocabulary: Optional[List[Union[str, CustomVocabularyItem]]] = None
default_intensity: Optional[float] = None
class CustomSpellingConfig(BaseModel):
"""Configuration for custom spelling rules.
Attributes:
spelling_dictionary: Mapping of correct spellings to phonetic variations
"""
spelling_dictionary: Optional[Dict[str, List[str]]] = None
class TranslationConfig(BaseModel):
"""Configuration for real-time translation.
Attributes:
target_languages: List of target language codes for translation
model: Translation model to use ("base" or "enhanced")
match_original_utterances: Whether to align translations with original utterances
"""
target_languages: Optional[List[str]] = None
model: Optional[str] = None
match_original_utterances: Optional[bool] = None
class RealtimeProcessingConfig(BaseModel):
"""Configuration for real-time processing features.
Attributes:
words_accurate_timestamps: Whether to provide per-word timestamps
custom_vocabulary: Whether to enable custom vocabulary
custom_vocabulary_config: Custom vocabulary configuration
custom_spelling: Whether to enable custom spelling
custom_spelling_config: Custom spelling configuration
translation: Whether to enable translation
translation_config: Translation configuration
named_entity_recognition: Whether to enable named entity recognition
sentiment_analysis: Whether to enable sentiment analysis
"""
words_accurate_timestamps: Optional[bool] = None
custom_vocabulary: Optional[bool] = None
custom_vocabulary_config: Optional[CustomVocabularyConfig] = None
custom_spelling: Optional[bool] = None
custom_spelling_config: Optional[CustomSpellingConfig] = None
translation: Optional[bool] = None
translation_config: Optional[TranslationConfig] = None
named_entity_recognition: Optional[bool] = None
sentiment_analysis: Optional[bool] = None
class MessagesConfig(BaseModel):
"""Configuration for controlling which message types are sent via WebSocket.
Attributes:
receive_partial_transcripts: Whether to receive intermediate transcription results
receive_final_transcripts: Whether to receive final transcription results
receive_speech_events: Whether to receive speech begin/end events
receive_pre_processing_events: Whether to receive pre-processing events
receive_realtime_processing_events: Whether to receive real-time processing events
receive_post_processing_events: Whether to receive post-processing events
receive_acknowledgments: Whether to receive acknowledgment messages
receive_errors: Whether to receive error messages
receive_lifecycle_events: Whether to receive lifecycle events
"""
receive_partial_transcripts: Optional[bool] = None
receive_final_transcripts: Optional[bool] = None
receive_speech_events: Optional[bool] = None
receive_pre_processing_events: Optional[bool] = None
receive_realtime_processing_events: Optional[bool] = None
receive_post_processing_events: Optional[bool] = None
receive_acknowledgments: Optional[bool] = None
receive_errors: Optional[bool] = None
receive_lifecycle_events: Optional[bool] = None
class GladiaInputParams(BaseModel):
"""Configuration parameters for the Gladia STT service.
Attributes:
encoding: Audio encoding format
bit_depth: Audio bit depth
channels: Number of audio channels
custom_metadata: Additional metadata to include with requests
endpointing: Silence duration in seconds to mark end of speech
maximum_duration_without_endpointing: Maximum utterance duration without silence
language: DEPRECATED - Use language_config instead
language_config: Detailed language configuration
pre_processing: Audio pre-processing options
realtime_processing: Real-time processing features
messages_config: WebSocket message filtering options
"""
encoding: Optional[str] = "wav/pcm"
bit_depth: Optional[int] = 16
channels: Optional[int] = 1
custom_metadata: Optional[Dict[str, Any]] = None
endpointing: Optional[float] = None
maximum_duration_without_endpointing: Optional[int] = 10
language: Optional[Language] = None # Deprecated
language_config: Optional[LanguageConfig] = None
pre_processing: Optional[PreProcessingConfig] = None
realtime_processing: Optional[RealtimeProcessingConfig] = None
messages_config: Optional[MessagesConfig] = None

401
src/pipecat/services/gladia/stt.py Normal file
View File

@@ -0,0 +1,401 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import base64
import json
import warnings
from typing import Any, AsyncGenerator, Dict, Optional
import aiohttp
from loguru import logger
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
TranscriptionFrame,
)
from pipecat.services.gladia.config import GladiaInputParams
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
try:
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Gladia, you need to `pip install pipecat-ai[gladia]`.")
raise Exception(f"Missing module: {e}")
def language_to_gladia_language(language: Language) -> Optional[str]:
"""Convert a Language enum to Gladia's language code format.
Args:
language: The Language enum value to convert
Returns:
The Gladia language code string or None if not supported
"""
BASE_LANGUAGES = {
Language.AF: "af",
Language.AM: "am",
Language.AR: "ar",
Language.AS: "as",
Language.AZ: "az",
Language.BA: "ba",
Language.BE: "be",
Language.BG: "bg",
Language.BN: "bn",
Language.BO: "bo",
Language.BR: "br",
Language.BS: "bs",
Language.CA: "ca",
Language.CS: "cs",
Language.CY: "cy",
Language.DA: "da",
Language.DE: "de",
Language.EL: "el",
Language.EN: "en",
Language.ES: "es",
Language.ET: "et",
Language.EU: "eu",
Language.FA: "fa",
Language.FI: "fi",
Language.FO: "fo",
Language.FR: "fr",
Language.GL: "gl",
Language.GU: "gu",
Language.HA: "ha",
Language.HAW: "haw",
Language.HE: "he",
Language.HI: "hi",
Language.HR: "hr",
Language.HT: "ht",
Language.HU: "hu",
Language.HY: "hy",
Language.ID: "id",
Language.IS: "is",
Language.IT: "it",
Language.JA: "ja",
Language.JV: "jv",
Language.KA: "ka",
Language.KK: "kk",
Language.KM: "km",
Language.KN: "kn",
Language.KO: "ko",
Language.LA: "la",
Language.LB: "lb",
Language.LN: "ln",
Language.LO: "lo",
Language.LT: "lt",
Language.LV: "lv",
Language.MG: "mg",
Language.MI: "mi",
Language.MK: "mk",
Language.ML: "ml",
Language.MN: "mn",
Language.MR: "mr",
Language.MS: "ms",
Language.MT: "mt",
Language.MY_MR: "mymr",
Language.NE: "ne",
Language.NL: "nl",
Language.NN: "nn",
Language.NO: "no",
Language.OC: "oc",
Language.PA: "pa",
Language.PL: "pl",
Language.PS: "ps",
Language.PT: "pt",
Language.RO: "ro",
Language.RU: "ru",
Language.SA: "sa",
Language.SD: "sd",
Language.SI: "si",
Language.SK: "sk",
Language.SL: "sl",
Language.SN: "sn",
Language.SO: "so",
Language.SQ: "sq",
Language.SR: "sr",
Language.SU: "su",
Language.SV: "sv",
Language.SW: "sw",
Language.TA: "ta",
Language.TE: "te",
Language.TG: "tg",
Language.TH: "th",
Language.TK: "tk",
Language.TL: "tl",
Language.TR: "tr",
Language.TT: "tt",
Language.UK: "uk",
Language.UR: "ur",
Language.UZ: "uz",
Language.VI: "vi",
Language.YI: "yi",
Language.YO: "yo",
Language.ZH: "zh",
}
result = BASE_LANGUAGES.get(language)
# If not found in base languages, try to find the base language from a variant
if not result:
# Convert enum value to string and get the base language part (e.g. es-ES -> es)
lang_str = str(language.value)
base_code = lang_str.split("-")[0].lower()
# Look up the base code in our supported languages
result = base_code if base_code in BASE_LANGUAGES.values() else None
return result
# Deprecation warning for nested InputParams
class _InputParamsDescriptor:
"""Descriptor for backward compatibility with deprecation warning."""
def __get__(self, obj, objtype=None):
warnings.warn(
"GladiaSTTService.InputParams is deprecated and will be removed in a future version. "
"Import and use GladiaInputParams directly instead.",
DeprecationWarning,
stacklevel=2,
)
return GladiaInputParams
class GladiaSTTService(STTService):
"""Speech-to-Text service using Gladia's API.
This service connects to Gladia's WebSocket API for real-time transcription
with support for multiple languages, custom vocabulary, and various processing options.
For complete API documentation, see: https://docs.gladia.io/api-reference/v2/live/init
"""
# Maintain backward compatibility
InputParams = _InputParamsDescriptor()
def __init__(
self,
*,
api_key: str,
url: str = "https://api.gladia.io/v2/live",
confidence: float = 0.5,
sample_rate: Optional[int] = None,
model: str = "solaria-1",
params: GladiaInputParams = GladiaInputParams(),
**kwargs,
):
"""Initialize the Gladia STT service.
Args:
api_key: Gladia API key
url: Gladia API URL
confidence: Minimum confidence threshold for transcriptions
sample_rate: Audio sample rate in Hz
model: Model to use ("solaria-1", "solaria-mini-1", "fast",
or "accurate")
params: Additional configuration parameters
**kwargs: Additional arguments passed to the STTService
"""
super().__init__(sample_rate=sample_rate, **kwargs)
# Warn about deprecated language parameter if it's used
if params.language is not None:
warnings.warn(
"The 'language' parameter is deprecated and will be removed in a future version. "
"Use 'language_config' instead.",
DeprecationWarning,
stacklevel=2,
)
self._api_key = api_key
self._url = url
self.set_model_name(model)
self._confidence = confidence
self._params = params
self._websocket = None
self._receive_task = None
self._keepalive_task = None
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to Gladia's language code."""
return language_to_gladia_language(language)
def _prepare_settings(self) -> Dict[str, Any]:
settings = {
"encoding": self._params.encoding or "wav/pcm",
"bit_depth": self._params.bit_depth or 16,
"sample_rate": self.sample_rate,
"channels": self._params.channels or 1,
"model": self._model_name,
}
# Add custom_metadata if provided
if self._params.custom_metadata:
settings["custom_metadata"] = self._params.custom_metadata
# Add endpointing parameters if provided
if self._params.endpointing is not None:
settings["endpointing"] = self._params.endpointing
if self._params.maximum_duration_without_endpointing is not None:
settings["maximum_duration_without_endpointing"] = (
self._params.maximum_duration_without_endpointing
)
# Add language configuration (prioritize language_config over deprecated language)
if self._params.language_config:
settings["language_config"] = self._params.language_config.model_dump(exclude_none=True)
elif self._params.language: # Backward compatibility for deprecated parameter
language_code = self.language_to_service_language(self._params.language)
if language_code:
settings["language_config"] = {
"languages": [language_code],
"code_switching": False,
}
# Add pre_processing configuration if provided
if self._params.pre_processing:
settings["pre_processing"] = self._params.pre_processing.model_dump(exclude_none=True)
# Add realtime_processing configuration if provided
if self._params.realtime_processing:
settings["realtime_processing"] = self._params.realtime_processing.model_dump(
exclude_none=True
)
# Add messages_config if provided
if self._params.messages_config:
settings["messages_config"] = self._params.messages_config.model_dump(exclude_none=True)
return settings
async def start(self, frame: StartFrame):
"""Start the Gladia STT websocket connection."""
await super().start(frame)
if self._websocket:
return
settings = self._prepare_settings()
response = await self._setup_gladia(settings)
self._websocket = await websockets.connect(response["url"])
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler())
if self._websocket and not self._keepalive_task:
self._keepalive_task = self.create_task(self._keepalive_task_handler())
async def stop(self, frame: EndFrame):
"""Stop the Gladia STT websocket connection."""
await super().stop(frame)
await self._send_stop_recording()
if self._keepalive_task:
await self.cancel_task(self._keepalive_task)
self._keepalive_task = None
if self._websocket:
await self._websocket.close()
self._websocket = None
if self._receive_task:
await self.wait_for_task(self._receive_task)
self._receive_task = None
async def cancel(self, frame: CancelFrame):
"""Cancel the Gladia STT websocket connection."""
await super().cancel(frame)
if self._keepalive_task:
await self.cancel_task(self._keepalive_task)
self._keepalive_task = None
if self._websocket:
await self._websocket.close()
self._websocket = None
if self._receive_task:
await self.cancel_task(self._receive_task)
self._receive_task = None
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Run speech-to-text on audio data."""
await self.start_processing_metrics()
await self._send_audio(audio)
await self.stop_processing_metrics()
yield None
async def _setup_gladia(self, settings: Dict[str, Any]):
async with aiohttp.ClientSession() as session:
async with session.post(
self._url,
headers={"X-Gladia-Key": self._api_key, "Content-Type": "application/json"},
json=settings,
) as response:
if response.ok:
return await response.json()
else:
error_text = await response.text()
logger.error(
f"Gladia error: {response.status}: {error_text or response.reason}"
)
raise Exception(
f"Failed to initialize Gladia session: {response.status} - {error_text}"
)
async def _send_audio(self, audio: bytes):
data = base64.b64encode(audio).decode("utf-8")
message = {"type": "audio_chunk", "data": {"chunk": data}}
await self._websocket.send(json.dumps(message))
async def _send_stop_recording(self):
if self._websocket and not self._websocket.closed:
await self._websocket.send(json.dumps({"type": "stop_recording"}))
async def _keepalive_task_handler(self):
"""Send periodic empty audio chunks to keep the connection alive."""
try:
while True:
# Send keepalive every 20 seconds (Gladia times out after 30 seconds)
await asyncio.sleep(20)
if self._websocket and not self._websocket.closed:
# Send an empty audio chunk as keepalive
empty_audio = b""
await self._send_audio(empty_audio)
else:
logger.debug("Websocket closed, stopping keepalive")
break
except websockets.exceptions.ConnectionClosed:
logger.debug("Connection closed during keepalive")
except Exception as e:
logger.error(f"Error in Gladia keepalive task: {e}")
async def _receive_task_handler(self):
try:
async for message in self._websocket:
content = json.loads(message)
if content["type"] == "transcript":
utterance = content["data"]["utterance"]
confidence = utterance.get("confidence", 0)
transcript = utterance["text"]
if confidence >= self._confidence:
if content["data"]["is_final"]:
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601())
)
else:
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601())
)
except websockets.exceptions.ConnectionClosed:
# Expected when closing the connection
pass
except Exception as e:
logger.error(f"Error in Gladia WebSocket handler: {e}")

2
src/pipecat/services/google/image.py
View File

@@ -17,7 +17,7 @@ from PIL import Image
from pydantic import BaseModel, Field
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.ai_services import ImageGenService
from pipecat.services.image_service import ImageGenService
try:
from google import genai

40
src/pipecat/services/google/llm.py
View File

@@ -9,21 +9,14 @@ import io
import json
import os
import uuid
from google.api_core.exceptions import DeadlineExceeded
from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from dataclasses import dataclass
from typing import Any, Dict, List, Mapping, Optional, Union
from typing import Any, Dict, List, Optional
from loguru import logger
from PIL import Image
from pydantic import BaseModel, Field
from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
@@ -39,23 +32,30 @@ from pipecat.frames.frames import (
VisionImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMUserAggregatorParams,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.services.google.frames import LLMSearchResponseFrame
from pipecat.services.llm_service import LLMService
from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
)
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
try:
import google.ai.generativelanguage as glm
import google.generativeai as gai
from google.api_core.exceptions import DeadlineExceeded
from google.generativeai.types import GenerationConfig
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
@@ -686,8 +686,8 @@ class GoogleLLMService(LLMService):
self,
context: OpenAILLMContext,
*,
user_kwargs: Mapping[str, Any] = {},
assistant_kwargs: Mapping[str, Any] = {},
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> GoogleContextAggregatorPair:
"""Create an instance of GoogleContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
@@ -695,12 +695,10 @@ class GoogleLLMService(LLMService):
Args:
context (OpenAILLMContext): The LLM context.
user_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the user context aggregator constructor. Defaults
to an empty mapping.
assistant_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the assistant context aggregator
constructor. Defaults to an empty mapping.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
Returns:
GoogleContextAggregatorPair: A pair of context aggregators, one for
@@ -712,6 +710,6 @@ class GoogleLLMService(LLMService):
if isinstance(context, OpenAILLMContext):
context = GoogleLLMContext.upgrade_to_google(context)
user = GoogleUserContextAggregator(context, **user_kwargs)
assistant = GoogleAssistantContextAggregator(context, **assistant_kwargs)
user = GoogleUserContextAggregator(context, params=user_params)
assistant = GoogleAssistantContextAggregator(context, params=assistant_params)
return GoogleContextAggregatorPair(_user=user, _assistant=assistant)

13
src/pipecat/services/google/llm_vertex.py
View File

@@ -17,6 +17,8 @@ from loguru import logger
from pipecat.services.openai.llm import OpenAILLMService
try:
from google.auth import default
from google.auth.exceptions import GoogleAuthError
from google.auth.transport.requests import Request
from google.oauth2 import service_account
@@ -65,7 +67,9 @@ class GoogleVertexLLMService(OpenAILLMService):
base_url = self._get_base_url(params)
self._api_key = self._get_api_token(credentials, credentials_path)
super().__init__(api_key=self._api_key, base_url=base_url, model=model, **kwargs)
super().__init__(
api_key=self._api_key, base_url=base_url, model=model, params=params, **kwargs
)
@staticmethod
def _get_base_url(params: InputParams) -> str:
@@ -98,6 +102,13 @@ class GoogleVertexLLMService(OpenAILLMService):
creds = service_account.Credentials.from_service_account_file(
credentials_path, scopes=["https://www.googleapis.com/auth/cloud-platform"]
)
else:
try:
creds, project_id = default(
scopes=["https://www.googleapis.com/auth/cloud-platform"]
)
except GoogleAuthError:
pass
if not creds:
raise ValueError("No valid credentials provided.")

16
src/pipecat/services/google/stt.py
View File

@@ -26,12 +26,14 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.ai_services import STTService
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
try:
from google.api_core.client_options import ClientOptions
from google.auth import default
from google.auth.exceptions import GoogleAuthError
from google.cloud import speech_v2
from google.cloud.speech_v2.types import cloud_speech
from google.oauth2 import service_account
@@ -451,6 +453,7 @@ class GoogleSTTService(STTService):
client_options = ClientOptions(api_endpoint=f"{self._location}-speech.googleapis.com")
# Extract project ID and create client
creds: Optional[service_account.Credentials] = None
if credentials:
json_account_info = json.loads(credentials)
self._project_id = json_account_info.get("project_id")
@@ -461,7 +464,16 @@ class GoogleSTTService(STTService):
self._project_id = json_account_info.get("project_id")
creds = service_account.Credentials.from_service_account_file(credentials_path)
else:
raise ValueError("Either credentials or credentials_path must be provided")
try:
creds, project_id = default(
scopes=["https://www.googleapis.com/auth/cloud-platform"]
)
self._project_id = project_id
except GoogleAuthError:
pass
if not creds:
raise ValueError("No valid credentials provided.")
if not self._project_id:
raise ValueError("Project ID not found in credentials")

20
src/pipecat/services/google/tts.py
View File

@@ -23,10 +23,12 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
try:
from google.auth import default
from google.auth.exceptions import GoogleAuthError
from google.cloud import texttospeech_v1
from google.oauth2 import service_account
@@ -251,6 +253,16 @@ class GoogleTTSService(TTSService):
elif credentials_path:
# Use service account JSON file if provided
creds = service_account.Credentials.from_service_account_file(credentials_path)
else:
try:
creds, project_id = default(
scopes=["https://www.googleapis.com/auth/cloud-platform"]
)
except GoogleAuthError:
pass
if not creds:
raise ValueError("No valid credentials provided.")
return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds)
@@ -346,9 +358,9 @@ class GoogleTTSService(TTSService):
audio_content = response.audio_content[44:]
# Read and yield audio data in chunks
chunk_size = 8192
for i in range(0, len(audio_content), chunk_size):
chunk = audio_content[i : i + chunk_size]
CHUNK_SIZE = 1024
for i in range(0, len(audio_content), CHUNK_SIZE):
chunk = audio_content[i : i + CHUNK_SIZE]
if not chunk:
break
await self.stop_ttfb_metrics()

27
src/pipecat/services/grok/llm.py
View File

@@ -5,11 +5,14 @@
#
from dataclasses import dataclass
from typing import Any, Mapping
from loguru import logger
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMUserAggregatorParams,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
@@ -39,7 +42,7 @@ class GrokLLMService(OpenAILLMService):
Args:
api_key (str): The API key for accessing Grok's API
base_url (str, optional): The base URL for Grok API. Defaults to "https://api.x.ai/v1"
model (str, optional): The model identifier to use. Defaults to "grok-2"
model (str, optional): The model identifier to use. Defaults to "grok-3-beta"
**kwargs: Additional keyword arguments passed to OpenAILLMService
"""
@@ -48,7 +51,7 @@ class GrokLLMService(OpenAILLMService):
*,
api_key: str,
base_url: str = "https://api.x.ai/v1",
model: str = "grok-2",
model: str = "grok-3-beta",
**kwargs,
):
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
@@ -124,8 +127,8 @@ class GrokLLMService(OpenAILLMService):
self,
context: OpenAILLMContext,
*,
user_kwargs: Mapping[str, Any] = {},
assistant_kwargs: Mapping[str, Any] = {},
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> GrokContextAggregatorPair:
"""Create an instance of GrokContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
@@ -133,12 +136,10 @@ class GrokLLMService(OpenAILLMService):
Args:
context (OpenAILLMContext): The LLM context.
user_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the user context aggregator constructor. Defaults
to an empty mapping.
assistant_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the assistant context aggregator
constructor. Defaults to an empty mapping.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
Returns:
GrokContextAggregatorPair: A pair of context aggregators, one for
@@ -148,6 +149,6 @@ class GrokLLMService(OpenAILLMService):
"""
context.set_llm_adapter(self.get_llm_adapter())
user = OpenAIUserContextAggregator(context, **user_kwargs)
assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs)
user = OpenAIUserContextAggregator(context, params=user_params)
assistant = OpenAIAssistantContextAggregator(context, params=assistant_params)
return GrokContextAggregatorPair(_user=user, _assistant=assistant)

2
src/pipecat/services/groq/tts.py
View File

@@ -10,7 +10,7 @@ from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
try:

33
src/pipecat/services/image_service.py Normal file
View File

@@ -0,0 +1,33 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from abc import abstractmethod
from typing import AsyncGenerator
from pipecat.frames.frames import Frame, TextFrame
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
class ImageGenService(AIService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Renders the image. Returns an Image object.
@abstractmethod
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
pass
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame):
await self.push_frame(frame, direction)
await self.start_processing_metrics()
await self.process_generator(self.run_image_gen(frame.text))
await self.stop_processing_metrics()
else:
await self.push_frame(frame, direction)

261
src/pipecat/services/llm_service.py Normal file
View File

@@ -0,0 +1,261 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from dataclasses import dataclass
from typing import Any, Optional, Set, Tuple, Type
from loguru import logger
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter
from pipecat.frames.frames import (
Frame,
FunctionCallCancelFrame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
StartInterruptionFrame,
UserImageRequestFrame,
)
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMUserAggregatorParams,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
@dataclass
class FunctionEntry:
function_name: Optional[str]
callback: Any # TODO(aleix): add proper typing.
cancel_on_interruption: bool
class LLMService(AIService):
"""This class is a no-op but serves as a base class for LLM services."""
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
# However, subclasses should override this with a more specific adapter when necessary.
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._functions = {}
self._start_callbacks = {}
self._adapter = self.adapter_class()
self._function_call_tasks: Set[Tuple[asyncio.Task, str, str]] = set()
self._register_event_handler("on_completion_timeout")
def get_llm_adapter(self) -> BaseLLMAdapter:
return self._adapter
def create_context_aggregator(
self,
context: OpenAILLMContext,
*,
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> Any:
pass
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame):
await self._handle_interruptions(frame)
async def _handle_interruptions(self, frame: StartInterruptionFrame):
for function_name, entry in self._functions.items():
if entry.cancel_on_interruption:
await self._cancel_function_call(function_name)
def register_function(
self,
function_name: Optional[str],
callback: Any,
start_callback=None,
*,
cancel_on_interruption: bool = False,
):
# Registering a function with the function_name set to None will run that callback
# for all functions
self._functions[function_name] = FunctionEntry(
function_name=function_name,
callback=callback,
cancel_on_interruption=cancel_on_interruption,
)
# Start callbacks are now deprecated.
if start_callback:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'start_callback' is deprecated, just put your code on top of the actual function call instead.",
DeprecationWarning,
)
self._start_callbacks[function_name] = start_callback
def unregister_function(self, function_name: Optional[str]):
del self._functions[function_name]
if self._start_callbacks[function_name]:
del self._start_callbacks[function_name]
def has_function(self, function_name: str):
if None in self._functions.keys():
return True
return function_name in self._functions.keys()
async def call_function(
self,
*,
context: OpenAILLMContext,
tool_call_id: str,
function_name: str,
arguments: str,
run_llm: bool = True,
):
if not function_name in self._functions.keys() and not None in self._functions.keys():
return
task = self.create_task(
self._run_function_call(context, tool_call_id, function_name, arguments, run_llm)
)
self._function_call_tasks.add((task, tool_call_id, function_name))
task.add_done_callback(self._function_call_task_finished)
async def call_start_function(self, context: OpenAILLMContext, function_name: str):
if function_name in self._start_callbacks.keys():
await self._start_callbacks[function_name](function_name, self, context)
elif None in self._start_callbacks.keys():
return await self._start_callbacks[None](function_name, self, context)
async def request_image_frame(
self,
user_id: str,
*,
function_name: Optional[str] = None,
tool_call_id: Optional[str] = None,
text_content: Optional[str] = None,
):
await self.push_frame(
UserImageRequestFrame(
user_id=user_id,
function_name=function_name,
tool_call_id=tool_call_id,
context=text_content,
),
FrameDirection.UPSTREAM,
)
async def _run_function_call(
self,
context: OpenAILLMContext,
tool_call_id: str,
function_name: str,
arguments: str,
run_llm: bool = True,
):
if function_name in self._functions.keys():
entry = self._functions[function_name]
elif None in self._functions.keys():
entry = self._functions[None]
else:
return
logger.debug(
f"{self} Calling function [{function_name}:{tool_call_id}] with arguments {arguments}"
)
# NOTE(aleix): This needs to be removed after we remove the deprecation.
await self.call_start_function(context, function_name)
# Push a SystemFrame downstream. This frame will let our assistant context aggregator
# know that we are in the middle of a function call. Some contexts/aggregators may
# not need this. But some definitely do (Anthropic, for example).
# Also push a SystemFrame upstream for use by other processors, like STTMuteFilter.
progress_frame_downstream = FunctionCallInProgressFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
cancel_on_interruption=entry.cancel_on_interruption,
)
progress_frame_upstream = FunctionCallInProgressFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
cancel_on_interruption=entry.cancel_on_interruption,
)
# Push frame both downstream and upstream
await self.push_frame(progress_frame_downstream, FrameDirection.DOWNSTREAM)
await self.push_frame(progress_frame_upstream, FrameDirection.UPSTREAM)
# Define a callback function that pushes a FunctionCallResultFrame upstream & downstream.
async def function_call_result_callback(result, *, properties=None):
result_frame_downstream = FunctionCallResultFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
result=result,
properties=properties,
)
result_frame_upstream = FunctionCallResultFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
result=result,
properties=properties,
)
await self.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM)
await self.push_frame(result_frame_upstream, FrameDirection.UPSTREAM)
await entry.callback(
function_name, tool_call_id, arguments, self, context, function_call_result_callback
)
async def _cancel_function_call(self, function_name: str):
cancelled_tasks = set()
for task, tool_call_id, name in self._function_call_tasks:
if name == function_name:
# We remove the callback because we are going to cancel the task
# now, otherwise we will be removing it from the set while we
# are iterating.
task.remove_done_callback(self._function_call_task_finished)
logger.debug(f"{self} Cancelling function call [{name}:{tool_call_id}]...")
await self.cancel_task(task)
frame = FunctionCallCancelFrame(
function_name=function_name, tool_call_id=tool_call_id
)
await self.push_frame(frame)
logger.debug(f"{self} Function call [{name}:{tool_call_id}] has been cancelled")
cancelled_tasks.add(task)
# Remove all cancelled tasks from our set.
for task in cancelled_tasks:
self._function_call_task_finished(task)
def _function_call_task_finished(self, task: asyncio.Task):
tuple_to_remove = next((t for t in self._function_call_tasks if t[0] == task), None)
if tuple_to_remove:
self._function_call_tasks.discard(tuple_to_remove)
# The task is finished so this should exit immediately. We need to
# do this because otherwise the task manager would report a dangling
# task if we don't remove it.
asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop())

16
src/pipecat/services/lmnt/tts.py
View File

@@ -21,7 +21,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import InterruptibleTTSService
from pipecat.services.tts_service import InterruptibleTTSService
from pipecat.transcriptions.language import Language
# See .env.example for LMNT configuration needed
@@ -109,7 +109,7 @@ class LmntTTSService(InterruptibleTTSService):
async def _connect(self):
await self._connect_websocket()
if not self._receive_task:
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self):
@@ -122,7 +122,7 @@ class LmntTTSService(InterruptibleTTSService):
async def _connect_websocket(self):
"""Connect to LMNT websocket."""
try:
if self._websocket:
if self._websocket and self._websocket.open:
return
logger.debug("Connecting to LMNT")
@@ -158,11 +158,11 @@ class LmntTTSService(InterruptibleTTSService):
# errors on the websocket, so we just skip it for now.
# await self._websocket.send(json.dumps({"eof": True}))
await self._websocket.close()
self._websocket = None
self._started = False
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
finally:
self._started = False
self._websocket = None
def _get_websocket(self):
if self._websocket:
@@ -170,7 +170,7 @@ class LmntTTSService(InterruptibleTTSService):
raise Exception("Websocket not connected")
async def flush_audio(self):
if not self._websocket:
if not self._websocket or self._websocket.closed:
return
await self._get_websocket().send(json.dumps({"flush": True}))
@@ -203,7 +203,7 @@ class LmntTTSService(InterruptibleTTSService):
logger.debug(f"{self}: Generating TTS [{text}]")
try:
if not self._websocket:
if not self._websocket or self._websocket.closed:
await self._connect()
try:

213
src/pipecat/services/mcp_service.py Normal file
View File

@@ -0,0 +1,213 @@
import json
from typing import Any, Dict, List, Optional, Union
from loguru import logger
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.utils.base_object import BaseObject
try:
from mcp import ClientSession, StdioServerParameters, types
from mcp.client.session import ClientSession
from mcp.client.sse import sse_client
from mcp.client.stdio import stdio_client
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use an MCP client, you need to `pip install pipecat-ai[mcp]`.")
raise Exception(f"Missing module: {e}")
class MCPClient(BaseObject):
def __init__(
self,
server_params: Union[StdioServerParameters, str],
**kwargs,
):
super().__init__(**kwargs)
self._server_params = server_params
self._session = ClientSession
if isinstance(server_params, StdioServerParameters):
self._client = stdio_client
self._register_tools = self._stdio_register_tools
elif isinstance(server_params, str):
self._client = sse_client
self._register_tools = self._sse_register_tools
else:
raise TypeError(
f"{self} invalid argument type: `server_params` must be either StdioServerParameters or an SSE server url string."
)
async def register_tools(self, llm) -> ToolsSchema:
tools_schema = await self._register_tools(llm)
return tools_schema
def _convert_mcp_schema_to_pipecat(
self, tool_name: str, tool_schema: Dict[str, Any]
) -> FunctionSchema:
"""Convert an mcp tool schema to Pipecat's FunctionSchema format.
Args:
tool_name: The name of the tool
tool_schema: The mcp tool schema
Returns:
A FunctionSchema instance
"""
logger.debug(f"Converting schema for tool '{tool_name}'")
logger.trace(f"Original schema: {json.dumps(tool_schema, indent=2)}")
properties = tool_schema["input_schema"].get("properties", {})
required = tool_schema["input_schema"].get("required", [])
schema = FunctionSchema(
name=tool_name,
description=tool_schema["description"],
properties=properties,
required=required,
)
logger.trace(f"Converted schema: {json.dumps(schema.to_default_dict(), indent=2)}")
return schema
async def _sse_register_tools(self, llm) -> ToolsSchema:
"""Register all available mcp.run tools with the LLM service.
Args:
llm: The Pipecat LLM service to register tools with
Returns:
A ToolsSchema containing all registered tools
"""
async def mcp_tool_wrapper(
function_name: str,
tool_call_id: str,
arguments: Dict[str, Any],
llm: any,
context: any,
result_callback: any,
) -> None:
"""Wrapper for mcp.run tool calls to match Pipecat's function call interface."""
logger.debug(f"Executing tool '{function_name}' with call ID: {tool_call_id}")
logger.trace(f"Tool arguments: {json.dumps(arguments, indent=2)}")
try:
async with self._client(self._server_params) as (read, write):
async with self._session(read, write) as session:
await session.initialize()
await self._call_tool(session, function_name, arguments, result_callback)
except Exception as e:
error_msg = f"Error calling mcp tool {function_name}: {str(e)}"
logger.error(error_msg)
logger.exception("Full exception details:")
await result_callback(error_msg)
logger.debug("Starting registration of mcp.run tools")
tool_schemas: List[FunctionSchema] = []
async with self._client(self._server_params) as (read, write):
async with self._session(read, write) as session:
await session.initialize()
tools_schema = await self._list_tools(session, mcp_tool_wrapper, llm)
return tools_schema
async def _stdio_register_tools(self, llm) -> ToolsSchema:
"""Register all available mcp.run tools with the LLM service.
Args:
llm: The Pipecat LLM service to register tools with
Returns:
A ToolsSchema containing all registered tools
"""
async def mcp_tool_wrapper(
function_name: str,
tool_call_id: str,
arguments: Dict[str, Any],
llm: any,
context: any,
result_callback: any,
) -> None:
"""Wrapper for mcp.run tool calls to match Pipecat's function call interface."""
logger.debug(f"Executing tool '{function_name}' with call ID: {tool_call_id}")
logger.trace(f"Tool arguments: {json.dumps(arguments, indent=2)}")
try:
async with self._client(self._server_params) as streams:
async with self._session(streams[0], streams[1]) as session:
await session.initialize()
await self._call_tool(session, function_name, arguments, result_callback)
except Exception as e:
error_msg = f"Error calling mcp tool {function_name}: {str(e)}"
logger.error(error_msg)
logger.exception("Full exception details:")
await result_callback(error_msg)
logger.debug("Starting registration of mcp.run tools")
async with self._client(self._server_params) as streams:
async with self._session(streams[0], streams[1]) as session:
await session.initialize()
tools_schema = await self._list_tools(session, mcp_tool_wrapper, llm)
return tools_schema
async def _call_tool(self, session, function_name, arguments, result_callback):
logger.debug(f"Calling mcp tool '{function_name}'")
try:
results = await session.call_tool(function_name, arguments=arguments)
except Exception as e:
error_msg = f"Error calling mcp tool {function_name}: {str(e)}"
logger.error(error_msg)
response = "Sorry, could not call the mcp tool"
image_url = None
if results:
if hasattr(results, "content") and results.content:
for i, content in enumerate(results.content):
if hasattr(content, "text") and content.text:
logger.debug(f"Tool response chunk {i}: {content.text}")
response += content.text
else:
# logger.debug(f"Non-text result content: '{content}'")
pass
logger.info(f"Tool '{function_name}' completed successfully")
logger.debug(f"Final response: {response}")
else:
logger.error(f"Error getting content from {function_name} results.")
await result_callback(response)
async def _list_tools(self, session, mcp_tool_wrapper, llm):
available_tools = await session.list_tools()
tool_schemas: List[FunctionSchema] = []
try:
logger.debug(f"Found {len(available_tools)} available tools")
except:
pass
for tool in available_tools.tools:
tool_name = tool.name
logger.debug(f"Processing tool: {tool_name}")
logger.debug(f"Tool description: {tool.description}")
try:
# Convert the schema
function_schema = self._convert_mcp_schema_to_pipecat(
tool_name,
{"description": tool.description, "input_schema": tool.inputSchema},
)
# Register the wrapped function
logger.debug(f"Registering function handler for '{tool_name}'")
llm.register_function(tool_name, mcp_tool_wrapper)
# Add to list of schemas
tool_schemas.append(function_schema)
logger.debug(f"Successfully registered tool '{tool_name}'")
except Exception as e:
logger.error(f"Failed to register tool '{tool_name}': {str(e)}")
logger.exception("Full exception details:")
continue
logger.debug(f"Completed registration of {len(tool_schemas)} tools")
tools_schema = ToolsSchema(standard_tools=tool_schemas)
return tools_schema

2
src/pipecat/services/moondream/vision.py
View File

@@ -11,7 +11,7 @@ from loguru import logger
from PIL import Image
from pipecat.frames.frames import ErrorFrame, Frame, TextFrame, VisionImageRawFrame
from pipecat.services.ai_services import VisionService
from pipecat.services.vision_service import VisionService
try:
import torch

24
src/pipecat/services/neuphonic/tts.py
View File

@@ -27,7 +27,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import InterruptibleTTSService, TTSService
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language
try:
@@ -106,6 +106,9 @@ class NeuphonicTTSService(InterruptibleTTSService):
self._started = False
self._cumulative_time = 0
self._receive_task = None
self._keepalive_task = None
def can_generate_metrics(self) -> bool:
return True
@@ -159,8 +162,11 @@ class NeuphonicTTSService(InterruptibleTTSService):
async def _connect(self):
await self._connect_websocket()
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
self._keepalive_task = self.create_task(self._keepalive_task_handler())
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
if self._websocket and not self._keepalive_task:
self._keepalive_task = self.create_task(self._keepalive_task_handler())
async def _disconnect(self):
if self._receive_task:
@@ -175,6 +181,9 @@ class NeuphonicTTSService(InterruptibleTTSService):
async def _connect_websocket(self):
try:
if self._websocket and self._websocket.open:
return
logger.debug("Connecting to Neuphonic")
tts_config = {
@@ -190,7 +199,6 @@ class NeuphonicTTSService(InterruptibleTTSService):
url = f"{self._url}/speak/{self._settings['lang_code']}?{'&'.join(query_params)}"
self._websocket = await websockets.connect(url)
except Exception as e:
logger.error(f"{self} initialization error: {e}")
self._websocket = None
@@ -203,11 +211,11 @@ class NeuphonicTTSService(InterruptibleTTSService):
if self._websocket:
logger.debug("Disconnecting from Neuphonic")
await self._websocket.close()
self._websocket = None
self._started = False
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
finally:
self._started = False
self._websocket = None
async def _receive_messages(self):
async for message in self._websocket:
@@ -235,7 +243,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
logger.debug(f"Generating TTS: [{text}]")
try:
if not self._websocket:
if not self._websocket or self._websocket.closed:
await self._connect()
try:

2
src/pipecat/services/openai/base_llm.py
View File

@@ -34,7 +34,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.services.llm_service import LLMService
class OpenAIUnhandledFunctionException(Exception):

2
src/pipecat/services/openai/image.py
View File

@@ -17,7 +17,7 @@ from pipecat.frames.frames import (
Frame,
URLImageRawFrame,
)
from pipecat.services.ai_services import ImageGenService
from pipecat.services.image_service import ImageGenService
class OpenAIImageGenService(ImageGenService):

22
src/pipecat/services/openai/llm.py
View File

@@ -6,7 +6,7 @@
import json
from dataclasses import dataclass
from typing import Any, Mapping
from typing import Any
from pipecat.frames.frames import (
FunctionCallCancelFrame,
@@ -15,7 +15,9 @@ from pipecat.frames.frames import (
UserImageRawFrame,
)
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMAssistantContextAggregator,
LLMUserAggregatorParams,
LLMUserContextAggregator,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
@@ -38,7 +40,7 @@ class OpenAILLMService(BaseOpenAILLMService):
def __init__(
self,
*,
model: str = "gpt-4o",
model: str = "gpt-4.1",
params: BaseOpenAILLMService.InputParams = BaseOpenAILLMService.InputParams(),
**kwargs,
):
@@ -48,8 +50,8 @@ class OpenAILLMService(BaseOpenAILLMService):
self,
context: OpenAILLMContext,
*,
user_kwargs: Mapping[str, Any] = {},
assistant_kwargs: Mapping[str, Any] = {},
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> OpenAIContextAggregatorPair:
"""Create an instance of OpenAIContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
@@ -57,12 +59,8 @@ class OpenAILLMService(BaseOpenAILLMService):
Args:
context (OpenAILLMContext): The LLM context.
user_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the user context aggregator constructor. Defaults
to an empty mapping.
assistant_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the assistant context aggregator
constructor. Defaults to an empty mapping.
user_params (LLMUserAggregatorParams, optional): User aggregator parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User aggregator parameters.
Returns:
OpenAIContextAggregatorPair: A pair of context aggregators, one for
@@ -71,8 +69,8 @@ class OpenAILLMService(BaseOpenAILLMService):
"""
context.set_llm_adapter(self.get_llm_adapter())
user = OpenAIUserContextAggregator(context, **user_kwargs)
assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs)
user = OpenAIUserContextAggregator(context, params=user_params)
assistant = OpenAIAssistantContextAggregator(context, params=assistant_params)
return OpenAIContextAggregatorPair(_user=user, _assistant=assistant)

15
src/pipecat/services/openai/tts.py
View File

@@ -17,17 +17,24 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
ValidVoice = Literal[
"alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse"
]
VALID_VOICES: Dict[str, ValidVoice] = {
"alloy": "alloy",
"ash": "ash",
"ballad": "ballad",
"coral": "coral",
"echo": "echo",
"fable": "fable",
"onyx": "onyx",
"nova": "nova",
"sage": "sage",
"shimmer": "shimmer",
"verse": "verse",
}
@@ -63,7 +70,7 @@ class OpenAITTSService(TTSService):
if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE:
logger.warning(
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {self.sample_rate}Hz may cause issues."
f"Current rate of {sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
@@ -98,7 +105,7 @@ class OpenAITTSService(TTSService):
extra_body["instructions"] = self._instructions
async with self._client.audio.speech.with_streaming_response.create(
input=text or " ", # Text must contain at least one character
input=text,
model=self.model_name,
voice=VALID_VOICES[self._voice_id],
response_format="pcm",

44
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -8,19 +8,9 @@ import base64
import json
import time
from dataclasses import dataclass
from typing import Any, Mapping
from loguru import logger
try:
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter
from pipecat.frames.frames import (
BotStoppedSpeakingFrame,
@@ -48,12 +38,16 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMUserAggregatorParams,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.services.llm_service import LLMService
from pipecat.services.openai.llm import OpenAIContextAggregatorPair
from pipecat.utils.time import time_now_iso8601
@@ -65,6 +59,13 @@ from .context import (
)
from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame
try:
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use OpenAI, you need to `pip install pipecat-ai[openai]`.")
raise Exception(f"Missing module: {e}")
@dataclass
class CurrentAudioResponse:
@@ -650,8 +651,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self,
context: OpenAILLMContext,
*,
user_kwargs: Mapping[str, Any] = {},
assistant_kwargs: Mapping[str, Any] = {},
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> OpenAIContextAggregatorPair:
"""Create an instance of OpenAIContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
@@ -659,12 +660,10 @@ class OpenAIRealtimeBetaLLMService(LLMService):
Args:
context (OpenAILLMContext): The LLM context.
user_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the user context aggregator constructor. Defaults
to an empty mapping.
assistant_kwargs (Mapping[str, Any], optional): Additional keyword
arguments for the assistant context aggregator
constructor. Defaults to an empty mapping.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
Returns:
OpenAIContextAggregatorPair: A pair of context aggregators, one for
@@ -675,9 +674,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
context.set_llm_adapter(self.get_llm_adapter())
OpenAIRealtimeLLMContext.upgrade_to_realtime(context)
user = OpenAIRealtimeUserContextAggregator(context, **user_kwargs)
user = OpenAIRealtimeUserContextAggregator(context, params=user_params)
default_assistant_kwargs = {"expect_stripped_words": False}
default_assistant_kwargs.update(assistant_kwargs)
assistant = OpenAIRealtimeAssistantContextAggregator(context, **default_assistant_kwargs)
assistant_params.expect_stripped_words = False
assistant = OpenAIRealtimeAssistantContextAggregator(context, params=assistant_params)
return OpenAIContextAggregatorPair(_user=user, _assistant=assistant)

2
src/pipecat/services/openpipe/llm.py
View File

@@ -25,7 +25,7 @@ class OpenPipeLLMService(OpenAILLMService):
def __init__(
self,
*,
model: str = "gpt-4o",
model: str = "gpt-4.1",
api_key: Optional[str] = None,
base_url: Optional[str] = None,
openpipe_api_key: Optional[str] = None,

2
src/pipecat/services/piper/tts.py
View File

@@ -16,7 +16,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
# This assumes a running TTS service running: https://github.com/rhasspy/piper/blob/master/src/python_run/README_http.md

12
src/pipecat/services/playht/tts.py
View File

@@ -27,7 +27,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import InterruptibleTTSService, TTSService
from pipecat.services.tts_service import InterruptibleTTSService, TTSService
from pipecat.transcriptions.language import Language
try:
@@ -157,7 +157,7 @@ class PlayHTTTSService(InterruptibleTTSService):
async def _connect(self):
await self._connect_websocket()
if not self._receive_task:
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self):
@@ -169,7 +169,7 @@ class PlayHTTTSService(InterruptibleTTSService):
async def _connect_websocket(self):
try:
if self._websocket:
if self._websocket and self._websocket.open:
return
logger.debug("Connecting to PlayHT")
@@ -197,11 +197,11 @@ class PlayHTTTSService(InterruptibleTTSService):
if self._websocket:
logger.debug("Disconnecting from PlayHT")
await self._websocket.close()
self._websocket = None
self._request_id = None
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
finally:
self._request_id = None
self._websocket = None
async def _get_websocket_url(self):
async with aiohttp.ClientSession() as session:

13
src/pipecat/services/rime/tts.py
View File

@@ -25,7 +25,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AudioContextWordTTSService, TTSService
from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
@@ -168,7 +168,7 @@ class RimeTTSService(AudioContextWordTTSService):
"""Establish websocket connection and start receive task."""
await self._connect_websocket()
if not self._receive_task:
if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self):
@@ -182,7 +182,7 @@ class RimeTTSService(AudioContextWordTTSService):
async def _connect_websocket(self):
"""Connect to Rime websocket API with configured settings."""
try:
if self._websocket:
if self._websocket and self._websocket.open:
return
params = "&".join(f"{k}={v}" for k, v in self._settings.items())
@@ -201,10 +201,11 @@ class RimeTTSService(AudioContextWordTTSService):
if self._websocket:
await self._websocket.send(json.dumps(self._build_eos_msg()))
await self._websocket.close()
self._websocket = None
self._context_id = None
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
finally:
self._context_id = None
self._websocket = None
def _get_websocket(self):
"""Get active websocket connection or raise exception."""
@@ -316,7 +317,7 @@ class RimeTTSService(AudioContextWordTTSService):
"""
logger.debug(f"{self}: Generating TTS [{text}]")
try:
if not self._websocket:
if not self._websocket or self._websocket.closed:
await self._connect()
try:

2
src/pipecat/services/riva/stt.py
View File

@@ -18,7 +18,7 @@ from pipecat.frames.frames import (
StartFrame,
TranscriptionFrame,
)
from pipecat.services.ai_services import STTService
from pipecat.services.stt_service import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601

2
src/pipecat/services/riva/tts.py
View File

@@ -16,7 +16,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
try:

171
src/pipecat/services/stt_service.py Normal file
View File

@@ -0,0 +1,171 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import io
import wave
from abc import abstractmethod
from typing import Any, AsyncGenerator, Dict, Mapping, Optional
from loguru import logger
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
StartFrame,
STTMuteFrame,
STTUpdateSettingsFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.transcriptions.language import Language
class STTService(AIService):
"""STTService is a base class for speech-to-text services."""
def __init__(
self,
audio_passthrough=True,
# STT input sample rate
sample_rate: Optional[int] = None,
**kwargs,
):
super().__init__(**kwargs)
self._audio_passthrough = audio_passthrough
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._settings: Dict[str, Any] = {}
self._muted: bool = False
@property
def is_muted(self) -> bool:
"""Returns whether the STT service is currently muted."""
return self._muted
@property
def sample_rate(self) -> int:
return self._sample_rate
async def set_model(self, model: str):
self.set_model_name(model)
async def set_language(self, language: Language):
pass
@abstractmethod
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Returns transcript as a string"""
pass
async def start(self, frame: StartFrame):
await super().start(frame)
self._sample_rate = self._init_sample_rate or frame.audio_in_sample_rate
async def _update_settings(self, settings: Mapping[str, Any]):
logger.info(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating STT setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
await self.set_language(value)
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for STT service: {key}")
async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
if self._muted:
return
await self.process_generator(self.run_stt(frame.audio))
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Processes a frame of audio data, either buffering or transcribing it."""
await super().process_frame(frame, direction)
if isinstance(frame, AudioRawFrame):
# In this service we accumulate audio internally and at the end we
# push a TextFrame. We also push audio downstream in case someone
# else needs it.
await self.process_audio_frame(frame, direction)
if self._audio_passthrough:
await self.push_frame(frame, direction)
elif isinstance(frame, STTUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, STTMuteFrame):
self._muted = frame.mute
logger.debug(f"STT service {'muted' if frame.mute else 'unmuted'}")
else:
await self.push_frame(frame, direction)
class SegmentedSTTService(STTService):
"""SegmentedSTTService is an STTService that uses VAD events to detect
speech and will run speech-to-text on speech segments only, instead of a
continous stream. Since it uses VAD it means that VAD needs to be enabled in
the pipeline.
This service always keeps a small audio buffer to take into account that VAD
events are delayed from when the user speech really starts.
"""
def __init__(self, *, sample_rate: Optional[int] = None, **kwargs):
super().__init__(sample_rate=sample_rate, **kwargs)
self._content = None
self._wave = None
self._audio_buffer = bytearray()
self._audio_buffer_size_1s = 0
self._user_speaking = False
async def start(self, frame: StartFrame):
await super().start(frame)
self._audio_buffer_size_1s = self.sample_rate * 2
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame):
await self._handle_user_started_speaking(frame)
elif isinstance(frame, UserStoppedSpeakingFrame):
await self._handle_user_stopped_speaking(frame)
async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame):
if frame.emulated:
return
self._user_speaking = True
async def _handle_user_stopped_speaking(self, frame: UserStoppedSpeakingFrame):
if frame.emulated:
return
self._user_speaking = False
content = io.BytesIO()
wav = wave.open(content, "wb")
wav.setsampwidth(2)
wav.setnchannels(1)
wav.setframerate(self.sample_rate)
wav.writeframes(self._audio_buffer)
wav.close()
content.seek(0)
await self.process_generator(self.run_stt(content.read()))
# Start clean.
self._audio_buffer.clear()
async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
# If the user is speaking the audio buffer will keep growing.
self._audio_buffer += frame.audio
# If the user is not speaking we keep just a little bit of audio.
if not self._user_speaking and len(self._audio_buffer) > self._audio_buffer_size_1s:
discarded = len(self._audio_buffer) - self._audio_buffer_size_1s
self._audio_buffer = self._audio_buffer[discarded:]

106
src/pipecat/services/tavus/video.py
View File

@@ -6,7 +6,9 @@
"""This module implements Tavus as a sink transport layer"""
import asyncio
import base64
from typing import Optional
import aiohttp
from loguru import logger
@@ -16,6 +18,7 @@ from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
StartFrame,
StartInterruptionFrame,
TransportMessageUrgentFrame,
TTSAudioRawFrame,
@@ -23,7 +26,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AIService
from pipecat.services.ai_service import AIService
class TavusVideoService(AIService):
@@ -50,6 +53,10 @@ class TavusVideoService(AIService):
self._resampler = create_default_resampler()
self._audio_buffer = bytearray()
self._queue = asyncio.Queue()
self._send_task: Optional[asyncio.Task] = None
async def initialize(self) -> str:
url = "https://tavusapi.com/v2/conversations"
headers = {"Content-Type": "application/json", "x-api-key": self._api_key}
@@ -78,45 +85,98 @@ class TavusVideoService(AIService):
logger.debug(f"TavusVideoService persona grabbed {response_json}")
return response_json["persona_name"]
async def start(self, frame: StartFrame):
await super().start(frame)
await self._create_send_task()
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._end_conversation()
await self._cancel_send_task()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._end_conversation()
await self._cancel_send_task()
async def _end_conversation(self) -> None:
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame):
await self._handle_interruptions()
await self.push_frame(frame, direction)
elif isinstance(frame, TTSStartedFrame):
await self.start_processing_metrics()
await self.start_ttfb_metrics()
self._current_idx_str = str(frame.id)
elif isinstance(frame, TTSAudioRawFrame):
await self._queue_audio(frame.audio, frame.sample_rate, done=False)
elif isinstance(frame, TTSStoppedFrame):
await self._queue_audio(b"\x00\x00", self._sample_rate, done=True)
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
else:
await self.push_frame(frame, direction)
async def _handle_interruptions(self):
await self._cancel_send_task()
await self._create_send_task()
await self._send_interrupt_message()
async def _end_conversation(self):
url = f"https://tavusapi.com/v2/conversations/{self._conversation_id}/end"
headers = {"Content-Type": "application/json", "x-api-key": self._api_key}
async with self._session.post(url, headers=headers) as r:
r.raise_for_status()
async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None:
async def _queue_audio(self, audio: bytes, in_rate: int, done: bool):
await self._queue.put((audio, in_rate, done))
async def _create_send_task(self):
if not self._send_task:
self._queue = asyncio.Queue()
self._send_task = self.create_task(self._send_task_handler())
async def _cancel_send_task(self):
if self._send_task:
await self.cancel_task(self._send_task)
self._send_task = None
async def _send_task_handler(self):
# Daily app-messages have a 4kb limit and also a rate limit of 20
# messages per second. Below, we only consider the rate limit because 1
# second of a 24000 sample rate would be 48000 bytes (16-bit samples and
# 1 channel). So, that is 48000 / 20 = 2400, which is below the 4kb
# limit (even including base64 encoding). For a sample rate of 16000,
# that would be 32000 / 20 = 1600.
MAX_CHUNK_SIZE = int((self._sample_rate * 2) / 20)
SLEEP_TIME = 1 / 20
audio_buffer = bytearray()
while True:
(audio, in_rate, done) = await self._queue.get()
if done:
# Send any remaining audio.
if len(audio_buffer) > 0:
await self._encode_audio_and_send(bytes(audio_buffer), done)
await self._encode_audio_and_send(audio, done)
audio_buffer.clear()
else:
audio = await self._resampler.resample(audio, in_rate, self._sample_rate)
audio_buffer.extend(audio)
while len(audio_buffer) >= MAX_CHUNK_SIZE:
chunk = audio_buffer[:MAX_CHUNK_SIZE]
audio_buffer = audio_buffer[MAX_CHUNK_SIZE:]
await self._encode_audio_and_send(bytes(chunk), done)
await asyncio.sleep(SLEEP_TIME)
async def _encode_audio_and_send(self, audio: bytes, done: bool):
"""Encodes audio to base64 and sends it to Tavus"""
if not done:
audio = await self._resampler.resample(audio, in_rate, self._sample_rate)
audio_base64 = base64.b64encode(audio).decode("utf-8")
logger.trace(f"{self}: sending {len(audio)} bytes")
await self._send_audio_message(audio_base64, done=done)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TTSStartedFrame):
await self.start_processing_metrics()
await self.start_ttfb_metrics()
self._current_idx_str = str(frame.id)
elif isinstance(frame, TTSAudioRawFrame):
await self._encode_audio_and_send(frame.audio, frame.sample_rate, done=False)
elif isinstance(frame, TTSStoppedFrame):
await self._encode_audio_and_send(b"\x00", self._sample_rate, done=True)
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
elif isinstance(frame, StartInterruptionFrame):
await self._send_interrupt_message()
else:
await self.push_frame(frame, direction)
async def _send_interrupt_message(self) -> None:
transport_frame = TransportMessageUrgentFrame(
message={
@@ -127,7 +187,7 @@ class TavusVideoService(AIService):
)
await self.push_frame(transport_frame)
async def _send_audio_message(self, audio_base64: str, done: bool) -> None:
async def _send_audio_message(self, audio_base64: str, done: bool):
transport_frame = TransportMessageUrgentFrame(
message={
"message_type": "conversation",

603
src/pipecat/services/tts_service.py Normal file
View File

@@ -0,0 +1,603 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from abc import abstractmethod
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple
from loguru import logger
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterimTranscriptionFrame,
LLMFullResponseEndFrame,
StartFrame,
StartInterruptionFrame,
TextFrame,
TranscriptionFrame,
TTSAudioRawFrame,
TTSSpeakFrame,
TTSStartedFrame,
TTSStoppedFrame,
TTSTextFrame,
TTSUpdateSettingsFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
from pipecat.services.websocket_service import WebsocketService
from pipecat.transcriptions.language import Language
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.base_text_filter import BaseTextFilter
from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator
from pipecat.utils.time import seconds_to_nanoseconds
class TTSService(AIService):
def __init__(
self,
*,
aggregate_sentences: bool = True,
# if True, TTSService will push TextFrames and LLMFullResponseEndFrames,
# otherwise subclass must do it
push_text_frames: bool = True,
# if True, TTSService will push TTSStoppedFrames, otherwise subclass must do it
push_stop_frames: bool = False,
# if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
stop_frame_timeout_s: float = 2.0,
# if True, TTSService will push silence audio frames after TTSStoppedFrame
push_silence_after_stop: bool = False,
# if push_silence_after_stop is True, send this amount of audio silence
silence_time_s: float = 2.0,
# if True, we will pause processing frames while we are receiving audio
pause_frame_processing: bool = False,
# TTS output sample rate
sample_rate: Optional[int] = None,
# Text aggregator to aggregate incoming tokens and decide when to push to the TTS.
text_aggregator: Optional[BaseTextAggregator] = None,
# Text filter executed after text has been aggregated.
text_filters: Sequence[BaseTextFilter] = [],
text_filter: Optional[BaseTextFilter] = None,
**kwargs,
):
super().__init__(**kwargs)
self._aggregate_sentences: bool = aggregate_sentences
self._push_text_frames: bool = push_text_frames
self._push_stop_frames: bool = push_stop_frames
self._stop_frame_timeout_s: float = stop_frame_timeout_s
self._push_silence_after_stop: bool = push_silence_after_stop
self._silence_time_s: float = silence_time_s
self._pause_frame_processing: bool = pause_frame_processing
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
self._text_filters: Sequence[BaseTextFilter] = text_filters
if text_filter:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'text_filter' is deprecated, use 'text_filters' instead.",
DeprecationWarning,
)
self._text_filters = [text_filter]
self._stop_frame_task: Optional[asyncio.Task] = None
self._stop_frame_queue: asyncio.Queue = asyncio.Queue()
self._processing_text: bool = False
@property
def sample_rate(self) -> int:
return self._sample_rate
async def set_model(self, model: str):
self.set_model_name(model)
def set_voice(self, voice: str):
self._voice_id = voice
# Converts the text to audio.
@abstractmethod
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
pass
def language_to_service_language(self, language: Language) -> Optional[str]:
return Language(language)
async def update_setting(self, key: str, value: Any):
pass
async def flush_audio(self):
pass
async def start(self, frame: StartFrame):
await super().start(frame)
self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
if self._push_stop_frames and not self._stop_frame_task:
self._stop_frame_task = self.create_task(self._stop_frame_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
if self._stop_frame_task:
await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
if self._stop_frame_task:
await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
async def _update_settings(self, settings: Mapping[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating TTS setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
self._settings[key] = self.language_to_service_language(value)
elif key == "model":
self.set_model_name(value)
elif key == "voice":
self.set_voice(value)
elif key == "text_filter":
for filter in self._text_filters:
filter.update_settings(value)
else:
logger.warning(f"Unknown setting for TTS service: {key}")
async def say(self, text: str):
await self.queue_frame(TTSSpeakFrame(text))
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if (
isinstance(frame, TextFrame)
and not isinstance(frame, InterimTranscriptionFrame)
and not isinstance(frame, TranscriptionFrame)
):
await self._process_text_frame(frame)
elif isinstance(frame, StartInterruptionFrame):
await self._handle_interruption(frame, direction)
await self.push_frame(frame, direction)
elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
# We pause processing incoming frames if the LLM response included
# text (it might be that it's only a function calling response). We
# pause to avoid audio overlapping.
await self._maybe_pause_frame_processing()
sentence = self._text_aggregator.text
self._text_aggregator.reset()
self._processing_text = False
await self._push_tts_frames(sentence)
if isinstance(frame, LLMFullResponseEndFrame):
if self._push_text_frames:
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
elif isinstance(frame, TTSSpeakFrame):
# Store if we were processing text or not so we can set it back.
processing_text = self._processing_text
await self._push_tts_frames(frame.text)
# We pause processing incoming frames because we are sending data to
# the TTS. We pause to avoid audio overlapping.
await self._maybe_pause_frame_processing()
await self.flush_audio()
self._processing_text = processing_text
elif isinstance(frame, TTSUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, BotStoppedSpeakingFrame):
await self._maybe_resume_frame_processing()
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
if self._push_silence_after_stop and isinstance(frame, TTSStoppedFrame):
silence_num_bytes = int(self._silence_time_s * self.sample_rate * 2) # 16-bit
await self.push_frame(
TTSAudioRawFrame(
audio=b"\x00" * silence_num_bytes,
sample_rate=self.sample_rate,
num_channels=1,
)
)
await super().push_frame(frame, direction)
if self._push_stop_frames and (
isinstance(frame, StartInterruptionFrame)
or isinstance(frame, TTSStartedFrame)
or isinstance(frame, TTSAudioRawFrame)
or isinstance(frame, TTSStoppedFrame)
):
await self._stop_frame_queue.put(frame)
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
self._processing_text = False
self._text_aggregator.handle_interruption()
for filter in self._text_filters:
filter.handle_interruption()
async def _maybe_pause_frame_processing(self):
if self._processing_text and self._pause_frame_processing:
await self.pause_processing_frames()
async def _maybe_resume_frame_processing(self):
if self._pause_frame_processing:
await self.resume_processing_frames()
async def _process_text_frame(self, frame: TextFrame):
text: Optional[str] = None
if not self._aggregate_sentences:
text = frame.text
else:
text = self._text_aggregator.aggregate(frame.text)
if text:
await self._push_tts_frames(text)
async def _push_tts_frames(self, text: str):
# Remove leading newlines only
text = text.lstrip("\n")
# Don't send only whitespace. This causes problems for some TTS models. But also don't
# strip all whitespace, as whitespace can influence prosody.
if not text.strip():
return
# This is just a flag that indicates if we sent something to the TTS
# service. It will be cleared if we sent text because of a TTSSpeakFrame
# or when we received an LLMFullResponseEndFrame
self._processing_text = True
await self.start_processing_metrics()
# Process all filter.
for filter in self._text_filters:
filter.reset_interruption()
text = filter.filter(text)
if text:
await self.process_generator(self.run_tts(text))
await self.stop_processing_metrics()
if self._push_text_frames:
# We send the original text after the audio. This way, if we are
# interrupted, the text is not added to the assistant context.
await self.push_frame(TTSTextFrame(text))
async def _stop_frame_handler(self):
has_started = False
while True:
try:
frame = await asyncio.wait_for(
self._stop_frame_queue.get(), self._stop_frame_timeout_s
)
if isinstance(frame, TTSStartedFrame):
has_started = True
elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
has_started = False
except asyncio.TimeoutError:
if has_started:
await self.push_frame(TTSStoppedFrame())
has_started = False
class WordTTSService(TTSService):
"""This is a base class for TTS services that support word timestamps. Word
timestamps are useful to synchronize audio with text of the spoken
words. This way only the spoken words are added to the conversation context.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._initial_word_timestamp = -1
self._words_queue = asyncio.Queue()
self._words_task = None
def start_word_timestamps(self):
if self._initial_word_timestamp == -1:
self._initial_word_timestamp = self.get_clock().get_time()
def reset_word_timestamps(self):
self._initial_word_timestamp = -1
async def add_word_timestamps(self, word_times: List[Tuple[str, float]]):
for word, timestamp in word_times:
await self._words_queue.put((word, seconds_to_nanoseconds(timestamp)))
async def start(self, frame: StartFrame):
await super().start(frame)
self._create_words_task()
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._stop_words_task()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._stop_words_task()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
await self.flush_audio()
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
self.reset_word_timestamps()
def _create_words_task(self):
if not self._words_task:
self._words_task = self.create_task(self._words_task_handler())
async def _stop_words_task(self):
if self._words_task:
await self.cancel_task(self._words_task)
self._words_task = None
async def _words_task_handler(self):
last_pts = 0
while True:
(word, timestamp) = await self._words_queue.get()
if word == "Reset" and timestamp == 0:
self.reset_word_timestamps()
frame = None
elif word == "LLMFullResponseEndFrame" and timestamp == 0:
frame = LLMFullResponseEndFrame()
frame.pts = last_pts
elif word == "TTSStoppedFrame" and timestamp == 0:
frame = TTSStoppedFrame()
frame.pts = last_pts
else:
frame = TTSTextFrame(word)
frame.pts = self._initial_word_timestamp + timestamp
if frame:
last_pts = frame.pts
await self.push_frame(frame)
self._words_queue.task_done()
class WebsocketTTSService(TTSService, WebsocketService):
"""This is a base class for websocket-based TTS services.
If an error occurs with the websocket, an "on_connection_error" event will
be triggered:
@tts.event_handler("on_connection_error")
async def on_connection_error(tts: TTSService, error: str):
...
"""
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
TTSService.__init__(self, **kwargs)
WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
self._register_event_handler("on_connection_error")
async def _report_error(self, error: ErrorFrame):
await self._call_event_handler("on_connection_error", error.error)
await self.push_error(error)
class InterruptibleTTSService(WebsocketTTSService):
"""This is a base class for websocket-based TTS services that don't support
word timestamps and that don't offer a way to correlate the generated audio
to the requested text.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Indicates if the bot is speaking. If the bot is not speaking we don't
# need to reconnect when the user speaks. If the bot is speaking and the
# user interrupts we need to reconnect.
self._bot_speaking = False
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
if self._bot_speaking:
await self._disconnect()
await self._connect()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, BotStartedSpeakingFrame):
self._bot_speaking = True
elif isinstance(frame, BotStoppedSpeakingFrame):
self._bot_speaking = False
class WebsocketWordTTSService(WordTTSService, WebsocketService):
"""This is a base class for websocket-based TTS services that support word
timestamps.
If an error occurs with the websocket a "on_connection_error" event will be
triggered:
@tts.event_handler("on_connection_error")
async def on_connection_error(tts: TTSService, error: str):
...
"""
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
WordTTSService.__init__(self, **kwargs)
WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
self._register_event_handler("on_connection_error")
async def _report_error(self, error: ErrorFrame):
await self._call_event_handler("on_connection_error", error.error)
await self.push_error(error)
class InterruptibleWordTTSService(WebsocketWordTTSService):
"""This is a base class for websocket-based TTS services that support word
timestamps but don't offer a way to correlate the generated audio to the
requested text.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Indicates if the bot is speaking. If the bot is not speaking we don't
# need to reconnect when the user speaks. If the bot is speaking and the
# user interrupts we need to reconnect.
self._bot_speaking = False
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
if self._bot_speaking:
await self._disconnect()
await self._connect()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, BotStartedSpeakingFrame):
self._bot_speaking = True
elif isinstance(frame, BotStoppedSpeakingFrame):
self._bot_speaking = False
class AudioContextWordTTSService(WebsocketWordTTSService):
"""This is a base class for websocket-based TTS services that support word
timestamps and also allow correlating the generated audio with the requested
text.
Each request could be multiple sentences long which are grouped by
context. For this to work, the TTS service needs to support handling
multiple requests at once (i.e. multiple simultaneous contexts).
The audio received from the TTS will be played in context order. That is, if
we requested audio for a context "A" and then audio for context "B", the
audio from context ID "A" will be played first.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._contexts_queue = asyncio.Queue()
self._contexts: Dict[str, asyncio.Queue] = {}
self._audio_context_task = None
async def create_audio_context(self, context_id: str):
"""Create a new audio context."""
await self._contexts_queue.put(context_id)
self._contexts[context_id] = asyncio.Queue()
logger.trace(f"{self} created audio context {context_id}")
async def append_to_audio_context(self, context_id: str, frame: TTSAudioRawFrame):
"""Append audio to an existing context."""
if self.audio_context_available(context_id):
logger.trace(f"{self} appending audio {frame} to audio context {context_id}")
await self._contexts[context_id].put(frame)
else:
logger.warning(f"{self} unable to append audio to context {context_id}")
async def remove_audio_context(self, context_id: str):
"""Remove an existing audio context."""
if self.audio_context_available(context_id):
# We just mark the audio context for deletion by appending
# None. Once we reach None while handling audio we know we can
# safely remove the context.
logger.trace(f"{self} marking audio context {context_id} for deletion")
await self._contexts[context_id].put(None)
else:
logger.warning(f"{self} unable to remove context {context_id}")
def audio_context_available(self, context_id: str) -> bool:
"""Checks whether the given audio context is registered."""
return context_id in self._contexts
async def start(self, frame: StartFrame):
await super().start(frame)
self._create_audio_context_task()
async def stop(self, frame: EndFrame):
await super().stop(frame)
if self._audio_context_task:
# Indicate no more audio contexts are available. this will end the
# task cleanly after all contexts have been processed.
await self._contexts_queue.put(None)
await self.wait_for_task(self._audio_context_task)
self._audio_context_task = None
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._stop_audio_context_task()
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
await self._stop_audio_context_task()
self._create_audio_context_task()
def _create_audio_context_task(self):
if not self._audio_context_task:
self._contexts_queue = asyncio.Queue()
self._contexts: Dict[str, asyncio.Queue] = {}
self._audio_context_task = self.create_task(self._audio_context_task_handler())
async def _stop_audio_context_task(self):
if self._audio_context_task:
await self.cancel_task(self._audio_context_task)
self._audio_context_task = None
async def _audio_context_task_handler(self):
"""In this task we process audio contexts in order."""
running = True
while running:
context_id = await self._contexts_queue.get()
if context_id:
# Process the audio context until the context doesn't have more
# audio available (i.e. we find None).
await self._handle_audio_context(context_id)
# We just finished processing the context, so we can safely remove it.
del self._contexts[context_id]
# Append some silence between sentences.
silence = b"\x00" * self.sample_rate
frame = TTSAudioRawFrame(
audio=silence, sample_rate=self.sample_rate, num_channels=1
)
await self.push_frame(frame)
else:
running = False
self._contexts_queue.task_done()
async def _handle_audio_context(self, context_id: str):
# If we don't receive any audio during this time, we consider the context finished.
AUDIO_CONTEXT_TIMEOUT = 3.0
queue = self._contexts[context_id]
running = True
while running:
try:
frame = await asyncio.wait_for(queue.get(), timeout=AUDIO_CONTEXT_TIMEOUT)
if frame:
await self.push_frame(frame)
running = frame is not None
except asyncio.TimeoutError:
# We didn't get audio, so let's consider this context finished.
logger.trace(f"{self} time out on audio context {context_id}")
break

2
src/pipecat/services/ultravox/stt.py
View File

@@ -29,7 +29,7 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AIService
from pipecat.services.ai_service import AIService
try:
from transformers import AutoTokenizer

34
src/pipecat/services/vision_service.py Normal file
View File

@@ -0,0 +1,34 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from abc import abstractmethod
from typing import AsyncGenerator
from pipecat.frames.frames import Frame, VisionImageRawFrame
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_service import AIService
class VisionService(AIService):
"""VisionService is a base class for vision services."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._describe_text = None
@abstractmethod
async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]:
pass
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, VisionImageRawFrame):
await self.start_processing_metrics()
await self.process_generator(self.run_vision(frame))
await self.stop_processing_metrics()
else:
await self.push_frame(frame, direction)

2
src/pipecat/services/websocket_service.py
View File

@@ -31,7 +31,7 @@ class WebsocketService(ABC):
bool: True if connection is verified working, False otherwise
"""
try:
if not self._websocket:
if not self._websocket or self._websocket.closed:
return False
await self._websocket.ping()
return True

2
src/pipecat/services/whisper/base_stt.py
View File

@@ -11,7 +11,7 @@ from openai import AsyncOpenAI
from openai.types.audio import Transcription
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.ai_services import SegmentedSTTService
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601

2
src/pipecat/services/whisper/stt.py
View File

@@ -15,7 +15,7 @@ from loguru import logger
from typing_extensions import TYPE_CHECKING, override
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.services.ai_services import SegmentedSTTService
from pipecat.services.stt_service import SegmentedSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601

2
src/pipecat/services/xtts/tts.py
View File

@@ -18,7 +18,7 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.services.tts_service import TTSService
from pipecat.transcriptions.language import Language
# The server below can connect to XTTS through a local running docker

25
src/pipecat/transcriptions/language.py
View File

@@ -83,6 +83,10 @@ class Language(StrEnum):
CA = "ca"
CA_ES = "ca-ES"
# Mandarin Chinese
CMN = "cmn"
CMN_CN = "cmn-CN"
# Czech
CS = "cs"
CS_CZ = "cs-CZ"
@@ -182,6 +186,9 @@ class Language(StrEnum):
GA = "ga"
GA_IE = "ga-IE"
# Gaelic
GD = "gd"
# Galician
GL = "gl"
GL_ES = "gl-ES"
@@ -193,6 +200,9 @@ class Language(StrEnum):
# Hausa
HA = "ha"
# Hawaiian
HAW = "haw"
# Hebrew
HE = "he"
HE_IL = "he-IL"
@@ -288,6 +298,9 @@ class Language(StrEnum):
# Malagasy
MG = "mg"
# Maori
MI = "mi"
# Macedonian
MK = "mk"
MK_MK = "mk-MK"
@@ -300,9 +313,6 @@ class Language(StrEnum):
MN = "mn"
MN_MN = "mn-MN"
# Maori
MI = "mi"
# Marathi
MR = "mr"
MR_IN = "mr-IN"
@@ -318,6 +328,7 @@ class Language(StrEnum):
# Burmese
MY = "my"
MY_MM = "my-MM"
MY_MR = "mymr"
# Norwegian
NB = "nb" # Norwegian Bokmål
@@ -414,9 +425,6 @@ class Language(StrEnum):
SW_KE = "sw-KE"
SW_TZ = "sw-TZ"
# Tagalog
TL = "tl"
# Tamil
TA = "ta"
TA_IN = "ta-IN"
@@ -438,6 +446,9 @@ class Language(StrEnum):
# Turkmen
TK = "tk"
# Tagalog
TL = "tl"
# Turkish
TR = "tr"
TR_TR = "tr-TR"
@@ -489,7 +500,7 @@ class Language(StrEnum):
ZH_TW = "zh-TW"
# Xhosa
XH = "xh"
XH = "xh-ZA"
# Zulu
ZU = "zu"

123
src/pipecat/transports/base_input.py
View File

@@ -10,6 +10,10 @@ from typing import Optional
from loguru import logger
from pipecat.audio.turn.base_turn_analyzer import (
BaseTurnAnalyzer,
EndOfTurnState,
)
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
from pipecat.frames.frames import (
BotInterruptionFrame,
@@ -20,6 +24,7 @@ from pipecat.frames.frames import (
FilterUpdateSettingsFrame,
Frame,
InputAudioRawFrame,
MetricsFrame,
StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
@@ -28,6 +33,7 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
VADParamsUpdateFrame,
)
from pipecat.metrics.metrics import MetricsData
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_transport import TransportParams
@@ -49,6 +55,46 @@ class BaseInputTransport(FrameProcessor):
# if passthrough is enabled.
self._audio_task = None
if self._params.vad_enabled:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'vad_enabled' is deprecated, use 'audio_in_enabled' and 'vad_analyzer' instead.",
DeprecationWarning,
)
self._params.audio_in_enabled = True
if self._params.vad_audio_passthrough:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter 'vad_audio_passthrough' is deprecated, audio passthrough is now always enabled. Use 'audio_in_passthrough' to disable.",
DeprecationWarning,
)
self._params.audio_in_passthrough = True
if self._params.camera_in_enabled or self._params.camera_out_enabled:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameters 'camera_*' are deprecated, use 'video_*' instead.",
DeprecationWarning,
)
self._params.video_in_enabled = self._params.camera_in_enabled
self._params.video_out_enabled = self._params.camera_out_enabled
self._params.video_out_is_live = self._params.camera_out_is_live
self._params.video_out_width = self._params.camera_out_width
self._params.video_out_height = self._params.camera_out_height
self._params.video_out_bitrate = self._params.camera_out_bitrate
self._params.video_out_framerate = self._params.camera_out_framerate
self._params.video_out_color_format = self._params.camera_out_color_format
def enable_audio_in_stream_on_start(self, enabled: bool) -> None:
logger.debug(f"Enabling audio on start. {enabled}")
self._params.audio_in_stream_on_start = enabled
@@ -64,23 +110,31 @@ class BaseInputTransport(FrameProcessor):
def vad_analyzer(self) -> Optional[VADAnalyzer]:
return self._params.vad_analyzer
@property
def turn_analyzer(self) -> Optional[BaseTurnAnalyzer]:
return self._params.turn_analyzer
async def start(self, frame: StartFrame):
self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
# Configure VAD analyzer.
if self._params.vad_enabled and self._params.vad_analyzer:
if self._params.vad_analyzer:
self._params.vad_analyzer.set_sample_rate(self._sample_rate)
# Configure End of turn analyzer.
if self._params.turn_analyzer:
self._params.turn_analyzer.set_sample_rate(self._sample_rate)
# Start audio filter.
if self._params.audio_in_filter:
await self._params.audio_in_filter.start(self._sample_rate)
# Create audio input queue and task if needed.
if not self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if not self._audio_task and self._params.audio_in_enabled:
self._audio_in_queue = asyncio.Queue()
self._audio_task = self.create_task(self._audio_task_handler())
async def stop(self, frame: EndFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if self._audio_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_task)
self._audio_task = None
# Stop audio filter.
@@ -89,12 +143,12 @@ class BaseInputTransport(FrameProcessor):
async def cancel(self, frame: CancelFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if self._audio_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_task)
self._audio_task = None
async def push_audio_frame(self, frame: InputAudioRawFrame):
if self._params.audio_in_enabled or self._params.vad_enabled:
if self._params.audio_in_enabled:
await self._audio_in_queue.put(frame)
#
@@ -187,10 +241,18 @@ class BaseInputTransport(FrameProcessor):
and new_vad_state != VADState.STOPPING
):
frame = None
if new_vad_state == VADState.SPEAKING:
frame = UserStartedSpeakingFrame()
elif new_vad_state == VADState.QUIET:
frame = UserStoppedSpeakingFrame()
# If the turn analyser is enabled, this will prevent:
# - Creating the UserStoppedSpeakingFrame
# - Creating the UserStartedSpeakingFrame multiple times
can_create_user_frames = (
self._params.turn_analyzer is None
or not self._params.turn_analyzer.speech_triggered
)
if can_create_user_frames:
if new_vad_state == VADState.SPEAKING:
frame = UserStartedSpeakingFrame()
elif new_vad_state == VADState.QUIET:
frame = UserStoppedSpeakingFrame()
if frame:
await self._handle_user_interruption(frame)
@@ -198,25 +260,56 @@ class BaseInputTransport(FrameProcessor):
vad_state = new_vad_state
return vad_state
async def _handle_end_of_turn(self):
if self.turn_analyzer:
state, prediction = await self.turn_analyzer.analyze_end_of_turn()
await self._handle_prediction_result(prediction)
await self._handle_end_of_turn_complete(state)
async def _handle_end_of_turn_complete(self, state: EndOfTurnState):
if state == EndOfTurnState.COMPLETE:
await self._handle_user_interruption(UserStoppedSpeakingFrame())
async def _run_turn_analyzer(
self, frame: InputAudioRawFrame, vad_state: VADState, previous_vad_state: VADState
):
is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
# If silence exceeds threshold, we are going to receive EndOfTurnState.COMPLETE
end_of_turn_state = self._params.turn_analyzer.append_audio(frame.audio, is_speech)
if end_of_turn_state == EndOfTurnState.COMPLETE:
await self._handle_end_of_turn_complete(end_of_turn_state)
# Otherwise we are going to trigger to check if the turn is completed based on the VAD
elif vad_state == VADState.QUIET and vad_state != previous_vad_state:
await self._handle_end_of_turn()
async def _audio_task_handler(self):
vad_state: VADState = VADState.QUIET
while True:
frame: InputAudioRawFrame = await self._audio_in_queue.get()
audio_passthrough = True
# If an audio filter is available, run it before VAD.
if self._params.audio_in_filter:
frame.audio = await self._params.audio_in_filter.filter(frame.audio)
# Check VAD and push event if necessary. We just care about
# changes from QUIET to SPEAKING and vice versa.
if self._params.vad_enabled:
previous_vad_state = vad_state
if self._params.vad_analyzer:
vad_state = await self._handle_vad(frame, vad_state)
audio_passthrough = self._params.vad_audio_passthrough
# Push audio downstream if passthrough.
if audio_passthrough:
if self._params.turn_analyzer:
await self._run_turn_analyzer(frame, vad_state, previous_vad_state)
# Push audio downstream if passthrough is set.
if self._params.audio_in_passthrough:
await self.push_frame(frame)
self._audio_in_queue.task_done()
async def _handle_prediction_result(self, result: MetricsData):
"""Handle a prediction result event from the turn analyzer.
Args:
result: The prediction result MetricsData.
"""
await self.push_frame(MetricsFrame(data=[result]))

144
src/pipecat/transports/base_output.py
View File

@@ -37,7 +37,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_transport import TransportParams
from pipecat.utils.time import nanoseconds_to_seconds
BOT_VAD_STOP_SECS = 0.3
BOT_VAD_STOP_SECS = 0.35
class BaseOutputTransport(FrameProcessor):
@@ -53,11 +53,10 @@ class BaseOutputTransport(FrameProcessor):
self._sink_clock_task = None
# Task to write/send audio and image frames.
self._camera_out_task = None
self._video_out_task = None
# These are the images that we should send to the camera at our desired
# framerate.
self._camera_images = None
# These are the images that we should send at our desired framerate.
self._video_images = None
# Output sample rate. It will be initialized on StartFrame.
self._sample_rate = 0
@@ -79,15 +78,16 @@ class BaseOutputTransport(FrameProcessor):
async def start(self, frame: StartFrame):
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
# We will write 20ms audio at a time. If we receive long audio frames we
# We will write 10ms*CHUNKS of audio at a time (where CHUNKS is the
# `audio_out_10ms_chunks` parameter). If we receive long audio frames we
# will chunk them. This will help with interruption handling.
audio_bytes_10ms = int(self._sample_rate / 100) * self._params.audio_out_channels * 2
self._audio_chunk_size = audio_bytes_10ms * 2
self._audio_chunk_size = audio_bytes_10ms * self._params.audio_out_10ms_chunks
# Start audio mixer.
if self._params.audio_out_mixer:
await self._params.audio_out_mixer.start(self._sample_rate)
self._create_camera_task()
self._create_video_task()
self._create_sink_tasks()
async def stop(self, frame: EndFrame):
@@ -97,26 +97,26 @@ class BaseOutputTransport(FrameProcessor):
# At this point we have enqueued an EndFrame and we need to wait for
# that EndFrame to be processed by the sink tasks. We also need to wait
# for these tasks before cancelling the camera and audio tasks below
# for these tasks before cancelling the video and audio tasks below
# because they might be still rendering.
if self._sink_task:
await self.wait_for_task(self._sink_task)
if self._sink_clock_task:
await self.wait_for_task(self._sink_clock_task)
# We can now cancel the camera task.
await self._cancel_camera_task()
# We can now cancel the video task.
await self._cancel_video_task()
async def cancel(self, frame: CancelFrame):
# Since we are cancelling everything it doesn't matter if we cancel sink
# tasks first or not.
await self._cancel_sink_tasks()
await self._cancel_camera_task()
await self._cancel_video_task()
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
pass
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
pass
async def write_raw_audio_frames(self, frames: bytes):
@@ -180,11 +180,11 @@ class BaseOutputTransport(FrameProcessor):
return
if isinstance(frame, StartInterruptionFrame):
# Cancel sink and camera tasks.
# Cancel sink and video tasks.
await self._cancel_sink_tasks()
await self._cancel_camera_task()
# Create sink and camera tasks.
self._create_camera_task()
await self._cancel_video_task()
# Create sink and video tasks.
self._create_video_task()
self._create_sink_tasks()
# Let's send a bot stopped speaking if we have to.
await self._bot_stopped_speaking()
@@ -211,11 +211,11 @@ class BaseOutputTransport(FrameProcessor):
self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
async def _handle_image(self, frame: OutputImageRawFrame | SpriteFrame):
if not self._params.camera_out_enabled:
if not self._params.video_out_enabled:
return
if self._params.camera_out_is_live:
await self._camera_out_queue.put(frame)
if self._params.video_out_is_live:
await self._video_out_queue.put(frame)
else:
await self._sink_queue.put(frame)
@@ -260,9 +260,9 @@ class BaseOutputTransport(FrameProcessor):
async def _sink_frame_handler(self, frame: Frame):
if isinstance(frame, OutputImageRawFrame):
await self._set_camera_image(frame)
await self._set_video_image(frame)
elif isinstance(frame, SpriteFrame):
await self._set_camera_images(frame.images)
await self._set_video_images(frame.images)
elif isinstance(frame, TransportMessageFrame):
await self.send_message(frame)
@@ -335,13 +335,22 @@ class BaseOutputTransport(FrameProcessor):
return without_mixer(BOT_VAD_STOP_SECS)
async def _sink_task_handler(self):
# Push a BotSpeakingFrame every 200ms, we don't really need to push it
# at every audio chunk. If the audio chunk is bigger than 200ms, push at
# every audio chunk.
TOTAL_CHUNK_MS = self._params.audio_out_10ms_chunks * 10
BOT_SPEAKING_CHUNK_PERIOD = max(int(200 / TOTAL_CHUNK_MS), 1)
bot_speaking_counter = 0
async for frame in self._next_frame():
# Notify the bot started speaking upstream if necessary and that
# it's actually speaking.
if isinstance(frame, TTSAudioRawFrame):
await self._bot_started_speaking()
await self.push_frame(BotSpeakingFrame())
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
if bot_speaking_counter % BOT_SPEAKING_CHUNK_PERIOD == 0:
await self.push_frame(BotSpeakingFrame())
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
bot_speaking_counter = 0
bot_speaking_counter += 1
# No need to push EndFrame, it's pushed from process_frame().
if isinstance(frame, EndFrame):
@@ -358,76 +367,79 @@ class BaseOutputTransport(FrameProcessor):
await self.write_raw_audio_frames(frame.audio)
#
# Camera task
# Video task
#
def _create_camera_task(self):
# Create camera output queue and task if needed.
if not self._camera_out_task and self._params.camera_out_enabled:
self._camera_out_queue = asyncio.Queue()
self._camera_out_task = self.create_task(self._camera_out_task_handler())
def _create_video_task(self):
# Create video output queue and task if needed.
if not self._video_out_task and self._params.video_out_enabled:
self._video_out_queue = asyncio.Queue()
self._video_out_task = self.create_task(self._video_out_task_handler())
async def _cancel_camera_task(self):
# Stop camera output task.
if self._camera_out_task and self._params.camera_out_enabled:
await self.cancel_task(self._camera_out_task)
self._camera_out_task = None
async def _cancel_video_task(self):
# Stop video output task.
if self._video_out_task and self._params.video_out_enabled:
await self.cancel_task(self._video_out_task)
self._video_out_task = None
async def _draw_image(self, frame: OutputImageRawFrame):
desired_size = (self._params.camera_out_width, self._params.camera_out_height)
desired_size = (self._params.video_out_width, self._params.video_out_height)
# TODO: we should refactor in the future to support dynamic resolutions
# which is kind of what happens in P2P connections.
# We need to add support for that inside the DailyTransport
if frame.size != desired_size:
image = Image.frombytes(frame.format, frame.size, frame.image)
resized_image = image.resize(desired_size)
logger.warning(f"{frame} does not have the expected size {desired_size}, resizing")
# logger.warning(f"{frame} does not have the expected size {desired_size}, resizing")
frame = OutputImageRawFrame(
resized_image.tobytes(), resized_image.size, resized_image.format
)
await self.write_frame_to_camera(frame)
await self.write_raw_video_frame(frame)
async def _set_camera_image(self, image: OutputImageRawFrame):
self._camera_images = itertools.cycle([image])
async def _set_video_image(self, image: OutputImageRawFrame):
self._video_images = itertools.cycle([image])
async def _set_camera_images(self, images: List[OutputImageRawFrame]):
self._camera_images = itertools.cycle(images)
async def _set_video_images(self, images: List[OutputImageRawFrame]):
self._video_images = itertools.cycle(images)
async def _camera_out_task_handler(self):
self._camera_out_start_time = None
self._camera_out_frame_index = 0
self._camera_out_frame_duration = 1 / self._params.camera_out_framerate
self._camera_out_frame_reset = self._camera_out_frame_duration * 5
async def _video_out_task_handler(self):
self._video_out_start_time = None
self._video_out_frame_index = 0
self._video_out_frame_duration = 1 / self._params.video_out_framerate
self._video_out_frame_reset = self._video_out_frame_duration * 5
while True:
if self._params.camera_out_is_live:
await self._camera_out_is_live_handler()
elif self._camera_images:
image = next(self._camera_images)
if self._params.video_out_is_live:
await self._video_out_is_live_handler()
elif self._video_images:
image = next(self._video_images)
await self._draw_image(image)
await asyncio.sleep(self._camera_out_frame_duration)
await asyncio.sleep(self._video_out_frame_duration)
else:
await asyncio.sleep(self._camera_out_frame_duration)
await asyncio.sleep(self._video_out_frame_duration)
async def _camera_out_is_live_handler(self):
image = await self._camera_out_queue.get()
async def _video_out_is_live_handler(self):
image = await self._video_out_queue.get()
# We get the start time as soon as we get the first image.
if not self._camera_out_start_time:
self._camera_out_start_time = time.time()
self._camera_out_frame_index = 0
if not self._video_out_start_time:
self._video_out_start_time = time.time()
self._video_out_frame_index = 0
# Calculate how much time we need to wait before rendering next image.
real_elapsed_time = time.time() - self._camera_out_start_time
real_render_time = self._camera_out_frame_index * self._camera_out_frame_duration
delay_time = self._camera_out_frame_duration + real_render_time - real_elapsed_time
real_elapsed_time = time.time() - self._video_out_start_time
real_render_time = self._video_out_frame_index * self._video_out_frame_duration
delay_time = self._video_out_frame_duration + real_render_time - real_elapsed_time
if abs(delay_time) > self._camera_out_frame_reset:
self._camera_out_start_time = time.time()
self._camera_out_frame_index = 0
if abs(delay_time) > self._video_out_frame_reset:
self._video_out_start_time = time.time()
self._video_out_frame_index = 0
elif delay_time > 0:
await asyncio.sleep(delay_time)
self._camera_out_frame_index += 1
self._video_out_frame_index += 1
# Render image
await self._draw_image(image)
self._camera_out_queue.task_done()
self._video_out_queue.task_done()

12
src/pipecat/transports/base_transport.py
View File

@@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict
from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer
from pipecat.audio.vad.vad_analyzer import VADAnalyzer
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.utils.base_object import BaseObject
@@ -31,15 +32,26 @@ class TransportParams(BaseModel):
audio_out_sample_rate: Optional[int] = None
audio_out_channels: int = 1
audio_out_bitrate: int = 96000
audio_out_10ms_chunks: int = 4
audio_out_mixer: Optional[BaseAudioMixer] = None
audio_in_enabled: bool = False
audio_in_sample_rate: Optional[int] = None
audio_in_channels: int = 1
audio_in_filter: Optional[BaseAudioFilter] = None
audio_in_stream_on_start: bool = True
audio_in_passthrough: bool = True
video_in_enabled: bool = False
video_out_enabled: bool = False
video_out_is_live: bool = False
video_out_width: int = 1024
video_out_height: int = 768
video_out_bitrate: int = 800000
video_out_framerate: int = 30
video_out_color_format: str = "RGB"
vad_enabled: bool = False
vad_audio_passthrough: bool = False
vad_analyzer: Optional[VADAnalyzer] = None
turn_analyzer: Optional[BaseTurnAnalyzer] = None
class BaseTransport(BaseObject):

2
src/pipecat/transports/local/tk.py
View File

@@ -137,7 +137,7 @@ class TkOutputTransport(BaseOutputTransport):
self._executor, self._out_stream.write, frames
)
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
def _write_frame_to_tk(self, frame: OutputImageRawFrame):

12
src/pipecat/transports/network/fastapi_websocket.py
View File

@@ -61,6 +61,10 @@ class FastAPIWebsocketClient:
self._closing = False
self._is_binary = is_binary
self._callbacks = callbacks
self._leave_counter = 0
async def setup(self, _: StartFrame):
self._leave_counter += 1
def receive(self) -> typing.AsyncIterator[bytes | str]:
return self._websocket.iter_bytes() if self._is_binary else self._websocket.iter_text()
@@ -73,6 +77,10 @@ class FastAPIWebsocketClient:
await self._websocket.send_text(data)
async def disconnect(self):
self._leave_counter -= 1
if self._leave_counter > 0:
return
if self.is_connected and not self.is_closing:
self._closing = True
await self._websocket.close()
@@ -116,6 +124,7 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.setup(frame)
await self._params.serializer.setup(frame)
if not self._monitor_websocket_task and self._params.session_timeout:
self._monitor_websocket_task = self.create_task(self._monitor_websocket())
@@ -192,15 +201,18 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.setup(frame)
await self._params.serializer.setup(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._write_frame(frame)
await self._client.disconnect()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._write_frame(frame)
await self._client.disconnect()
async def cleanup(self):

185
src/pipecat/transports/network/small_webrtc.py
View File

@@ -17,13 +17,19 @@ from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
InputImageRawFrame,
OutputAudioRawFrame,
OutputImageRawFrame,
SpriteFrame,
StartFrame,
TransportMessageFrame,
TransportMessageUrgentFrame,
UserImageRawFrame,
UserImageRequestFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -51,61 +57,52 @@ class RawAudioTrack(AudioStreamTrack):
def __init__(self, sample_rate):
super().__init__()
self._sample_rate = sample_rate
self._samples_per_frame = self._sample_rate // 50 # 20ms per frame
self._samples_per_10ms = sample_rate * 10 // 1000
self._bytes_per_10ms = self._samples_per_10ms * 2 # 16-bit (2 bytes per sample)
self._timestamp = 0
self._audio_buffer = deque()
self._start = time.time()
# Queue of (bytes, future), broken into 10ms sub chunks as needed
self._chunk_queue = deque()
def add_audio_bytes(self, audio_bytes: bytes):
"""
Adds bytes to the audio buffer and returns a Future that completes when the data is processed.
"""
if len(audio_bytes) % 2 != 0:
raise ValueError("Audio bytes length must be even (16-bit samples).")
"""Adds bytes to the audio buffer and returns a Future that completes when the data is processed."""
if len(audio_bytes) % self._bytes_per_10ms != 0:
raise ValueError("Audio bytes must be a multiple of 10ms size.")
future = asyncio.get_running_loop().create_future()
self._audio_buffer.append((audio_bytes, future))
# Break input into 10ms chunks
for i in range(0, len(audio_bytes), self._bytes_per_10ms):
chunk = audio_bytes[i : i + self._bytes_per_10ms]
# Only the last chunk carries the future to be resolved once fully consumed
fut = future if i + self._bytes_per_10ms >= len(audio_bytes) else None
self._chunk_queue.append((chunk, fut))
return future
async def recv(self):
"""
Returns the next audio frame, generating silence if needed.
"""
"""Returns the next audio frame, generating silence if needed."""
# Compute required wait time for synchronization
if self._timestamp > 0:
wait = self._start + (self._timestamp / self._sample_rate) - time.time()
if wait > 0:
await asyncio.sleep(wait)
# Check if we have enough data
needed_bytes = self._samples_per_frame * 2 # 16-bit (2 bytes per sample)
available_bytes = sum(len(audio_bytes) for audio_bytes, _ in self._audio_buffer)
consumed_futures = [] # Track futures for processed data
if available_bytes >= needed_bytes:
# Extract data from deque
chunk = bytearray()
while len(chunk) < needed_bytes:
audio_bytes, future = self._audio_buffer.popleft()
chunk.extend(audio_bytes)
consumed_futures.append(future) # Track the future
chunk = bytes(chunk[:needed_bytes]) # Trim excess bytes
if self._chunk_queue:
chunk, future = self._chunk_queue.popleft()
if future and not future.done():
future.set_result(True)
else:
chunk = bytes(needed_bytes) # Generate silent frame
chunk = bytes(self._bytes_per_10ms) # silence
# Convert the byte data to an ndarray of int16 samples
samples = np.frombuffer(chunk, dtype=np.int16)
# Create AudioFrame
frame = AudioFrame.from_ndarray(samples[None, :], layout="mono")
self._timestamp += self._samples_per_frame
frame.pts = self._timestamp
frame.sample_rate = self._sample_rate
frame.pts = self._timestamp
frame.time_base = fractions.Fraction(1, self._sample_rate)
# Resolve all futures corresponding to consumed data
for future in consumed_futures:
if not future.done():
future.set_result(True)
self._timestamp += self._samples_per_10ms
return frame
@@ -138,6 +135,13 @@ class RawVideoTrack(VideoStreamTrack):
class SmallWebRTCClient:
FORMAT_CONVERSIONS = {
"yuv420p": cv2.COLOR_YUV2RGB_I420,
"yuvj420p": cv2.COLOR_YUV2RGB_I420, # OpenCV treats both the same
"nv12": cv2.COLOR_YUV2RGB_NV12,
"gray": cv2.COLOR_GRAY2RGB,
}
def __init__(self, webrtc_connection: SmallWebRTCConnection, callbacks: SmallWebRTCCallbacks):
self._webrtc_connection = webrtc_connection
self._closing = False
@@ -176,9 +180,31 @@ class SmallWebRTCClient:
async def on_app_message(connection: SmallWebRTCConnection, message: Any):
await self._handle_app_message(message)
async def read_video_frame(self):
def _convert_frame(self, frame_array: np.ndarray, format_name: str) -> np.ndarray:
"""Convert a given frame to RGB format based on the input format.
Args:
frame_array (np.ndarray): The input frame.
format_name (str): The format of the input frame.
Returns:
np.ndarray: The converted RGB frame.
Raises:
ValueError: If the format is unsupported.
"""
Reads a video frame from the given MediaStreamTrack, converts it to RGB,
if format_name.startswith("rgb"): # Already in RGB, no conversion needed
return frame_array
conversion_code = SmallWebRTCClient.FORMAT_CONVERSIONS.get(format_name)
if conversion_code is None:
raise ValueError(f"Unsupported format: {format_name}")
return cv2.cvtColor(frame_array, conversion_code)
async def read_video_frame(self):
"""Reads a video frame from the given MediaStreamTrack, converts it to RGB,
and creates an InputImageRawFrame.
"""
while True:
@@ -203,21 +229,9 @@ class SmallWebRTCClient:
continue
format_name = frame.format.name
# Convert frame to NumPy array in its native format
frame_array = frame.to_ndarray(format=format_name)
# Handle different formats dynamically
if format_name == "yuv420p":
frame_rgb = cv2.cvtColor(frame_array, cv2.COLOR_YUV2RGB_I420)
elif format_name == "nv12":
frame_rgb = cv2.cvtColor(frame_array, cv2.COLOR_YUV2RGB_NV12)
elif format_name == "gray":
frame_rgb = cv2.cvtColor(frame_array, cv2.COLOR_GRAY2RGB)
elif format_name.startswith("rgb"): # Already RGB, no conversion needed
frame_rgb = frame_array
else:
raise ValueError(f"Unsupported format: {format_name}")
frame_rgb = self._convert_frame(frame_array, format_name)
image_frame = InputImageRawFrame(
image=frame_rgb.tobytes(),
@@ -228,9 +242,7 @@ class SmallWebRTCClient:
yield image_frame
async def read_audio_frame(self):
"""
Reads 20ms of audio from the given MediaStreamTrack and creates an InputAudioRawFrame.
"""
"""Reads 20ms of audio from the given MediaStreamTrack and creates an InputAudioRawFrame."""
while True:
if self._audio_input_track is None:
await asyncio.sleep(0.01)
@@ -276,7 +288,7 @@ class SmallWebRTCClient:
if self._can_send() and self._audio_output_track:
await self._audio_output_track.add_audio_bytes(data)
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
if self._can_send() and self._video_output_track:
self._video_output_track.add_video_frame(frame)
@@ -298,7 +310,7 @@ class SmallWebRTCClient:
if self.is_connected and not self.is_closing:
logger.info(f"Disconnecting to Small WebRTC")
self._closing = True
await self._webrtc_connection.close()
await self._webrtc_connection.disconnect()
await self._handle_client_disconnected()
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
@@ -316,9 +328,9 @@ class SmallWebRTCClient:
self._audio_output_track = RawAudioTrack(sample_rate=self._out_sample_rate)
self._webrtc_connection.replace_audio_track(self._audio_output_track)
if self._params.camera_out_enabled:
if self._params.video_out_enabled:
self._video_output_track = RawVideoTrack(
width=self._params.camera_out_width, height=self._params.camera_out_height
width=self._params.video_out_width, height=self._params.video_out_height
)
self._webrtc_connection.replace_video_track(self._video_output_track)
@@ -365,16 +377,21 @@ class SmallWebRTCInputTransport(BaseInputTransport):
self._params = params
self._receive_audio_task = None
self._receive_video_task = None
self._image_requests = {}
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, UserImageRequestFrame):
await self.request_participant_image(frame)
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.setup(self._params, frame)
await self._client.connect()
if not self._receive_audio_task and (
self._params.audio_in_enabled or self._params.vad_enabled
):
if not self._receive_audio_task and self._params.audio_in_enabled:
self._receive_audio_task = self.create_task(self._receive_audio())
if not self._receive_video_task and self._params.camera_in_enabled:
if not self._receive_video_task and self._params.video_in_enabled:
self._receive_video_task = self.create_task(self._receive_video())
async def _stop_tasks(self):
@@ -410,6 +427,22 @@ class SmallWebRTCInputTransport(BaseInputTransport):
if video_frame:
await self.push_frame(video_frame)
# Check if there are any pending image requests and create UserImageRawFrame
if self._image_requests:
for req_id, request_frame in list(self._image_requests.items()):
# Create UserImageRawFrame using the current video frame
image_frame = UserImageRawFrame(
user_id=request_frame.user_id,
request=request_frame,
image=video_frame.image,
size=video_frame.size,
format=video_frame.format,
)
# Push the frame to the pipeline
await self.push_frame(image_frame)
# Remove from pending requests
del self._image_requests[req_id]
except Exception as e:
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
@@ -418,6 +451,24 @@ class SmallWebRTCInputTransport(BaseInputTransport):
frame = TransportMessageUrgentFrame(message=message)
await self.push_frame(frame)
# Add this method similar to DailyInputTransport.request_participant_image
async def request_participant_image(self, frame: UserImageRequestFrame):
"""Requests an image frame from the participant's video stream.
When a UserImageRequestFrame is received, this method will store the request
and the next video frame received will be converted to a UserImageRawFrame.
"""
logger.debug(f"Requesting image from participant: {frame.user_id}")
# Store the request
request_id = f"{frame.function_name}:{frame.tool_call_id}"
self._image_requests[request_id] = frame
# If we're not already receiving video, try to get a frame now
if not self._receive_video_task and self._params.video_in_enabled:
# Start video reception if it's not already running
self._receive_video_task = self.create_task(self._receive_video())
class SmallWebRTCOutputTransport(BaseOutputTransport):
def __init__(
@@ -449,8 +500,8 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
async def write_raw_audio_frames(self, frames: bytes):
await self._client.write_raw_audio_frames(frames)
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
await self._client.write_frame_to_camera(frame)
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
await self._client.write_raw_video_frame(frame)
class SmallWebRTCTransport(BaseTransport):
@@ -473,10 +524,8 @@ class SmallWebRTCTransport(BaseTransport):
self._client = SmallWebRTCClient(webrtc_connection, self._callbacks)
self._input = SmallWebRTCInputTransport(self._client, self._params, name=self._input_name)
self._output = SmallWebRTCOutputTransport(
self._client, self._params, name=self._output_name
)
self._input: Optional[SmallWebRTCInputTransport] = None
self._output: Optional[SmallWebRTCOutputTransport] = None
# Register supported handlers. The user will only be able to register
# these handlers.
@@ -499,6 +548,14 @@ class SmallWebRTCTransport(BaseTransport):
)
return self._output
async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
if self._output:
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
async def send_audio(self, frame: OutputAudioRawFrame):
if self._output:
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
async def _on_app_message(self, message: Any):
if self._input:
await self._input.push_app_message(message)

154
src/pipecat/transports/network/webrtc_connection.py
View File

@@ -7,25 +7,84 @@
import asyncio
import json
import time
from enum import Enum
from typing import Any, Optional
from typing import Any, Literal, Optional, Union
from av.frame import Frame
from loguru import logger
from pydantic import BaseModel, TypeAdapter
from pipecat.utils.base_object import BaseObject
try:
from aiortc import RTCConfiguration, RTCIceServer, RTCPeerConnection, RTCSessionDescription
from aiortc import (
MediaStreamTrack,
RTCConfiguration,
RTCIceServer,
RTCPeerConnection,
RTCSessionDescription,
)
from aiortc.rtcrtpreceiver import RemoteStreamTrack
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the SmallWebRTC, you need to `pip install pipecat-ai[webrtc]`.")
raise Exception(f"Missing module: {e}")
SIGNALLING_TYPE = "signalling"
AUDIO_TRANSCEIVER_INDEX = 0
VIDEO_TRANSCEIVER_INDEX = 1
class SignallingMessage(Enum):
RENEGOTIATE = "renegotiate"
class TrackStatusMessage(BaseModel):
type: Literal["trackStatus"]
receiver_index: int
enabled: bool
class RenegotiateMessage(BaseModel):
type: Literal["renegotiate"] = "renegotiate"
class PeerLeftMessage(BaseModel):
type: Literal["peerLeft"] = "peerLeft"
class SignallingMessage:
Inbound = Union[TrackStatusMessage] # in case we need to add new messages in the future
outbound = Union[RenegotiateMessage]
class SmallWebRTCTrack:
def __init__(self, track: MediaStreamTrack):
self._track = track
self._enabled = True
def set_enabled(self, enabled: bool) -> None:
self._enabled = enabled
def is_enabled(self) -> bool:
return self._enabled
async def discard_old_frames(self):
remote_track = self._track
if isinstance(remote_track, RemoteStreamTrack):
if not hasattr(remote_track, "_queue") or not isinstance(
remote_track._queue, asyncio.Queue
):
print("Warning: _queue does not exist or has changed in aiortc.")
return
logger.debug("Discarding old frames")
while not remote_track._queue.empty():
remote_track._queue.get_nowait() # Remove the oldest frame
remote_track._queue.task_done()
async def recv(self) -> Optional[Frame]:
if not self._enabled:
return None
return await self._track.recv()
def __getattr__(self, name):
# Forward other attribute/method calls to the underlying track
return getattr(self._track, name)
class SmallWebRTCConnection(BaseObject):
@@ -36,6 +95,12 @@ class SmallWebRTCConnection(BaseObject):
else:
self.ice_servers = []
self._connect_invoked = False
self._track_map = {}
self._track_getters = {
AUDIO_TRANSCEIVER_INDEX: self.audio_input_track,
VIDEO_TRANSCEIVER_INDEX: self.video_input_track,
}
self._initialize()
# Register supported handlers. The user will only be able to register
@@ -67,16 +132,24 @@ class SmallWebRTCConnection(BaseObject):
self._pc = RTCPeerConnection(rtc_config)
self._pc_id = self.name
self._setup_listeners()
self._tracks = set()
self._data_channel = None
self._renegotiation_in_progress = False
self._last_received_time = None
self._message_queue = []
def _setup_listeners(self):
@self._pc.on("datachannel")
def on_datachannel(channel):
self._data_channel = channel
# Flush queued messages once the data channel is open
@channel.on("open")
async def on_open():
logger.debug("Data channel is open, flushing queued messages")
while self._message_queue:
message = self._message_queue.pop(0)
self._data_channel.send(message)
@channel.on("message")
async def on_message(message):
try:
@@ -86,7 +159,10 @@ class SmallWebRTCConnection(BaseObject):
self._last_received_time = time.time()
else:
json_message = json.loads(message)
await self._call_event_handler("app-message", json_message)
if json_message["type"] == SIGNALLING_TYPE and json_message.get("message"):
self._handle_signalling_message(json_message["message"])
else:
await self._call_event_handler("app-message", json_message)
except Exception as e:
logger.exception(f"Error parsing JSON message {message}, {e}")
@@ -111,13 +187,11 @@ class SmallWebRTCConnection(BaseObject):
@self._pc.on("track")
async def on_track(track):
logger.debug(f"Track {track.kind} received")
self._tracks.add(track)
await self._call_event_handler("track-started", track)
@track.on("ended")
async def on_ended():
logger.debug(f"Track {track.kind} ended")
self._tracks.discard(track)
await self._call_event_handler("track-ended", track)
async def _create_answer(self, sdp: str, type: str):
@@ -145,6 +219,9 @@ class SmallWebRTCConnection(BaseObject):
await self._call_event_handler("connected")
# We are renegotiating here, because likely we have loose the first video frames
# and aiortc does not handle that pretty well.
video_input_track = self.video_input_track()
if video_input_track:
await self.video_input_track().discard_old_frames()
self.ask_to_renegotiate()
async def renegotiate(self, sdp: str, type: str, restart_pc: bool = False):
@@ -155,7 +232,7 @@ class SmallWebRTCConnection(BaseObject):
logger.debug("Closing old peer connection")
# removing the listeners to prevent the bot from closing
self._pc.remove_all_listeners()
await self.close()
await self._close()
# we are initializing a new peer connection in this case.
self._initialize()
@@ -200,9 +277,15 @@ class SmallWebRTCConnection(BaseObject):
else:
logger.warning("Video transceiver not found. Cannot replace video track.")
async def close(self):
async def disconnect(self):
self.send_app_message({"type": SIGNALLING_TYPE, "message": PeerLeftMessage().model_dump()})
await self._close()
async def _close(self):
if self._pc:
await self._pc.close()
self._message_queue.clear()
self._track_map = {}
def get_answer(self):
if not self._answer:
@@ -216,11 +299,14 @@ class SmallWebRTCConnection(BaseObject):
async def _handle_new_connection_state(self):
state = self._pc.connectionState
if state == "connected" and not self._connect_invoked:
# We are going to wait until the pipeline is ready before triggering the event
return
logger.debug(f"Connection state changed to: {state}")
await self._call_event_handler(state)
if state == "failed":
logger.warning("Connection failed, closing peer connection.")
await self.close()
await self._close()
# Despite the fact that aiortc provides this listener, they don't have a status for "disconnected"
# So, there is no advantage in looking at self._pc.connectionState
@@ -239,34 +325,46 @@ class SmallWebRTCConnection(BaseObject):
return (time.time() - self._last_received_time) < 3
def audio_input_track(self):
if self._track_map.get(AUDIO_TRANSCEIVER_INDEX):
return self._track_map[AUDIO_TRANSCEIVER_INDEX]
# Transceivers always appear in creation-order for both peers
# For now we are only considering that we are going to have 02 transceivers,
# one for audio and one for video
transceivers = self._pc.getTransceivers()
if len(transceivers) == 0 or not transceivers[0].receiver:
if len(transceivers) == 0 or not transceivers[AUDIO_TRANSCEIVER_INDEX].receiver:
logger.warning("No audio transceiver is available")
return None
return transceivers[0].receiver.track
track = transceivers[AUDIO_TRANSCEIVER_INDEX].receiver.track
audio_track = SmallWebRTCTrack(track) if track else None
self._track_map[AUDIO_TRANSCEIVER_INDEX] = audio_track
return audio_track
def video_input_track(self):
if self._track_map.get(VIDEO_TRANSCEIVER_INDEX):
return self._track_map[VIDEO_TRANSCEIVER_INDEX]
# Transceivers always appear in creation-order for both peers
# For now we are only considering that we are going to have 02 transceivers,
# one for audio and one for video
transceivers = self._pc.getTransceivers()
if len(transceivers) <= 1 or not transceivers[1].receiver:
if len(transceivers) <= 1 or not transceivers[VIDEO_TRANSCEIVER_INDEX].receiver:
logger.warning("No video transceiver is available")
return None
return transceivers[1].receiver.track
def tracks(self):
return self._tracks
track = transceivers[VIDEO_TRANSCEIVER_INDEX].receiver.track
video_track = SmallWebRTCTrack(track) if track else None
self._track_map[VIDEO_TRANSCEIVER_INDEX] = video_track
return video_track
def send_app_message(self, message: Any):
if self._data_channel:
json_message = json.dumps(message)
json_message = json.dumps(message)
if self._data_channel and self._data_channel.readyState == "open":
self._data_channel.send(json_message)
else:
logger.debug("Data channel not ready, queuing message")
self._message_queue.append(json_message)
def ask_to_renegotiate(self):
if self._renegotiation_in_progress:
@@ -274,5 +372,17 @@ class SmallWebRTCConnection(BaseObject):
self._renegotiation_in_progress = True
self.send_app_message(
{"type": SIGNALLING_TYPE, "message": SignallingMessage.RENEGOTIATE.value}
{"type": SIGNALLING_TYPE, "message": RenegotiateMessage().model_dump()}
)
def _handle_signalling_message(self, message):
logger.debug(f"Signalling message received: {message}")
inbound_adapter = TypeAdapter(SignallingMessage.Inbound)
signalling_message = inbound_adapter.validate_python(message)
match signalling_message:
case TrackStatusMessage():
track = (
self._track_getters.get(signalling_message.receiver_index) or (lambda: None)
)()
if track:
track.set_enabled(signalling_message.enabled)

6
src/pipecat/transports/network/websocket_client.py
View File

@@ -56,8 +56,8 @@ class WebsocketClientSession:
self._callbacks = callbacks
self._transport_name = transport_name
self._leave_counter = 0
self._task_manager: Optional[BaseTaskManager] = None
self._websocket: Optional[websockets.WebSocketClientProtocol] = None
@property
@@ -69,6 +69,7 @@ class WebsocketClientSession:
return self._task_manager
async def setup(self, frame: StartFrame):
self._leave_counter += 1
if not self._task_manager:
self._task_manager = frame.task_manager
@@ -87,7 +88,8 @@ class WebsocketClientSession:
logger.error(f"Timeout connecting to {self._uri}")
async def disconnect(self):
if not self._websocket:
self._leave_counter -= 1
if not self._websocket or self._leave_counter > 0:
return
await self.task_manager.cancel_task(self._client_task)

11
src/pipecat/transports/network/websocket_server.py
View File

@@ -157,7 +157,8 @@ class WebsocketServerInputTransport(BaseInputTransport):
self, websocket: websockets.WebSocketServerProtocol, session_timeout: int
):
"""Wait for session_timeout seconds, if the websocket is still open,
trigger timeout event."""
trigger timeout event.
"""
try:
await asyncio.sleep(session_timeout)
if not websocket.closed:
@@ -195,6 +196,14 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
await self._params.serializer.setup(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._write_frame(frame)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._write_frame(frame)
async def cleanup(self):
await super().cleanup()
await self._transport.cleanup()

49
src/pipecat/transports/services/daily.py
View File

@@ -169,6 +169,8 @@ class DailyCallbacks(BaseModel):
on_error: Called when an error occurs.
on_app_message: Called when receiving an app message.
on_call_state_updated: Called when call state changes.
on_client_connected: Called when a client (participant) connects.
on_client_disconnected: Called when a client (participant) disconnects.
on_dialin_connected: Called when dial-in is connected.
on_dialin_ready: Called when dial-in is ready.
on_dialin_stopped: Called when dial-in is stopped.
@@ -193,6 +195,8 @@ class DailyCallbacks(BaseModel):
on_error: Callable[[str], Awaitable[None]]
on_app_message: Callable[[Any, str], Awaitable[None]]
on_call_state_updated: Callable[[str], Awaitable[None]]
on_client_connected: Callable[[Mapping[str, Any]], Awaitable[None]]
on_client_disconnected: Callable[[Mapping[str, Any]], Awaitable[None]]
on_dialin_connected: Callable[[Any], Awaitable[None]]
on_dialin_ready: Callable[[str], Awaitable[None]]
on_dialin_stopped: Callable[[Any], Awaitable[None]]
@@ -369,7 +373,7 @@ class DailyTransportClient(EventHandler):
self._mic.write_frames(frames, completion=completion_callback(future))
await future
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
if not self._camera:
return None
@@ -379,12 +383,12 @@ class DailyTransportClient(EventHandler):
self._in_sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
if self._params.camera_out_enabled and not self._camera:
if self._params.video_out_enabled and not self._camera:
self._camera = Daily.create_camera_device(
self._camera_name(),
width=self._params.camera_out_width,
height=self._params.camera_out_height,
color_format=self._params.camera_out_color_format,
width=self._params.video_out_width,
height=self._params.video_out_height,
color_format=self._params.video_out_color_format,
)
if self._params.audio_out_enabled and not self._mic:
@@ -395,7 +399,7 @@ class DailyTransportClient(EventHandler):
non_blocking=True,
)
if (self._params.audio_in_enabled or self._params.vad_enabled) and not self._speaker:
if self._params.audio_in_enabled and not self._speaker:
self._speaker = Daily.create_speaker_device(
self._speaker_name(),
sample_rate=self._in_sample_rate,
@@ -483,7 +487,7 @@ class DailyTransportClient(EventHandler):
client_settings={
"inputs": {
"camera": {
"isEnabled": self._params.camera_out_enabled,
"isEnabled": self._params.video_out_enabled,
"settings": {
"deviceId": self._camera_name(),
},
@@ -506,8 +510,8 @@ class DailyTransportClient(EventHandler):
"maxQuality": "low",
"encodings": {
"low": {
"maxBitrate": self._params.camera_out_bitrate,
"maxFramerate": self._params.camera_out_framerate,
"maxBitrate": self._params.video_out_bitrate,
"maxFramerate": self._params.video_out_framerate,
}
},
}
@@ -842,7 +846,7 @@ class DailyInputTransport(BaseInputTransport):
def start_audio_in_streaming(self):
# Create audio task. It reads audio frames from Daily and push them
# internally for VAD processing.
if not self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if not self._audio_in_task and self._params.audio_in_enabled:
logger.debug(f"Start receiving audio")
self._audio_in_task = self.create_task(self._audio_in_task_handler())
@@ -859,9 +863,6 @@ class DailyInputTransport(BaseInputTransport):
await self._client.setup(frame)
# Join the room.
await self._client.join()
# Inialize WebRTC VAD if needed.
if self._params.vad_enabled and not self._params.vad_analyzer:
self._vad_analyzer = WebRTCVADAnalyzer(sample_rate=self.sample_rate)
if self._params.audio_in_stream_on_start:
self.start_audio_in_streaming()
@@ -871,7 +872,7 @@ class DailyInputTransport(BaseInputTransport):
# Leave the room.
await self._client.leave()
# Stop audio thread.
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if self._audio_in_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_in_task)
self._audio_in_task = None
@@ -881,7 +882,7 @@ class DailyInputTransport(BaseInputTransport):
# Leave the room.
await self._client.leave()
# Stop audio thread.
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if self._audio_in_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_in_task)
self._audio_in_task = None
@@ -1034,8 +1035,8 @@ class DailyOutputTransport(BaseOutputTransport):
async def write_raw_audio_frames(self, frames: bytes):
await self._client.write_raw_audio_frames(frames)
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
await self._client.write_frame_to_camera(frame)
async def write_raw_video_frame(self, frame: OutputImageRawFrame):
await self._client.write_raw_video_frame(frame)
class DailyTransport(BaseTransport):
@@ -1070,6 +1071,8 @@ class DailyTransport(BaseTransport):
on_error=self._on_error,
on_app_message=self._on_app_message,
on_call_state_updated=self._on_call_state_updated,
on_client_connected=self._on_client_connected,
on_client_disconnected=self._on_client_disconnected,
on_dialin_connected=self._on_dialin_connected,
on_dialin_ready=self._on_dialin_ready,
on_dialin_stopped=self._on_dialin_stopped,
@@ -1103,6 +1106,8 @@ class DailyTransport(BaseTransport):
self._register_event_handler("on_error")
self._register_event_handler("on_app_message")
self._register_event_handler("on_call_state_updated")
self._register_event_handler("on_client_connected")
self._register_event_handler("on_client_disconnected")
self._register_event_handler("on_dialin_connected")
self._register_event_handler("on_dialin_ready")
self._register_event_handler("on_dialin_stopped")
@@ -1246,6 +1251,12 @@ class DailyTransport(BaseTransport):
async def _on_call_state_updated(self, state: str):
await self._call_event_handler("on_call_state_updated", state)
async def _on_client_connected(self, participant: Any):
await self._call_event_handler("on_client_connected", participant)
async def _on_client_disconnected(self, participant: Any):
await self._call_event_handler("on_client_disconnected", participant)
async def _handle_dialin_ready(self, sip_endpoint: str):
if not self._params.dialin_settings:
return
@@ -1321,11 +1332,15 @@ class DailyTransport(BaseTransport):
await self._call_event_handler("on_first_participant_joined", participant)
await self._call_event_handler("on_participant_joined", participant)
# Also call on_client_connected for compatibility with other transports
await self._call_event_handler("on_client_connected", participant)
async def _on_participant_left(self, participant, reason):
id = participant["id"]
logger.info(f"Participant left {id}")
await self._call_event_handler("on_participant_left", participant, reason)
# Also call on_client_disconnected for compatibility with other transports
await self._call_event_handler("on_client_disconnected", participant)
async def _on_participant_updated(self, participant):
await self._call_event_handler("on_participant_updated", participant)

11
src/pipecat/transports/services/helpers/daily_rest.py
View File

@@ -68,9 +68,9 @@ class DailyRoomProperties(BaseModel, extra="allow"):
exp: Optional[float] = None
enable_chat: bool = False
enable_prejoin_ui: bool = True
enable_prejoin_ui: bool = False
enable_emoji_reactions: bool = False
eject_at_room_exp: bool = True
eject_at_room_exp: bool = False
enable_dialout: Optional[bool] = None
enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = None
geo: Optional[str] = None
@@ -291,6 +291,7 @@ class DailyRESTHelper:
self,
room_url: str,
expiry_time: float = 60 * 60,
eject_at_token_exp: bool = False,
owner: bool = True,
params: Optional[DailyMeetingTokenParams] = None,
) -> str:
@@ -324,12 +325,16 @@ class DailyRESTHelper:
if params is None:
params = DailyMeetingTokenParams(
properties=DailyMeetingTokenProperties(
room_name=room_name, is_owner=owner, exp=expiration
room_name=room_name,
is_owner=owner,
exp=expiration,
eject_at_token_exp=eject_at_token_exp,
)
)
else:
params.properties.room_name = room_name
params.properties.exp = expiration
params.properties.eject_at_token_exp = eject_at_token_exp
params.properties.is_owner = owner
json = params.model_dump(exclude_none=True)

4
src/pipecat/transports/services/livekit.py
View File

@@ -368,7 +368,7 @@ class LiveKitInputTransport(BaseInputTransport):
await super().start(frame)
await self._client.setup(frame)
await self._client.connect()
if not self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if not self._audio_in_task and self._params.audio_in_enabled:
self._audio_in_task = self.create_task(self._audio_in_task_handler())
logger.info("LiveKitInputTransport started")
@@ -382,7 +382,7 @@ class LiveKitInputTransport(BaseInputTransport):
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._client.disconnect()
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
if self._audio_in_task and self._params.audio_in_enabled:
await self.cancel_task(self._audio_in_task)
async def cleanup(self):

2
src/pipecat/utils/base_object.py
View File

@@ -38,7 +38,7 @@ class BaseObject(ABC):
async def cleanup(self):
if self._event_tasks:
event_names, tasks = zip(*self._event_tasks)
logger.debug(f"{self} wating on event handlers to finish {list(event_names)}...")
logger.debug(f"{self} waiting on event handlers to finish {list(event_names)}...")
await asyncio.wait(tasks)
def event_handler(self, event_name: str):