wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:pipecat-ai/pipecat

Browse Source
This commit is contained in:
Matej Marinko
2025-07-08 08:20:10 +02:00
parent c54084b7a4 a6de16f92f
commit 0f727248d2
482 changed files with 32975 additions and 7233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/pipecat/adapters/base_llm_adapter.py
View File

@@ -1,3 +1,15 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base adapter for LLM provider integration.
This module provides the abstract base class for implementing LLM provider-specific
adapters that handle tool format conversion and standardization.
"""
from abc import ABC, abstractmethod
from typing import Any, List, Union, cast
@@ -7,12 +19,35 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
class BaseLLMAdapter(ABC):
"""Abstract base class for LLM provider adapters.
Provides a standard interface for converting between Pipecat's standardized
tool schemas and provider-specific tool formats. Subclasses must implement
provider-specific conversion logic.
"""
@abstractmethod
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Any]:
"""Converts tools to the provider's format."""
"""Convert tools schema to the provider's specific format.
Args:
tools_schema: The standardized tools schema to convert.
Returns:
List of tools in the provider's expected format.
"""
pass
def from_standard_tools(self, tools: Any) -> List[Any]:
"""Convert tools from standard format to provider format.
Args:
tools: Tools in standard format or provider-specific format.
Returns:
List of tools converted to provider format, or original tools
if not in standard format.
"""
if isinstance(tools, ToolsSchema):
logger.debug(f"Retrieving the tools using the adapter: {type(self)}")
return self.to_provider_tools_format(tools)

296
src/pipecat/adapters/schemas/direct_function.py Normal file
View File

@@ -0,0 +1,296 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Direct function wrapper utilities for LLM function calling.
This module provides utilities for wrapping "direct" functions that handle LLM
function calls. Direct functions have their metadata automatically extracted
from function signatures and docstrings, allowing them to be used without
accompanying configurations (as FunctionSchemas or in provider-specific
formats).
"""
import inspect
import types
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
List,
Mapping,
Protocol,
Set,
Tuple,
Union,
get_args,
get_origin,
get_type_hints,
)
import docstring_parser
from pipecat.adapters.schemas.function_schema import FunctionSchema
if TYPE_CHECKING:
from pipecat.services.llm_service import FunctionCallParams
class DirectFunction(Protocol):
"""Protocol for a "direct" function that handles LLM function calls.
"Direct" functions' metadata is automatically extracted from their function signature and
docstrings, allowing them to be used without accompanying function configurations (as
`FunctionSchema`s or in provider-specific formats).
"""
async def __call__(self, params: "FunctionCallParams", **kwargs: Any) -> None:
"""Execute the direct function.
Args:
params: Function call parameters from the LLM service.
**kwargs: Additional keyword arguments passed to the function.
"""
...
class BaseDirectFunctionWrapper:
"""Base class for a wrapper around a DirectFunction.
Provides functionality to:
- extract metadata from the function signature and docstring
- use that metadata to generate a corresponding FunctionSchema
"""
def __init__(self, function: Callable):
"""Initialize the direct function wrapper.
Args:
function: The function to wrap and extract metadata from.
"""
self.__class__.validate_function(function)
self.function = function
self._initialize_metadata()
@classmethod
def special_first_param_name(cls) -> str:
"""Get the name of the special first function parameter.
The special first parameter is ignored by metadata extraction as it's
not relevant to the LLM (e.g., 'params' for FunctionCallParams).
Returns:
The name of the special first parameter.
"""
raise NotImplementedError("Subclasses must define the special first parameter name.")
@classmethod
def validate_function(cls, function: Callable) -> None:
"""Validate that the function meets direct function requirements.
Args:
function: The function to validate.
Raises:
Exception: If function doesn't meet requirements (not async, missing
parameters, incorrect first parameter name).
"""
if not inspect.iscoroutinefunction(function):
raise Exception(f"Direct function {function.__name__} must be async")
params = list(inspect.signature(function).parameters.items())
special_first_param_name = cls.special_first_param_name()
if len(params) == 0:
raise Exception(
f"Direct function {function.__name__} must have at least one parameter ({special_first_param_name})"
)
first_param_name = params[0][0]
if first_param_name != special_first_param_name:
raise Exception(
f"Direct function {function.__name__} first parameter must be named '{special_first_param_name}'"
)
def to_function_schema(self) -> FunctionSchema:
"""Convert the wrapped function to a FunctionSchema.
Returns:
A FunctionSchema instance with extracted metadata.
"""
return FunctionSchema(
name=self.name,
description=self.description,
properties=self.properties,
required=self.required,
)
def _initialize_metadata(self):
"""Initialize metadata from function signature and docstring."""
# Get function name
self.name = self.function.__name__
# Parse docstring for description and parameters
docstring = docstring_parser.parse(inspect.getdoc(self.function))
# Get function description
self.description = (docstring.description or "").strip()
# Get function parameters as JSON schemas, and the list of required parameters
self.properties, self.required = self._get_parameters_as_jsonschema(
self.function, docstring.params
)
# TODO: maybe to better support things like enums, check if each type is a pydantic type and use its convert-to-jsonschema function
def _get_parameters_as_jsonschema(
self, func: Callable, docstring_params: List[docstring_parser.DocstringParam]
) -> Tuple[Dict[str, Any], List[str]]:
"""Get function parameters as a dictionary of JSON schemas and a list of required parameters.
Ignore the first parameter, as it's expected to be the "special" one.
Args:
func: Function to get parameters from.
docstring_params: List of parameters extracted from the function's docstring.
Returns:
A tuple containing:
- A dictionary mapping each function parameter to its JSON schema
- A list of required parameter names
"""
sig = inspect.signature(func)
hints = get_type_hints(func)
properties = {}
required = []
for name, param in sig.parameters.items():
# Ignore 'self' parameter
if name == "self":
continue
# Ignore the first parameter, which is expected to be the "special" one
# (We have already validated that this is the case in validate_function())
is_first_param = name == next(iter(sig.parameters))
if is_first_param:
continue
type_hint = hints.get(name)
# Convert type hint to JSON schema
properties[name] = self._typehint_to_jsonschema(type_hint)
# Add whether the parameter is required
# If the parameter has no default value, it's required
if param.default is inspect.Parameter.empty:
required.append(name)
# Add parameter description from docstring
for doc_param in docstring_params:
if doc_param.arg_name == name:
properties[name]["description"] = doc_param.description or ""
return properties, required
def _typehint_to_jsonschema(self, type_hint: Any) -> Dict[str, Any]:
"""Convert a Python type hint to a JSON Schema.
Args:
type_hint: A Python type hint
Returns:
A dictionary representing the JSON Schema
"""
if type_hint is None:
return {}
# Handle basic types
if type_hint is type(None):
return {"type": "null"}
if type_hint is str:
return {"type": "string"}
elif type_hint is int:
return {"type": "integer"}
elif type_hint is float:
return {"type": "number"}
elif type_hint is bool:
return {"type": "boolean"}
elif type_hint is dict or type_hint is Dict:
return {"type": "object"}
elif type_hint is list or type_hint is List:
return {"type": "array"}
# Get origin and arguments for complex types
origin = get_origin(type_hint)
args = get_args(type_hint)
# Handle Optional/Union types
if origin is Union or origin is types.UnionType:
return {"anyOf": [self._typehint_to_jsonschema(arg) for arg in args]}
# Handle List, Tuple, Set with specific item types
if origin in (list, List, tuple, Tuple, set, Set) and args:
return {"type": "array", "items": self._typehint_to_jsonschema(args[0])}
# Handle Dict with specific key/value types
if origin in (dict, Dict) and len(args) == 2:
# For JSON Schema, keys must be strings
return {"type": "object", "additionalProperties": self._typehint_to_jsonschema(args[1])}
# Handle TypedDict
if hasattr(type_hint, "__annotations__"):
properties = {}
required = []
# NOTE: this does not yet support some fields being required and others not, which could happen when:
# - the base class is a TypedDict with required fields (total=True or not specified) and the derived class has optional fields (total=False)
# - Python 3.11+ NotRequired is used
all_fields_required = getattr(type_hint, "__total__", True)
for field_name, field_type in get_type_hints(type_hint).items():
properties[field_name] = self._typehint_to_jsonschema(field_type)
if all_fields_required:
required.append(field_name)
schema = {"type": "object", "properties": properties}
if required:
schema["required"] = required
return schema
# Default to any type if we can't determine the specific schema
return {}
class DirectFunctionWrapper(BaseDirectFunctionWrapper):
"""Wrapper around a DirectFunction for LLM function calling.
This class:
- Extracts metadata from the function signature and docstring
- Generates a corresponding FunctionSchema
- Helps with function invocation
"""
@classmethod
def special_first_param_name(cls) -> str:
"""Get the special first parameter name for direct functions.
Returns:
The string "params" which is expected as the first parameter.
"""
return "params"
async def invoke(self, args: Mapping[str, Any], params: "FunctionCallParams"):
"""Invoke the wrapped function with the provided arguments.
Args:
args: Arguments to pass to the function.
params: Function call parameters from the LLM service.
Returns:
The result of the function call.
"""
return await self.function(params=params, **args)

21
src/pipecat/adapters/schemas/function_schema.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Function schema utilities for AI tool definitions.
This module provides standardized function schema representation for defining
tools and functions used with AI models, ensuring consistent formatting
across different AI service providers.
"""
from typing import Any, Dict, List
@@ -13,17 +20,19 @@ class FunctionSchema:
Provides a structured way to define function tools used with AI models like OpenAI.
This schema defines the function's name, description, parameter properties, and
required parameters, following specifications required by AI service providers.
Args:
name: Name of the function to be called.
description: Description of what the function does.
properties: Dictionary defining parameter types, descriptions, and constraints.
required: List of property names that are required parameters.
"""
def __init__(
self, name: str, description: str, properties: Dict[str, Any], required: List[str]
) -> None:
"""Initialize the function schema.
Args:
name: Name of the function to be called.
description: Description of what the function does.
properties: Dictionary defining parameter types, descriptions, and constraints.
required: List of property names that are required parameters.
"""
self._name = name
self._description = description
self._properties = properties

62
src/pipecat/adapters/schemas/tools_schema.py
View File

@@ -4,40 +4,88 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Tools schema definitions for function calling adapters.
This module provides schemas for managing both standardized function tools
and custom adapter-specific tools in the Pipecat framework.
"""
from enum import Enum
from typing import Any, Dict, List, Optional
from pipecat.adapters.schemas.direct_function import DirectFunction, DirectFunctionWrapper
from pipecat.adapters.schemas.function_schema import FunctionSchema
class AdapterType(Enum):
"""Supported adapter types for custom tools.
Parameters:
GEMINI: Google Gemini adapter - currently the only service supporting custom tools.
"""
GEMINI = "gemini" # that is the only service where we are able to add custom tools for now
class ToolsSchema:
"""Schema for managing both standard and custom function calling tools.
This class provides a unified interface for handling standardized function
schemas alongside custom tools that may not follow the standard format,
such as adapter-specific search tools.
"""
def __init__(
self,
standard_tools: List[FunctionSchema],
standard_tools: List[FunctionSchema | DirectFunction],
custom_tools: Optional[Dict[AdapterType, List[Dict[str, Any]]]] = None,
) -> None:
"""
A schema for tools that includes both standardized function schemas
and custom tools that do not follow the FunctionSchema format.
"""Initialize the tools schema.
:param standard_tools: List of tools following FunctionSchema.
:param custom_tools: List of tools in a custom format (e.g., search_tool).
Args:
standard_tools: List of tools following the standardized FunctionSchema format.
custom_tools: Dictionary mapping adapter types to their custom tool definitions.
These tools may not follow the FunctionSchema format (e.g., search_tool).
"""
self._standard_tools = standard_tools
def _map_standard_tools(tools):
schemas = []
for tool in tools:
if isinstance(tool, FunctionSchema):
schemas.append(tool)
elif callable(tool):
wrapper = DirectFunctionWrapper(tool)
schemas.append(wrapper.to_function_schema())
else:
raise TypeError(f"Unsupported tool type: {type(tool)}")
return schemas
self._standard_tools = _map_standard_tools(standard_tools)
self._custom_tools = custom_tools
@property
def standard_tools(self) -> List[FunctionSchema]:
"""Get the list of standard function schema tools.
Returns:
List of tools following the FunctionSchema format.
"""
return self._standard_tools
@property
def custom_tools(self) -> Dict[AdapterType, List[Dict[str, Any]]]:
"""Get the custom tools dictionary.
Returns:
Dictionary mapping adapter types to their custom tool definitions.
"""
return self._custom_tools
@custom_tools.setter
def custom_tools(self, value: Dict[AdapterType, List[Dict[str, Any]]]) -> None:
"""Set the custom tools dictionary.
Args:
value: Dictionary mapping adapter types to their custom tool definitions.
"""
self._custom_tools = value

25
src/pipecat/adapters/services/anthropic_adapter.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Anthropic LLM adapter for Pipecat."""
from typing import Any, Dict, List
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
@@ -12,8 +14,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
class AnthropicLLMAdapter(BaseLLMAdapter):
"""Adapter for converting tool schemas to Anthropic's function-calling format.
This adapter handles the conversion of Pipecat's standard function schemas
to the specific format required by Anthropic's Claude models for function calling.
"""
@staticmethod
def _to_anthropic_function_format(function: FunctionSchema) -> Dict[str, Any]:
"""Convert a single function schema to Anthropic's format.
Args:
function: The function schema to convert.
Returns:
Dictionary containing the function definition in Anthropic's format.
"""
return {
"name": function.name,
"description": function.description,
@@ -25,10 +41,13 @@ class AnthropicLLMAdapter(BaseLLMAdapter):
}
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Converts function schemas to Anthropic's function-calling format.
"""Convert function schemas to Anthropic's function-calling format.
:return: Anthropic formatted function call definition.
Args:
tools_schema: The tools schema containing functions to convert.
Returns:
List of function definitions formatted for Anthropic's API.
"""
functions_schema = tools_schema.standard_tools
return [self._to_anthropic_function_format(func) for func in functions_schema]

26
src/pipecat/adapters/services/aws_nova_sonic_adapter.py
View File

@@ -3,6 +3,9 @@
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AWS Nova Sonic LLM adapter for Pipecat."""
import json
from typing import Any, Dict, List
@@ -12,8 +15,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
class AWSNovaSonicLLMAdapter(BaseLLMAdapter):
"""Adapter for AWS Nova Sonic language models.
Converts Pipecat's standard function schemas into AWS Nova Sonic's
specific function-calling format, enabling tool use with Nova Sonic models.
"""
@staticmethod
def _to_aws_nova_sonic_function_format(function: FunctionSchema) -> Dict[str, Any]:
"""Convert a function schema to AWS Nova Sonic format.
Args:
function: The function schema to convert.
Returns:
Dictionary in AWS Nova Sonic function format with toolSpec structure.
"""
return {
"toolSpec": {
"name": function.name,
@@ -31,10 +48,13 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter):
}
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Converts function schemas to AWS Nova Sonic function-calling format.
"""Convert tools schema to AWS Nova Sonic function-calling format.
:return: AWS Nova Sonic formatted function call definition.
Args:
tools_schema: The tools schema containing function definitions to convert.
Returns:
List of dictionaries in AWS Nova Sonic function format.
"""
functions_schema = tools_schema.standard_tools
return [self._to_aws_nova_sonic_function_format(func) for func in functions_schema]

25
src/pipecat/adapters/services/bedrock_adapter.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AWS Bedrock LLM adapter for Pipecat."""
from typing import Any, Dict, List
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
@@ -12,8 +14,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
class AWSBedrockLLMAdapter(BaseLLMAdapter):
"""Adapter for AWS Bedrock LLM integration with Pipecat.
Provides conversion utilities for transforming Pipecat function schemas
into AWS Bedrock's expected tool format for function calling capabilities.
"""
@staticmethod
def _to_bedrock_function_format(function: FunctionSchema) -> Dict[str, Any]:
"""Convert a function schema to Bedrock's tool format.
Args:
function: The function schema to convert.
Returns:
Dictionary formatted for Bedrock's tool specification.
"""
return {
"toolSpec": {
"name": function.name,
@@ -29,10 +45,13 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter):
}
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Converts function schemas to Bedrock's function-calling format.
"""Convert function schemas to Bedrock's function-calling format.
:return: Bedrock formatted function call definition.
Args:
tools_schema: The tools schema containing functions to convert.
Returns:
List of Bedrock formatted function call definitions.
"""
functions_schema = tools_schema.standard_tools
return [self._to_bedrock_function_format(func) for func in functions_schema]

19
src/pipecat/adapters/services/gemini_adapter.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Gemini LLM adapter for Pipecat."""
from typing import Any, Dict, List, Union
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
@@ -11,12 +13,23 @@ from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
class GeminiLLMAdapter(BaseLLMAdapter):
"""LLM adapter for Google's Gemini service.
Provides tool schema conversion functionality to transform standard tool
definitions into Gemini's specific function-calling format for use with
Gemini LLM models.
"""
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Converts function schemas to Gemini's function-calling format.
"""Convert tool schemas to Gemini's function-calling format.
:return: Gemini formatted function call definition.
Args:
tools_schema: The tools schema containing standard and custom tool definitions.
Returns:
List of tool definitions formatted for Gemini's function-calling API.
Includes both converted standard tools and any custom Gemini-specific tools.
"""
functions_schema = tools_schema.standard_tools
formatted_standard_tools = [
{"function_declarations": [func.to_default_dict() for func in functions_schema]}

21
src/pipecat/adapters/services/open_ai_adapter.py
View File

@@ -3,6 +3,9 @@
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""OpenAI LLM adapter for Pipecat."""
from typing import List
from openai.types.chat import ChatCompletionToolParam
@@ -12,10 +15,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
class OpenAILLMAdapter(BaseLLMAdapter):
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ChatCompletionToolParam]:
"""Converts function schemas to OpenAI's function-calling format.
"""Adapter for converting tool schemas to OpenAI's format.
:return: OpenAI formatted function call definition.
Provides conversion utilities for transforming Pipecat's standard tool
schemas into the format expected by OpenAI's ChatCompletion API for
function calling capabilities.
"""
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ChatCompletionToolParam]:
"""Convert function schemas to OpenAI's function-calling format.
Args:
tools_schema: The Pipecat tools schema to convert.
Returns:
List of OpenAI formatted function call definitions ready for use
with ChatCompletion API.
"""
functions_schema = tools_schema.standard_tools
return [

26
src/pipecat/adapters/services/open_ai_realtime_adapter.py
View File

@@ -3,6 +3,9 @@
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""OpenAI Realtime LLM adapter for Pipecat."""
from typing import Any, Dict, List, Union
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
@@ -11,8 +14,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
"""LLM adapter for OpenAI Realtime API function calling.
Converts Pipecat's tool schemas into the specific format required by
OpenAI's Realtime API for function calling capabilities.
"""
@staticmethod
def _to_openai_realtime_function_format(function: FunctionSchema) -> Dict[str, Any]:
"""Convert a function schema to OpenAI Realtime format.
Args:
function: The function schema to convert.
Returns:
Dictionary in OpenAI Realtime function format.
"""
return {
"type": "function",
"name": function.name,
@@ -25,10 +42,13 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
}
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Converts function schemas to Openai Realtime function-calling format.
"""Convert tool schemas to OpenAI Realtime function-calling format.
:return: Openai Realtime formatted function call definition.
Args:
tools_schema: The tools schema containing functions to convert.
Returns:
List of function definitions in OpenAI Realtime format.
"""
functions_schema = tools_schema.standard_tools
return [self._to_openai_realtime_function_format(func) for func in functions_schema]

36
src/pipecat/audio/filters/base_audio_filter.py
View File

@@ -4,44 +4,68 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base audio filter interface for input transport audio processing.
This module provides the abstract base class for implementing audio filters
that process audio data before VAD and downstream processing in input transports.
"""
from abc import ABC, abstractmethod
from pipecat.frames.frames import FilterControlFrame
class BaseAudioFilter(ABC):
"""This is a base class for input transport audio filters. If an audio
"""Base class for input transport audio filters.
This is a base class for input transport audio filters. If an audio
filter is provided to the input transport it will be used to process audio
before VAD and before pushing it downstream. There are control frames to
update filter settings or to enable or disable the filter at runtime.
"""
@abstractmethod
async def start(self, sample_rate: int):
"""This will be called from the input transport when the transport is
"""Initialize the filter when the input transport starts.
This will be called from the input transport when the transport is
started. It can be used to initialize the filter. The input transport
sample rate is provided so the filter can adjust to that sample rate.
Args:
sample_rate: The sample rate of the input transport in Hz.
"""
pass
@abstractmethod
async def stop(self):
"""This will be called from the input transport when the transport is
stopping.
"""Clean up the filter when the input transport stops.
This will be called from the input transport when the transport is
stopping.
"""
pass
@abstractmethod
async def process_frame(self, frame: FilterControlFrame):
"""This will be called when the input transport receives a
"""Process control frames for runtime filter configuration.
This will be called when the input transport receives a
FilterControlFrame.
Args:
frame: The control frame containing filter commands or settings.
"""
pass
@abstractmethod
async def filter(self, audio: bytes) -> bytes:
"""Apply the audio filter to the provided audio data.
Args:
audio: Raw audio data as bytes to be filtered.
Returns:
Filtered audio data as bytes.
"""
pass

39
src/pipecat/audio/filters/koala_filter.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Koala noise suppression audio filter for Pipecat.
This module provides an audio filter implementation using PicoVoice's Koala
Noise Suppression engine to reduce background noise in audio streams.
"""
from typing import Sequence
import numpy as np
@@ -21,12 +27,19 @@ except ModuleNotFoundError as e:
class KoalaFilter(BaseAudioFilter):
"""This is an audio filter that uses Koala Noise Suppression (from
PicoVoice).
"""Audio filter using Koala Noise Suppression from PicoVoice.
Provides real-time noise suppression for audio streams using PicoVoice's
Koala engine. The filter buffers audio data to match Koala's required
frame length and processes it in chunks.
"""
def __init__(self, *, access_key: str) -> None:
"""Initialize the Koala noise suppression filter.
Args:
access_key: PicoVoice access key for Koala engine authentication.
"""
self._access_key = access_key
self._filtering = True
@@ -36,6 +49,11 @@ class KoalaFilter(BaseAudioFilter):
self._audio_buffer = bytearray()
async def start(self, sample_rate: int):
"""Initialize the filter with the transport's sample rate.
Args:
sample_rate: The sample rate of the input transport in Hz.
"""
self._sample_rate = sample_rate
if self._sample_rate != self._koala.sample_rate:
logger.warning(
@@ -44,13 +62,30 @@ class KoalaFilter(BaseAudioFilter):
self._koala_ready = False
async def stop(self):
"""Clean up the Koala engine when stopping."""
self._koala.reset()
async def process_frame(self, frame: FilterControlFrame):
"""Process control frames to enable/disable filtering.
Args:
frame: The control frame containing filter commands.
"""
if isinstance(frame, FilterEnableFrame):
self._filtering = frame.enable
async def filter(self, audio: bytes) -> bytes:
"""Apply Koala noise suppression to audio data.
Buffers incoming audio and processes it in chunks that match Koala's
required frame length. Returns filtered audio data.
Args:
audio: Raw audio data as bytes to be filtered.
Returns:
Noise-suppressed audio data as bytes.
"""
if not self._koala_ready or not self._filtering:
return audio

65
src/pipecat/audio/filters/krisp_filter.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Krisp noise reduction audio filter for Pipecat.
This module provides an audio filter implementation using Krisp's noise
reduction technology to suppress background noise in audio streams.
"""
import os
import numpy as np
@@ -21,14 +27,27 @@ except ModuleNotFoundError as e:
class KrispProcessorManager:
"""
Ensures that only one KrispAudioProcessor instance exists for the entire program.
"""Singleton manager for KrispAudioProcessor instances.
Ensures that only one KrispAudioProcessor instance exists for the entire
program.
"""
_krisp_instance = None
@classmethod
def get_processor(cls, sample_rate: int, sample_type: str, channels: int, model_path: str):
"""Get or create a KrispAudioProcessor instance.
Args:
sample_rate: Audio sample rate in Hz.
sample_type: Audio sample type (e.g., "PCM_16").
channels: Number of audio channels.
model_path: Path to the Krisp model file.
Returns:
Shared KrispAudioProcessor instance.
"""
if cls._krisp_instance is None:
cls._krisp_instance = KrispAudioProcessor(
sample_rate, sample_type, channels, model_path
@@ -37,14 +56,26 @@ class KrispProcessorManager:
class KrispFilter(BaseAudioFilter):
"""Audio filter using Krisp noise reduction technology.
Provides real-time noise reduction for audio streams using Krisp's
proprietary noise suppression algorithms. Requires a Krisp model file
for operation.
"""
def __init__(
self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None
) -> None:
"""Initializes the KrispAudioProcessor with customizable audio processing settings.
"""Initialize the Krisp noise reduction filter.
:param sample_type: The type of audio sample, default is 'PCM_16'.
:param channels: Number of audio channels, default is 1.
:param model_path: Path to the Krisp model; defaults to environment variable KRISP_MODEL_PATH if not provided.
Args:
sample_type: The audio sample format. Defaults to "PCM_16".
channels: Number of audio channels. Defaults to 1.
model_path: Path to the Krisp model file. If None, uses KRISP_MODEL_PATH
environment variable.
Raises:
ValueError: If model_path is not provided and KRISP_MODEL_PATH is not set.
"""
super().__init__()
@@ -63,19 +94,41 @@ class KrispFilter(BaseAudioFilter):
self._krisp_processor = None
async def start(self, sample_rate: int):
"""Initialize the Krisp processor with the transport's sample rate.
Args:
sample_rate: The sample rate of the input transport in Hz.
"""
self._sample_rate = sample_rate
self._krisp_processor = KrispProcessorManager.get_processor(
self._sample_rate, self._sample_type, self._channels, self._model_path
)
async def stop(self):
"""Clean up the Krisp processor when stopping."""
self._krisp_processor = None
async def process_frame(self, frame: FilterControlFrame):
"""Process control frames to enable/disable filtering.
Args:
frame: The control frame containing filter commands.
"""
if isinstance(frame, FilterEnableFrame):
self._filtering = frame.enable
async def filter(self, audio: bytes) -> bytes:
"""Apply Krisp noise reduction to audio data.
Converts audio to float32, applies Krisp noise reduction processing,
and returns the filtered audio clipped to int16 range.
Args:
audio: Raw audio data as bytes to be filtered.
Returns:
Noise-reduced audio data as bytes.
"""
if not self._filtering:
return audio

37
src/pipecat/audio/filters/noisereduce_filter.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Noisereduce audio filter for Pipecat.
This module provides an audio filter implementation using the noisereduce
library to reduce background noise in audio streams through spectral
gating algorithms.
"""
import numpy as np
from loguru import logger
@@ -21,21 +28,51 @@ except ModuleNotFoundError as e:
class NoisereduceFilter(BaseAudioFilter):
"""Audio filter using the noisereduce library for noise suppression.
Applies spectral gating noise reduction algorithms to suppress background
noise in audio streams. Uses the noisereduce library's default noise
reduction parameters.
"""
def __init__(self) -> None:
"""Initialize the noisereduce filter."""
self._filtering = True
self._sample_rate = 0
async def start(self, sample_rate: int):
"""Initialize the filter with the transport's sample rate.
Args:
sample_rate: The sample rate of the input transport in Hz.
"""
self._sample_rate = sample_rate
async def stop(self):
"""Clean up the filter when stopping."""
pass
async def process_frame(self, frame: FilterControlFrame):
"""Process control frames to enable/disable filtering.
Args:
frame: The control frame containing filter commands.
"""
if isinstance(frame, FilterEnableFrame):
self._filtering = frame.enable
async def filter(self, audio: bytes) -> bytes:
"""Apply noise reduction to audio data using spectral gating.
Converts audio to float32, applies noisereduce processing, and returns
the filtered audio clipped to int16 range.
Args:
audio: Raw audio data as bytes to be filtered.
Returns:
Noise-reduced audio data as bytes.
"""
if not self._filtering:
return audio

30
src/pipecat/audio/interruptions/base_interruption_strategy.py
View File

@@ -4,31 +4,51 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base interruption strategy for determining when users can interrupt bot speech."""
from abc import ABC, abstractmethod
class BaseInterruptionStrategy(ABC):
"""This is a base class for interruption strategies. Interruption strategies
"""Base class for interruption strategies.
This is a base class for interruption strategies. Interruption strategies
decide when the user can interrupt the bot while the bot is speaking. For
example, there could be strategies based on audio volume or strategies based
on the number of words the user spoke.
"""
async def append_audio(self, audio: bytes, sample_rate: int):
"""Appends audio to the strategy. Not all strategies handle audio."""
"""Append audio data to the strategy for analysis.
Not all strategies handle audio. Default implementation does nothing.
Args:
audio: Raw audio bytes to append.
sample_rate: Sample rate of the audio data in Hz.
"""
pass
async def append_text(self, text: str):
"""Appends text to the strategy. Not all strategies handle text."""
"""Append text data to the strategy for analysis.
Not all strategies handle text. Default implementation does nothing.
Args:
text: Text string to append for analysis.
"""
pass
@abstractmethod
async def should_interrupt(self) -> bool:
"""This is called when the user stops speaking and it's time to decide
"""Determine if the user should interrupt the bot.
This is called when the user stops speaking and it's time to decide
whether the user should interrupt the bot. The decision will be based on
the aggregated audio and/or text.
Returns:
True if the user should interrupt the bot, False otherwise.
"""
pass

25
src/pipecat/audio/interruptions/min_words_interruption_strategy.py
View File

@@ -4,31 +4,47 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Minimum words interruption strategy for word count-based interruptions."""
from loguru import logger
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
"""This is an interruption strategy based on a minimum number of words said
"""Interruption strategy based on minimum number of words spoken.
This is an interruption strategy based on a minimum number of words said
by the user. That is, the strategy will be true if the user has said at
least that amount of words.
"""
def __init__(self, *, min_words: int):
"""Initialize the minimum words interruption strategy.
Args:
min_words: Minimum number of words required to trigger an interruption.
"""
super().__init__()
self._min_words = min_words
self._text = ""
async def append_text(self, text: str):
"""Appends text for later analysis. Not all strategies need to handle
text.
"""Append text for word count analysis.
Args:
text: Text string to append to the accumulated text.
Note: Not all strategies need to handle text.
"""
self._text += text
async def should_interrupt(self) -> bool:
"""Check if the minimum word count has been reached.
Returns:
True if the user has spoken at least the minimum number of words.
"""
word_count = len(self._text.split())
interrupt = word_count >= self._min_words
logger.debug(
@@ -37,4 +53,5 @@ class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
return interrupt
async def reset(self):
"""Reset the accumulated text for the next analysis cycle."""
self._text = ""

37
src/pipecat/audio/mixers/base_audio_mixer.py
View File

@@ -4,50 +4,73 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base audio mixer for output transport integration.
Provides the abstract base class for audio mixers that can be integrated with
output transports to mix incoming audio with generated audio from the mixer.
"""
from abc import ABC, abstractmethod
from pipecat.frames.frames import MixerControlFrame
class BaseAudioMixer(ABC):
"""This is a base class for output transport audio mixers. If an audio mixer
"""Base class for output transport audio mixers.
This is a base class for output transport audio mixers. If an audio mixer
is provided to the output transport it will be used to mix the audio frames
coming into to the transport with the audio generated from the mixer. There
are control frames to update mixer settings or to enable or disable the
mixer at runtime.
"""
@abstractmethod
async def start(self, sample_rate: int):
"""This will be called from the output transport when the transport is
"""Initialize the mixer when the output transport starts.
This will be called from the output transport when the transport is
started. It can be used to initialize the mixer. The output transport
sample rate is provided so the mixer can adjust to that sample rate.
Args:
sample_rate: The sample rate of the output transport in Hz.
"""
pass
@abstractmethod
async def stop(self):
"""This will be called from the output transport when the transport is
stopping.
"""Clean up the mixer when the output transport stops.
This will be called from the output transport when the transport is
stopping.
"""
pass
@abstractmethod
async def process_frame(self, frame: MixerControlFrame):
"""This will be called when the output transport receives a
"""Process mixer control frames from the transport.
This will be called when the output transport receives a
MixerControlFrame.
Args:
frame: The mixer control frame to process.
"""
pass
@abstractmethod
async def mix(self, audio: bytes) -> bytes:
"""This is called with the audio that is about to be sent from the
"""Mix transport audio with mixer-generated audio.
This is called with the audio that is about to be sent from the
output transport and that should be mixed with the mixer audio if the
mixer is enabled.
Args:
audio: Raw audio bytes from the transport to mix.
Returns:
Mixed audio bytes combining transport and mixer audio.
"""
pass

59
src/pipecat/audio/mixers/soundfile_mixer.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Soundfile-based audio mixer for file playback integration.
Provides an audio mixer that combines incoming audio with audio loaded from
files using the soundfile library. Supports multiple audio formats and
runtime configuration changes.
"""
import asyncio
from typing import Any, Dict, Mapping
@@ -24,7 +31,9 @@ except ModuleNotFoundError as e:
class SoundfileMixer(BaseAudioMixer):
"""This is an audio mixer that mixes incoming audio with audio from a
"""Audio mixer that combines incoming audio with file-based audio.
This is an audio mixer that mixes incoming audio with audio from a
file. It uses the soundfile library to load files so it supports multiple
formats. The audio files need to only have one channel (mono) and it needs
to match the sample rate of the output transport.
@@ -33,7 +42,6 @@ class SoundfileMixer(BaseAudioMixer):
`MixerUpdateSettingsFrame` has the following settings available: `sound`
(str) and `volume` (float) to be able to update to a different sound file or
to change the volume at runtime.
"""
def __init__(
@@ -46,6 +54,16 @@ class SoundfileMixer(BaseAudioMixer):
loop: bool = True,
**kwargs,
):
"""Initialize the soundfile mixer.
Args:
sound_files: Mapping of sound names to file paths for loading.
default_sound: Name of the default sound to play initially.
volume: Mixing volume level (0.0 to 1.0). Defaults to 0.4.
mixing: Whether mixing is initially enabled. Defaults to True.
loop: Whether to loop audio files when they end. Defaults to True.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._sound_files = sound_files
self._volume = volume
@@ -58,14 +76,28 @@ class SoundfileMixer(BaseAudioMixer):
self._loop = loop
async def start(self, sample_rate: int):
"""Initialize the mixer and load all sound files.
Args:
sample_rate: The sample rate of the output transport in Hz.
"""
self._sample_rate = sample_rate
for sound_name, file_name in self._sound_files.items():
await asyncio.to_thread(self._load_sound_file, sound_name, file_name)
async def stop(self):
"""Clean up mixer resources.
Currently performs no cleanup as sound data is managed by garbage collection.
"""
pass
async def process_frame(self, frame: MixerControlFrame):
"""Process mixer control frames to update settings or enable/disable mixing.
Args:
frame: The mixer control frame to process.
"""
if isinstance(frame, MixerUpdateSettingsFrame):
await self._update_settings(frame)
elif isinstance(frame, MixerEnableFrame):
@@ -73,12 +105,22 @@ class SoundfileMixer(BaseAudioMixer):
pass
async def mix(self, audio: bytes) -> bytes:
"""Mix transport audio with the current sound file.
Args:
audio: Raw audio bytes from the transport to mix.
Returns:
Mixed audio bytes combining transport and file audio.
"""
return self._mix_with_sound(audio)
async def _enable_mixing(self, enable: bool):
"""Enable or disable audio mixing."""
self._mixing = enable
async def _update_settings(self, frame: MixerUpdateSettingsFrame):
"""Update mixer settings from a control frame."""
for setting, value in frame.settings.items():
match setting:
case "sound":
@@ -89,6 +131,11 @@ class SoundfileMixer(BaseAudioMixer):
await self._update_loop(value)
async def _change_sound(self, sound: str):
"""Change the currently playing sound file.
Args:
sound: Name of the sound file to switch to.
"""
if sound in self._sound_files:
self._current_sound = sound
self._sound_pos = 0
@@ -96,12 +143,15 @@ class SoundfileMixer(BaseAudioMixer):
logger.error(f"Sound {sound} is not available")
async def _update_volume(self, volume: float):
"""Update the mixing volume level."""
self._volume = volume
async def _update_loop(self, loop: bool):
"""Update the looping behavior."""
self._loop = loop
def _load_sound_file(self, sound_name: str, file_name: str):
"""Load an audio file into memory for mixing."""
try:
logger.debug(f"Loading mixer sound from {file_name}")
sound, sample_rate = sf.read(file_name, dtype="int16")
@@ -118,10 +168,7 @@ class SoundfileMixer(BaseAudioMixer):
logger.error(f"Unable to open file {file_name}: {e}")
def _mix_with_sound(self, audio: bytes):
"""Mixes raw audio frames with chunks of the same length from the sound
file.
"""
"""Mix raw audio frames with chunks of the same length from the sound file."""
if not self._mixing or not self._current_sound in self._sounds:
return audio

26
src/pipecat/audio/resamplers/base_audio_resampler.py
View File

@@ -4,27 +4,35 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base audio resampler interface for Pipecat.
This module defines the abstract base class for audio resampling implementations,
providing a common interface for converting audio between different sample rates.
"""
from abc import ABC, abstractmethod
class BaseAudioResampler(ABC):
"""Abstract base class for audio resampling. This class defines an
interface for audio resampling implementations.
"""Abstract base class for audio resampling implementations.
This class defines the interface that all audio resampling implementations
must follow, providing a standardized way to convert audio data between
different sample rates.
"""
@abstractmethod
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
"""
Resamples the given audio data to a different sample rate.
"""Resamples the given audio data to a different sample rate.
This is an abstract method that must be implemented in subclasses.
Parameters:
audio (bytes): The audio data to be resampled, represented as a byte string.
in_rate (int): The original sample rate of the audio data (in Hz).
out_rate (int): The desired sample rate for the resampled audio data (in Hz).
Args:
audio: The audio data to be resampled, as raw bytes.
in_rate: The original sample rate of the audio data in Hz.
out_rate: The desired sample rate for the output audio in Hz.
Returns:
bytes: The resampled audio data as a byte string.
The resampled audio data as raw bytes.
"""
pass

27
src/pipecat/audio/resamplers/resampy_resampler.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Resampy-based audio resampler implementation.
This module provides an audio resampler that uses the resampy library
for high-quality audio sample rate conversion.
"""
import numpy as np
import resampy
@@ -11,12 +17,31 @@ from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
class ResampyResampler(BaseAudioResampler):
"""Audio resampler implementation using the resampy library."""
"""Audio resampler implementation using the resampy library.
This resampler uses the resampy library's Kaiser windowing filter
for high-quality audio resampling with good performance characteristics.
"""
def __init__(self, **kwargs):
"""Initialize the resampy resampler.
Args:
**kwargs: Additional keyword arguments (currently unused).
"""
pass
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
"""Resample audio data using resampy library.
Args:
audio: Input audio data as raw bytes (16-bit signed integers).
in_rate: Original sample rate in Hz.
out_rate: Target sample rate in Hz.
Returns:
Resampled audio data as raw bytes (16-bit signed integers).
"""
if in_rate == out_rate:
return audio
audio_data = np.frombuffer(audio, dtype=np.int16)

33
src/pipecat/audio/resamplers/soxr_resampler.py
View File

@@ -4,6 +4,17 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""SoX-based audio resampler implementation.
This module provides an audio resampler that uses the SoX resampler library
for very high-quality audio sample rate conversion.
When to use the SOXRAudioResampler:
1. For batch processing of complete audio files
2. When you have all the audio data available at once
"""
import numpy as np
import soxr
@@ -11,12 +22,32 @@ from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
class SOXRAudioResampler(BaseAudioResampler):
"""Audio resampler implementation using the SoX resampler library."""
"""Audio resampler implementation using the SoX resampler library.
This resampler uses the SoX resampler library configured for very high
quality (VHQ) resampling, providing excellent audio quality at the cost
of additional computational overhead.
"""
def __init__(self, **kwargs):
"""Initialize the SoX audio resampler.
Args:
**kwargs: Additional keyword arguments (currently unused).
"""
pass
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
"""Resample audio data using SoX resampler library.
Args:
audio: Input audio data as raw bytes (16-bit signed integers).
in_rate: Original sample rate in Hz.
out_rate: Target sample rate in Hz.
Returns:
Resampled audio data as raw bytes (16-bit signed integers).
"""
if in_rate == out_rate:
return audio
audio_data = np.frombuffer(audio, dtype=np.int16)

101
src/pipecat/audio/resamplers/soxr_stream_resampler.py Normal file
View File

@@ -0,0 +1,101 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""SoX-based audio resampler stream implementation.
This module provides an audio resampler that uses the SoX ResampleStream library
for very high quality audio sample rate conversion.
When to use the SOXRStreamAudioResampler:
1. For real-time processing scenarios
2. When dealing with very long audio signals
3. When processing audio in chunks or streams
4. When you need to reuse the same resampler configuration multiple times, as it saves initialization overhead
"""
import time
import numpy as np
import soxr
from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
CLEAR_STREAM_AFTER_SECS = 0.2
class SOXRStreamAudioResampler(BaseAudioResampler):
"""Audio resampler implementation using the SoX ResampleStream library.
This resampler uses the SoX ResampleStream library configured for very high
quality (VHQ) resampling, providing excellent audio quality at the cost
of additional computational overhead.
It keeps an internal history which avoids clicks at chunk boundaries.
Notes:
- Only supports mono audio (1 channel).
- Input must be 16-bit signed PCM audio as raw bytes.
"""
def __init__(self, **kwargs):
"""Initialize the resampler.
Args:
**kwargs: Additional keyword arguments (currently unused).
"""
self._in_rate: float | None = None
self._out_rate: float | None = None
self._last_resample_time: float = 0
self._soxr_stream: soxr.ResampleStream | None = None
def _initialize(self, in_rate: float, out_rate: float):
self._in_rate = in_rate
self._out_rate = out_rate
self._last_resample_time = time.time()
self._soxr_stream = soxr.ResampleStream(
in_rate=in_rate, out_rate=out_rate, num_channels=1, quality="VHQ", dtype="int16"
)
def _maybe_clear_internal_state(self):
current_time = time.time()
time_since_last_resample = current_time - self._last_resample_time
# If more than CLEAR_STREAM_AFTER_SECS seconds have passed, clear the resampler state
if time_since_last_resample > CLEAR_STREAM_AFTER_SECS:
if self._soxr_stream:
self._soxr_stream.clear()
self._last_resample_time = current_time
def _maybe_initialize_sox_stream(self, in_rate: int, out_rate: int):
if self._soxr_stream is None:
self._initialize(in_rate, out_rate)
else:
self._maybe_clear_internal_state()
if self._in_rate != in_rate or self._out_rate != out_rate:
raise ValueError(
f"SOXRStreamAudioResampler cannot be reused with different sample rates: "
f"expected {self._in_rate}->{self._out_rate}, got {in_rate}->{out_rate}"
)
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
"""Resample audio data using soxr.ResampleStream resampler library.
Args:
audio: Input audio data as raw bytes (16-bit signed integers).
in_rate: Original sample rate in Hz.
out_rate: Target sample rate in Hz.
Returns:
Resampled audio data as raw bytes (16-bit signed integers).
"""
if in_rate == out_rate:
return audio
self._maybe_initialize_sox_stream(in_rate, out_rate)
audio_data = np.frombuffer(audio, dtype=np.int16)
resampled_audio = self._soxr_stream.resample_chunk(audio_data)
result = resampled_audio.astype(np.int16).tobytes()
return result

24
src/pipecat/audio/turn/base_turn_analyzer.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base turn analyzer for determining end-of-turn in audio conversations.
This module provides the abstract base class and enumeration for analyzing
when a user has finished speaking in a conversation.
"""
from abc import ABC, abstractmethod
from enum import Enum
from typing import Optional, Tuple
@@ -12,6 +18,13 @@ from pipecat.metrics.metrics import MetricsData
class EndOfTurnState(Enum):
"""State enumeration for end-of-turn analysis results.
Parameters:
COMPLETE: The user has finished their turn and stopped speaking.
INCOMPLETE: The user is still speaking or may continue speaking.
"""
COMPLETE = 1
INCOMPLETE = 2
@@ -24,6 +37,12 @@ class BaseTurnAnalyzer(ABC):
"""
def __init__(self, *, sample_rate: Optional[int] = None):
"""Initialize the turn analyzer.
Args:
sample_rate: Optional initial sample rate for audio processing.
If provided, this will be used as the fixed sample rate.
"""
self._init_sample_rate = sample_rate
self._sample_rate = 0
@@ -78,3 +97,8 @@ class BaseTurnAnalyzer(ABC):
EndOfTurnState: The result of the end of turn analysis.
"""
pass
@abstractmethod
def clear(self):
"""Reset the turn analyzer to its initial state."""
pass

67
src/pipecat/audio/turn/smart_turn/base_smart_turn.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Smart turn analyzer base class using ML models for end-of-turn detection.
This module provides the base implementation for smart turn analyzers that use
machine learning models to determine when a user has finished speaking, going
beyond simple silence-based detection.
"""
import time
from abc import abstractmethod
from typing import Any, Dict, Optional, Tuple
@@ -23,6 +30,14 @@ USE_ONLY_LAST_VAD_SEGMENT = True
class SmartTurnParams(BaseModel):
"""Configuration parameters for smart turn analysis.
Parameters:
stop_secs: Maximum silence duration in seconds before ending turn.
pre_speech_ms: Milliseconds of audio to include before speech starts.
max_duration_secs: Maximum duration in seconds for audio segments.
"""
stop_secs: float = STOP_SECS
pre_speech_ms: float = PRE_SPEECH_MS
max_duration_secs: float = MAX_DURATION_SECONDS
@@ -31,13 +46,28 @@ class SmartTurnParams(BaseModel):
class SmartTurnTimeoutException(Exception):
"""Exception raised when smart turn analysis times out."""
pass
class BaseSmartTurn(BaseTurnAnalyzer):
"""Base class for smart turn analyzers using ML models.
Provides common functionality for smart turn detection including audio
buffering, speech tracking, and ML model integration. Subclasses must
implement the specific model prediction logic.
"""
def __init__(
self, *, sample_rate: Optional[int] = None, params: Optional[SmartTurnParams] = None
):
"""Initialize the smart turn analyzer.
Args:
sample_rate: Optional sample rate for audio processing.
params: Configuration parameters for turn analysis behavior.
"""
super().__init__(sample_rate=sample_rate)
self._params = params or SmartTurnParams()
# Configuration
@@ -50,9 +80,23 @@ class BaseSmartTurn(BaseTurnAnalyzer):
@property
def speech_triggered(self) -> bool:
"""Check if speech has been detected and triggered analysis.
Returns:
True if speech has been detected and turn analysis is active.
"""
return self._speech_triggered
def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
"""Append audio data for turn analysis.
Args:
buffer: Raw audio data bytes to append for analysis.
is_speech: Whether the audio buffer contains detected speech.
Returns:
Current end-of-turn state after processing the audio.
"""
# Convert raw audio to float32 format and append to the buffer
audio_int16 = np.frombuffer(buffer, dtype=np.int16)
audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
@@ -92,13 +136,24 @@ class BaseSmartTurn(BaseTurnAnalyzer):
return state
async def analyze_end_of_turn(self) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
"""Analyze the current audio state to determine if turn has ended.
Returns:
Tuple containing the end-of-turn state and optional metrics data
from the ML model analysis.
"""
state, result = await self._process_speech_segment(self._audio_buffer)
if state == EndOfTurnState.COMPLETE or USE_ONLY_LAST_VAD_SEGMENT:
self._clear(state)
logger.debug(f"End of Turn result: {state}")
return state, result
def clear(self):
"""Reset the turn analyzer to its initial state."""
self._clear(EndOfTurnState.COMPLETE)
def _clear(self, turn_state: EndOfTurnState):
"""Clear internal state based on turn completion status."""
# If the state is still incomplete, keep the _speech_triggered as True
self._speech_triggered = turn_state == EndOfTurnState.INCOMPLETE
self._audio_buffer = []
@@ -108,6 +163,7 @@ class BaseSmartTurn(BaseTurnAnalyzer):
async def _process_speech_segment(
self, audio_buffer
) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
"""Process accumulated audio segment using ML model."""
state = EndOfTurnState.INCOMPLETE
if not audio_buffer:
@@ -185,14 +241,5 @@ class BaseSmartTurn(BaseTurnAnalyzer):
@abstractmethod
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
"""Abstract method to predict if a turn has ended based on audio.
Args:
audio_array: Float32 numpy array of audio samples at 16kHz.
Returns:
Dictionary with:
- prediction: 1 if turn is complete, else 0
- probability: Confidence of the prediction
"""
"""Predict end-of-turn using ML model from audio data."""
pass

24
src/pipecat/audio/turn/smart_turn/fal_smart_turn.py
View File

@@ -4,6 +4,16 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Fal.ai smart turn analyzer implementation.
This module provides a smart turn analyzer that uses Fal.ai's hosted smart-turn model
for end-of-turn detection in conversations.
Note: To learn more about the smart-turn model, visit:
- https://fal.ai/models/fal-ai/smart-turn/playground
- https://github.com/pipecat-ai/smart-turn
"""
from typing import Optional
import aiohttp
@@ -12,6 +22,12 @@ from pipecat.audio.turn.smart_turn.http_smart_turn import HttpSmartTurnAnalyzer
class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
"""Smart turn analyzer using Fal.ai's hosted smart-turn model.
Extends HttpSmartTurnAnalyzer to provide integration with Fal.ai's
smart turn detection API endpoint with proper authentication.
"""
def __init__(
self,
*,
@@ -20,6 +36,14 @@ class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
api_key: Optional[str] = None,
**kwargs,
):
"""Initialize the Fal.ai smart turn analyzer.
Args:
aiohttp_session: HTTP client session for making API requests.
url: Fal.ai API endpoint URL for smart turn detection.
api_key: API key for authenticating with Fal.ai service.
**kwargs: Additional arguments passed to parent HttpSmartTurnAnalyzer.
"""
headers = {}
if api_key:
headers = {"Authorization": f"Key {api_key}"}

23
src/pipecat/audio/turn/smart_turn/http_smart_turn.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""HTTP-based smart turn analyzer for remote ML inference.
This module provides a smart turn analyzer that sends audio data to remote
HTTP endpoints for ML-based end-of-turn detection.
"""
import asyncio
import io
from typing import Any, Dict, Optional
@@ -16,6 +22,12 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn, SmartTu
class HttpSmartTurnAnalyzer(BaseSmartTurn):
"""Smart turn analyzer using HTTP-based ML inference.
Sends audio data to remote HTTP endpoints for ML-based end-of-turn
prediction. Handles serialization, HTTP communication, and error recovery.
"""
def __init__(
self,
*,
@@ -24,12 +36,21 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
headers: Optional[Dict[str, str]] = None,
**kwargs,
):
"""Initialize the HTTP smart turn analyzer.
Args:
url: HTTP endpoint URL for the smart turn ML service.
aiohttp_session: HTTP client session for making requests.
headers: Optional HTTP headers to include in requests.
**kwargs: Additional arguments passed to BaseSmartTurn.
"""
super().__init__(**kwargs)
self._url = url
self._headers = headers or {}
self._aiohttp_session = aiohttp_session
def _serialize_array(self, audio_array: np.ndarray) -> bytes:
"""Serialize NumPy audio array to bytes for HTTP transmission."""
logger.trace("Serializing NumPy array to bytes...")
buffer = io.BytesIO()
np.save(buffer, audio_array)
@@ -38,6 +59,7 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
return serialized_bytes
async def _send_raw_request(self, data_bytes: bytes) -> Dict[str, Any]:
"""Send raw audio data to the HTTP endpoint for prediction."""
headers = {"Content-Type": "application/octet-stream"}
headers.update(self._headers)
@@ -83,6 +105,7 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
raise Exception("Failed to send raw request to Daily Smart Turn.")
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
"""Predict end-of-turn using remote HTTP ML service."""
try:
serialized_array = self._serialize_array(audio_array)
return await self._send_raw_request(serialized_array)

23
src/pipecat/audio/turn/smart_turn/local_coreml_smart_turn.py
View File

@@ -4,6 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Local CoreML smart turn analyzer for on-device ML inference.
This module provides a smart turn analyzer that uses CoreML models for
local end-of-turn detection without requiring network connectivity.
"""
from typing import Any, Dict
@@ -25,7 +30,24 @@ except ModuleNotFoundError as e:
class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
"""Local smart turn analyzer using CoreML models.
Provides end-of-turn detection using locally-stored CoreML models,
enabling offline operation without network dependencies. Optimized
for Apple Silicon and other CoreML-compatible hardware.
"""
def __init__(self, *, smart_turn_model_path: str, **kwargs):
"""Initialize the local CoreML smart turn analyzer.
Args:
smart_turn_model_path: Path to directory containing the CoreML model
and feature extractor files.
**kwargs: Additional arguments passed to BaseSmartTurn.
Raises:
Exception: If smart_turn_model_path is not provided or model loading fails.
"""
super().__init__(**kwargs)
if not smart_turn_model_path:
@@ -41,6 +63,7 @@ class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
logger.debug("Loaded Local Smart Turn")
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
"""Predict end-of-turn using local CoreML model."""
inputs = self._turn_processor(
audio_array,
sampling_rate=16000,

20
src/pipecat/audio/turn/smart_turn/local_smart_turn.py
View File

@@ -4,6 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Local PyTorch smart turn analyzer for on-device ML inference.
This module provides a smart turn analyzer that uses PyTorch models for
local end-of-turn detection without requiring network connectivity.
"""
from typing import Any, Dict
@@ -24,7 +29,21 @@ except ModuleNotFoundError as e:
class LocalSmartTurnAnalyzer(BaseSmartTurn):
"""Local smart turn analyzer using PyTorch models.
Provides end-of-turn detection using locally-stored PyTorch models,
enabling offline operation without network dependencies. Uses
Wav2Vec2-BERT architecture for audio sequence classification.
"""
def __init__(self, *, smart_turn_model_path: str, **kwargs):
"""Initialize the local PyTorch smart turn analyzer.
Args:
smart_turn_model_path: Path to directory containing the PyTorch model
and feature extractor files. If empty, uses default HuggingFace model.
**kwargs: Additional arguments passed to BaseSmartTurn.
"""
super().__init__(**kwargs)
if not smart_turn_model_path:
@@ -46,6 +65,7 @@ class LocalSmartTurnAnalyzer(BaseSmartTurn):
logger.debug("Loaded Local Smart Turn")
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
"""Predict end-of-turn using local PyTorch model."""
inputs = self._turn_processor(
audio_array,
sampling_rate=16000,

161
src/pipecat/audio/utils.py
View File

@@ -4,21 +4,87 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Audio utility functions for Pipecat.
This module provides common audio processing utilities including mixing,
format conversion, volume calculation, and codec transformations for
various audio formats used in Pipecat pipelines.
"""
import audioop
import numpy as np
import pyloudnorm as pyln
import soxr
from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
from pipecat.audio.resamplers.soxr_resampler import SOXRAudioResampler
from pipecat.audio.resamplers.soxr_stream_resampler import SOXRStreamAudioResampler
def create_default_resampler(**kwargs) -> BaseAudioResampler:
"""Create a default audio resampler instance.
. deprecated:: 0.0.74
This function is deprecated and will be removed in a future version.
Use `create_stream_resampler` for real-time processing scenarios or
`create_file_resampler` for batch processing of complete audio files.
Args:
**kwargs: Additional keyword arguments passed to the resampler constructor.
Returns:
A configured SOXRAudioResampler instance.
"""
import warnings
warnings.warn(
"`create_default_resampler` is deprecated. "
"Use `create_stream_resampler` for real-time processing scenarios or "
"`create_file_resampler` for batch processing of complete audio files.",
DeprecationWarning,
stacklevel=2,
)
return SOXRAudioResampler(**kwargs)
def create_file_resampler(**kwargs) -> BaseAudioResampler:
"""Create an audio resampler instance for batch processing of complete audio files.
Args:
**kwargs: Additional keyword arguments passed to the resampler constructor.
Returns:
A configured SOXRAudioResampler instance.
"""
return SOXRAudioResampler(**kwargs)
def create_stream_resampler(**kwargs) -> BaseAudioResampler:
"""Create a stream audio resampler instance.
Args:
**kwargs: Additional keyword arguments passed to the resampler constructor.
Returns:
A configured SOXRStreamAudioResampler instance.
"""
return SOXRStreamAudioResampler(**kwargs)
def mix_audio(audio1: bytes, audio2: bytes) -> bytes:
"""Mix two audio streams together by adding their samples.
Both audio streams are assumed to be 16-bit signed integer PCM data.
If the streams have different lengths, the shorter one is zero-padded
to match the longer stream.
Args:
audio1: First audio stream as raw bytes (16-bit signed integers).
audio2: Second audio stream as raw bytes (16-bit signed integers).
Returns:
Mixed audio data as raw bytes with samples clipped to 16-bit range.
"""
data1 = np.frombuffer(audio1, dtype=np.int16)
data2 = np.frombuffer(audio2, dtype=np.int16)
@@ -37,6 +103,19 @@ def mix_audio(audio1: bytes, audio2: bytes) -> bytes:
def interleave_stereo_audio(left_audio: bytes, right_audio: bytes) -> bytes:
"""Interleave left and right mono audio channels into stereo audio.
Takes two mono audio streams and combines them into a single stereo
stream by interleaving the samples (L, R, L, R, ...). If the channels
have different lengths, both are truncated to the shorter length.
Args:
left_audio: Left channel audio as raw bytes (16-bit signed integers).
right_audio: Right channel audio as raw bytes (16-bit signed integers).
Returns:
Interleaved stereo audio data as raw bytes.
"""
left = np.frombuffer(left_audio, dtype=np.int16)
right = np.frombuffer(right_audio, dtype=np.int16)
@@ -50,12 +129,34 @@ def interleave_stereo_audio(left_audio: bytes, right_audio: bytes) -> bytes:
def normalize_value(value, min_value, max_value):
"""Normalize a value to the range [0, 1] and clamp it to bounds.
Args:
value: The value to normalize.
min_value: The minimum value of the input range.
max_value: The maximum value of the input range.
Returns:
Normalized value clamped to the range [0, 1].
"""
normalized = (value - min_value) / (max_value - min_value)
normalized_clamped = max(0, min(1, normalized))
return normalized_clamped
def calculate_audio_volume(audio: bytes, sample_rate: int) -> float:
"""Calculate the loudness level of audio data using EBU R128 standard.
Uses the pyloudnorm library to calculate integrated loudness according
to the EBU R128 recommendation, then normalizes the result to [0, 1].
Args:
audio: Audio data as raw bytes (16-bit signed integers).
sample_rate: Sample rate of the audio in Hz.
Returns:
Normalized loudness value between 0 (quiet) and 1 (loud).
"""
audio_np = np.frombuffer(audio, dtype=np.int16)
audio_float = audio_np.astype(np.float64)
@@ -71,12 +172,37 @@ def calculate_audio_volume(audio: bytes, sample_rate: int) -> float:
def exp_smoothing(value: float, prev_value: float, factor: float) -> float:
"""Apply exponential smoothing to a value.
Exponential smoothing is used to reduce noise in time-series data by
giving more weight to recent values while still considering historical data.
Args:
value: The new value to incorporate.
prev_value: The previous smoothed value.
factor: Smoothing factor between 0 and 1. Higher values give more
weight to the new value.
Returns:
The exponentially smoothed value.
"""
return prev_value + factor * (value - prev_value)
async def ulaw_to_pcm(
ulaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
):
"""Convert μ-law encoded audio to PCM and optionally resample.
Args:
ulaw_bytes: μ-law encoded audio data as raw bytes.
in_rate: Original sample rate of the μ-law audio in Hz.
out_rate: Desired output sample rate in Hz.
resampler: Audio resampler instance for rate conversion.
Returns:
PCM audio data as raw bytes at the specified output rate.
"""
# Convert μ-law to PCM
in_pcm_bytes = audioop.ulaw2lin(ulaw_bytes, 2)
@@ -87,6 +213,17 @@ async def ulaw_to_pcm(
async def pcm_to_ulaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
"""Convert PCM audio to μ-law encoding and optionally resample.
Args:
pcm_bytes: PCM audio data as raw bytes (16-bit signed integers).
in_rate: Original sample rate of the PCM audio in Hz.
out_rate: Desired output sample rate in Hz.
resampler: Audio resampler instance for rate conversion.
Returns:
μ-law encoded audio data as raw bytes at the specified output rate.
"""
# Resample
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
@@ -99,6 +236,17 @@ async def pcm_to_ulaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler:
async def alaw_to_pcm(
alaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
) -> bytes:
"""Convert A-law encoded audio to PCM and optionally resample.
Args:
alaw_bytes: A-law encoded audio data as raw bytes.
in_rate: Original sample rate of the A-law audio in Hz.
out_rate: Desired output sample rate in Hz.
resampler: Audio resampler instance for rate conversion.
Returns:
PCM audio data as raw bytes at the specified output rate.
"""
# Convert a-law to PCM
in_pcm_bytes = audioop.alaw2lin(alaw_bytes, 2)
@@ -109,6 +257,17 @@ async def alaw_to_pcm(
async def pcm_to_alaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
"""Convert PCM audio to A-law encoding and optionally resample.
Args:
pcm_bytes: PCM audio data as raw bytes (16-bit signed integers).
in_rate: Original sample rate of the PCM audio in Hz.
out_rate: Desired output sample rate in Hz.
resampler: Audio resampler instance for rate conversion.
Returns:
A-law encoded audio data as raw bytes at the specified output rate.
"""
# Resample
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)

63
src/pipecat/audio/vad/silero.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Silero Voice Activity Detection (VAD) implementation for Pipecat.
This module provides a VAD analyzer based on the Silero VAD ONNX model,
which can detect voice activity in audio streams with high accuracy.
Supports 8kHz and 16kHz sample rates.
"""
import time
from typing import Optional
@@ -25,11 +32,20 @@ except ModuleNotFoundError as e:
class SileroOnnxModel:
"""ONNX runtime wrapper for the Silero VAD model.
Provides voice activity detection using the pre-trained Silero VAD model
with ONNX runtime for efficient inference. Handles model state management
and input validation for audio processing.
"""
def __init__(self, path, force_onnx_cpu=True):
import numpy as np
global np
"""Initialize the Silero ONNX model.
Args:
path: Path to the ONNX model file.
force_onnx_cpu: Whether to force CPU execution provider.
"""
opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1
@@ -45,6 +61,7 @@ class SileroOnnxModel:
self.sample_rates = [8000, 16000]
def _validate_input(self, x, sr: int):
"""Validate and preprocess input audio data."""
if np.ndim(x) == 1:
x = np.expand_dims(x, 0)
if np.ndim(x) > 2:
@@ -60,12 +77,18 @@ class SileroOnnxModel:
return x, sr
def reset_states(self, batch_size=1):
"""Reset the internal model states.
Args:
batch_size: Batch size for state initialization. Defaults to 1.
"""
self._state = np.zeros((2, batch_size, 128), dtype="float32")
self._context = np.zeros((batch_size, 0), dtype="float32")
self._last_sr = 0
self._last_batch_size = 0
def __call__(self, x, sr: int):
"""Process audio input through the VAD model."""
x, sr = self._validate_input(x, sr)
num_samples = 512 if sr == 16000 else 256
@@ -105,7 +128,20 @@ class SileroOnnxModel:
class SileroVADAnalyzer(VADAnalyzer):
"""Voice Activity Detection analyzer using the Silero VAD model.
Implements VAD analysis using the pre-trained Silero ONNX model for
accurate voice activity detection. Supports 8kHz and 16kHz sample rates
with automatic model state management and periodic resets.
"""
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
"""Initialize the Silero VAD analyzer.
Args:
sample_rate: Audio sample rate (8000 or 16000 Hz). If None, will be set later.
params: VAD parameters for detection thresholds and timing.
"""
super().__init__(sample_rate=sample_rate, params=params)
logger.debug("Loading Silero VAD model...")
@@ -137,6 +173,14 @@ class SileroVADAnalyzer(VADAnalyzer):
#
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate for audio processing.
Args:
sample_rate: Audio sample rate (must be 8000 or 16000 Hz).
Raises:
ValueError: If sample rate is not 8000 or 16000 Hz.
"""
if sample_rate != 16000 and sample_rate != 8000:
raise ValueError(
f"Silero VAD sample rate needs to be 16000 or 8000 (sample rate: {sample_rate})"
@@ -145,9 +189,22 @@ class SileroVADAnalyzer(VADAnalyzer):
super().set_sample_rate(sample_rate)
def num_frames_required(self) -> int:
"""Get the number of audio frames required for VAD analysis.
Returns:
Number of frames required (512 for 16kHz, 256 for 8kHz).
"""
return 512 if self.sample_rate == 16000 else 256
def voice_confidence(self, buffer) -> float:
"""Calculate voice activity confidence for the given audio buffer.
Args:
buffer: Audio buffer to analyze.
Returns:
Voice confidence score between 0.0 and 1.0.
"""
try:
audio_int16 = np.frombuffer(buffer, np.int16)
# Divide by 32768 because we have signed 16-bit data.

88
src/pipecat/audio/vad/vad_analyzer.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Voice Activity Detection (VAD) analyzer base classes and utilities.
This module provides the abstract base class for VAD analyzers and associated
data structures for voice activity detection in audio streams. Includes state
management, parameter configuration, and audio analysis framework.
"""
from abc import ABC, abstractmethod
from enum import Enum
from typing import Optional
@@ -20,6 +27,15 @@ VAD_MIN_VOLUME = 0.6
class VADState(Enum):
"""Voice Activity Detection states.
Parameters:
QUIET: No voice activity detected.
STARTING: Voice activity beginning, transitioning from quiet.
SPEAKING: Active voice detected and confirmed.
STOPPING: Voice activity ending, transitioning to quiet.
"""
QUIET = 1
STARTING = 2
SPEAKING = 3
@@ -27,6 +43,15 @@ class VADState(Enum):
class VADParams(BaseModel):
"""Configuration parameters for Voice Activity Detection.
Parameters:
confidence: Minimum confidence threshold for voice detection.
start_secs: Duration to wait before confirming voice start.
stop_secs: Duration to wait before confirming voice stop.
min_volume: Minimum audio volume threshold for voice detection.
"""
confidence: float = VAD_CONFIDENCE
start_secs: float = VAD_START_SECS
stop_secs: float = VAD_STOP_SECS
@@ -34,7 +59,20 @@ class VADParams(BaseModel):
class VADAnalyzer(ABC):
"""Abstract base class for Voice Activity Detection analyzers.
Provides the framework for implementing VAD analysis with configurable
parameters, state management, and audio processing capabilities.
Subclasses must implement the core voice confidence calculation.
"""
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
"""Initialize the VAD analyzer.
Args:
sample_rate: Audio sample rate in Hz. If None, will be set later.
params: VAD parameters for detection configuration.
"""
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._params = params or VADParams()
@@ -48,29 +86,67 @@ class VADAnalyzer(ABC):
@property
def sample_rate(self) -> int:
"""Get the current sample rate.
Returns:
Current audio sample rate in Hz.
"""
return self._sample_rate
@property
def num_channels(self) -> int:
"""Get the number of audio channels.
Returns:
Number of audio channels (always 1 for mono).
"""
return self._num_channels
@property
def params(self) -> VADParams:
"""Get the current VAD parameters.
Returns:
Current VAD configuration parameters.
"""
return self._params
@abstractmethod
def num_frames_required(self) -> int:
"""Get the number of audio frames required for analysis.
Returns:
Number of frames needed for VAD processing.
"""
pass
@abstractmethod
def voice_confidence(self, buffer) -> float:
"""Calculate voice activity confidence for the given audio buffer.
Args:
buffer: Audio buffer to analyze.
Returns:
Voice confidence score between 0.0 and 1.0.
"""
pass
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate for audio processing.
Args:
sample_rate: Audio sample rate in Hz.
"""
self._sample_rate = self._init_sample_rate or sample_rate
self.set_params(self._params)
def set_params(self, params: VADParams):
"""Set VAD parameters and recalculate internal values.
Args:
params: VAD parameters for detection configuration.
"""
logger.debug(f"Setting VAD params to: {params}")
self._params = params
self._vad_frames = self.num_frames_required()
@@ -85,10 +161,22 @@ class VADAnalyzer(ABC):
self._vad_state: VADState = VADState.QUIET
def _get_smoothed_volume(self, audio: bytes) -> float:
"""Calculate smoothed audio volume using exponential smoothing."""
volume = calculate_audio_volume(audio, self.sample_rate)
return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
def analyze_audio(self, buffer) -> VADState:
"""Analyze audio buffer and return current VAD state.
Processes incoming audio data, maintains internal state, and determines
voice activity status based on confidence and volume thresholds.
Args:
buffer: Audio buffer to analyze.
Returns:
Current VAD state after processing the buffer.
"""
self._vad_buffer += buffer
num_required_bytes = self._vad_frames_num_bytes

19
src/pipecat/clocks/base_clock.py
View File

@@ -4,14 +4,33 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base clock interface for Pipecat timing operations."""
from abc import ABC, abstractmethod
class BaseClock(ABC):
"""Abstract base class for clock implementations.
Provides a common interface for timing operations used in Pipecat
for synchronization, scheduling, and time-based processing.
"""
@abstractmethod
def get_time(self) -> int:
"""Get the current time value.
Returns:
The current time as an integer value. The specific unit and
reference point depend on the concrete implementation.
"""
pass
@abstractmethod
def start(self):
"""Start or initialize the clock.
Performs any necessary initialization or starts the timing mechanism.
This method should be called before using get_time().
"""
pass

25
src/pipecat/clocks/system_clock.py
View File

@@ -4,17 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""System clock implementation for Pipecat."""
import time
from pipecat.clocks.base_clock import BaseClock
class SystemClock(BaseClock):
"""A monotonic clock implementation using system time.
Provides high-precision timing using the system's monotonic clock,
which is not affected by system clock adjustments and is suitable
for measuring elapsed time in real-time applications.
"""
def __init__(self):
"""Initialize the system clock.
The clock starts in an uninitialized state and must be started
explicitly using the start() method before time measurement begins.
"""
self._time = 0
def get_time(self) -> int:
"""Get the elapsed time since the clock was started.
Returns:
The elapsed time in nanoseconds since start() was called.
Returns 0 if the clock has not been started yet.
"""
return time.monotonic_ns() - self._time if self._time > 0 else 0
def start(self):
"""Start the clock and begin time measurement.
Records the current monotonic time as the reference point
for all subsequent get_time() calls.
"""
self._time = time.monotonic_ns()

25
src/pipecat/examples/daily_runner.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Daily.co room configuration utilities for Pipecat examples."""
import argparse
import os
from typing import Optional
@@ -14,6 +16,17 @@ from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
async def configure(aiohttp_session: aiohttp.ClientSession):
"""Configure Daily.co room URL and token from arguments or environment.
Args:
aiohttp_session: HTTP session for making API requests.
Returns:
Tuple containing the room URL and authentication token.
Raises:
Exception: If room URL or API key are not provided.
"""
(url, token, _) = await configure_with_args(aiohttp_session)
return (url, token)
@@ -21,6 +34,18 @@ async def configure(aiohttp_session: aiohttp.ClientSession):
async def configure_with_args(
aiohttp_session: aiohttp.ClientSession, parser: Optional[argparse.ArgumentParser] = None
):
"""Configure Daily.co room with command-line argument parsing.
Args:
aiohttp_session: HTTP session for making API requests.
parser: Optional argument parser. If None, creates a default one.
Returns:
Tuple containing room URL, authentication token, and parsed arguments.
Raises:
Exception: If room URL or API key are not provided via arguments or environment.
"""
if not parser:
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
parser.add_argument(

187
src/pipecat/examples/run.py
View File

@@ -4,10 +4,18 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Pipecat example runner with support for multiple transport types.
This module provides a unified interface for running Pipecat examples across
different transport types including Daily.co, WebRTC, and Twilio. It handles
setup, configuration, and lifecycle management for each transport type.
"""
import argparse
import asyncio
import json
import os
import re
import sys
from contextlib import asynccontextmanager
from typing import Any, Callable, Dict, Mapping, Optional
@@ -35,6 +43,15 @@ load_dotenv(override=True)
def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
"""Get client identifier from transport-specific client object.
Args:
transport: The transport instance.
client: Transport-specific client object.
Returns:
Client identifier string, empty if transport not supported.
"""
if isinstance(transport, SmallWebRTCTransport):
return client.pc_id
elif isinstance(transport, DailyTransport):
@@ -46,6 +63,13 @@ def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
async def maybe_capture_participant_camera(
transport: BaseTransport, client: Any, framerate: int = 0
):
"""Capture participant camera video if transport supports it.
Args:
transport: The transport instance.
client: Transport-specific client object.
framerate: Video capture framerate. Defaults to 0 (auto).
"""
if isinstance(transport, DailyTransport):
await transport.capture_participant_video(
client["id"], framerate=framerate, video_source="camera"
@@ -55,17 +79,84 @@ async def maybe_capture_participant_camera(
async def maybe_capture_participant_screen(
transport: BaseTransport, client: Any, framerate: int = 0
):
"""Capture participant screen video if transport supports it.
Args:
transport: The transport instance.
client: Transport-specific client object.
framerate: Video capture framerate. Defaults to 0 (auto).
"""
if isinstance(transport, DailyTransport):
await transport.capture_participant_video(
client["id"], framerate=framerate, video_source="screenVideo"
)
def smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str:
"""Clean up ICE candidates in SDP text for SmallWebRTC.
Args:
text: SDP text to clean up.
pattern: Pattern to match for candidate filtering.
Returns:
Cleaned SDP text with filtered ICE candidates.
"""
result = []
lines = text.splitlines()
for line in lines:
if re.search("a=candidate", line):
if re.search(pattern, line) and not re.search("raddr", line):
result.append(line)
else:
result.append(line)
return "\r\n".join(result)
def smallwebrtc_sdp_cleanup_fingerprints(text: str) -> str:
"""Remove unsupported fingerprint algorithms from SDP text.
Args:
text: SDP text to clean up.
Returns:
SDP text with sha-384 and sha-512 fingerprints removed.
"""
result = []
lines = text.splitlines()
for line in lines:
if not re.search("sha-384", line) and not re.search("sha-512", line):
result.append(line)
return "\r\n".join(result)
def smallwebrtc_sdp_munging(sdp: str, host: str) -> str:
"""Apply SDP modifications for SmallWebRTC compatibility.
Args:
sdp: Original SDP string.
host: Host address for ICE candidate filtering.
Returns:
Modified SDP string with fingerprint and ICE candidate cleanup.
"""
sdp = smallwebrtc_sdp_cleanup_fingerprints(sdp)
sdp = smallwebrtc_sdp_cleanup_ice_candidates(sdp, host)
return sdp
def run_example_daily(
run_example: Callable,
args: argparse.Namespace,
params: DailyParams,
transport_params: Mapping[str, Callable] = {},
):
"""Run example using Daily.co transport.
Args:
run_example: The example function to run.
args: Parsed command-line arguments.
transport_params: Mapping of transport names to parameter factory functions.
"""
logger.info("Running example with DailyTransport...")
from pipecat.examples.daily_runner import configure
@@ -75,6 +166,7 @@ def run_example_daily(
(room_url, token) = await configure(session)
# Run example function with DailyTransport transport arguments.
params: DailyParams = transport_params[args.transport]()
transport = DailyTransport(room_url, token, "Pipecat", params=params)
await run_example(transport, args, True)
@@ -84,8 +176,15 @@ def run_example_daily(
def run_example_webrtc(
run_example: Callable,
args: argparse.Namespace,
params: TransportParams,
transport_params: Mapping[str, Callable] = {},
):
"""Run example using WebRTC transport with FastAPI server.
Args:
run_example: The example function to run.
args: Parsed command-line arguments.
transport_params: Mapping of transport names to parameter factory functions.
"""
logger.info("Running example with SmallWebRTCTransport...")
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
@@ -95,21 +194,25 @@ def run_example_webrtc(
# Store connections by pc_id
pcs_map: Dict[str, SmallWebRTCConnection] = {}
ice_servers = [
IceServer(
urls="stun:stun.l.google.com:19302",
)
]
# Mount the frontend at /
app.mount("/client", SmallWebRTCPrebuiltUI)
@app.get("/", include_in_schema=False)
async def root_redirect():
"""Redirect root requests to client interface."""
return RedirectResponse(url="/client/")
@app.post("/api/offer")
async def offer(request: dict, background_tasks: BackgroundTasks):
"""Handle WebRTC offer requests and manage peer connections.
Args:
request: WebRTC offer request containing SDP and connection details.
background_tasks: FastAPI background tasks for running examples.
Returns:
WebRTC answer with connection details.
"""
pc_id = request.get("pc_id")
if pc_id and pc_id in pcs_map:
@@ -121,19 +224,29 @@ def run_example_webrtc(
restart_pc=request.get("restart_pc", False),
)
else:
pipecat_connection = SmallWebRTCConnection(ice_servers)
pipecat_connection = SmallWebRTCConnection()
await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
@pipecat_connection.event_handler("closed")
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
"""Handle WebRTC connection closure and cleanup.
Args:
webrtc_connection: The closed WebRTC connection.
"""
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
pcs_map.pop(webrtc_connection.pc_id, None)
# Run example function with SmallWebRTC transport arguments.
params: TransportParams = transport_params[args.transport]()
transport = SmallWebRTCTransport(params=params, webrtc_connection=pipecat_connection)
background_tasks.add_task(run_example, transport, args, False)
answer = pipecat_connection.get_answer()
if args.esp32 and args.host:
answer["sdp"] = smallwebrtc_sdp_munging(answer["sdp"], args.host)
# Updating the peer connection inside the map
pcs_map[answer["pc_id"]] = pipecat_connection
@@ -141,6 +254,14 @@ def run_example_webrtc(
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Manage FastAPI application lifecycle and cleanup connections.
Args:
app: The FastAPI application instance.
Yields:
Control to the FastAPI application runtime.
"""
yield # Run app
coros = [pc.disconnect() for pc in pcs_map.values()]
await asyncio.gather(*coros)
@@ -152,8 +273,15 @@ def run_example_webrtc(
def run_example_twilio(
run_example: Callable,
args: argparse.Namespace,
params: FastAPIWebsocketParams,
transport_params: Mapping[str, Callable] = {},
):
"""Run example using Twilio transport with FastAPI WebSocket server.
Args:
run_example: The example function to run.
args: Parsed command-line arguments.
transport_params: Mapping of transport names to parameter factory functions.
"""
logger.info("Running example with FastAPIWebsocketTransport (Twilio)...")
app = FastAPI()
@@ -168,6 +296,11 @@ def run_example_twilio(
@app.post("/")
async def start_call():
"""Handle Twilio webhook and return TwiML response.
Returns:
TwiML XML response directing call to WebSocket stream.
"""
logger.debug("POST TwiML")
xml_content = f"""<?xml version="1.0" encoding="UTF-8"?>
@@ -182,6 +315,11 @@ def run_example_twilio(
@app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
"""Handle Twilio WebSocket connections for voice streaming.
Args:
websocket: The WebSocket connection from Twilio.
"""
await websocket.accept()
logger.debug("WebSocket connection accepted")
@@ -195,6 +333,7 @@ def run_example_twilio(
call_sid = call_data["start"]["callSid"]
# Create websocket transport and update params.
params: FastAPIWebsocketParams = transport_params[args.transport]()
params.add_wav_header = False
params.serializer = TwilioFrameSerializer(
stream_sid=stream_sid,
@@ -213,18 +352,24 @@ def run_main(
args: argparse.Namespace,
transport_params: Mapping[str, Callable] = {},
):
"""Run the example with the specified transport type.
Args:
run_example: The example function to run.
args: Parsed command-line arguments.
transport_params: Mapping of transport names to parameter factory functions.
"""
if args.transport not in transport_params:
logger.error(f"Transport '{args.transport}' not supported by this example")
return
params = transport_params[args.transport]()
match args.transport:
case "daily":
run_example_daily(run_example, args, params)
run_example_daily(run_example, args, transport_params)
case "webrtc":
run_example_webrtc(run_example, args, params)
run_example_webrtc(run_example, args, transport_params)
case "twilio":
run_example_twilio(run_example, args, params)
run_example_twilio(run_example, args, transport_params)
def main(
@@ -233,6 +378,13 @@ def main(
parser: Optional[argparse.ArgumentParser] = None,
transport_params: Mapping[str, Callable] = {},
):
"""Main entry point for running Pipecat examples with transport selection.
Args:
run_example: The example function to run.
parser: Optional argument parser. If None, creates a default one.
transport_params: Mapping of transport names to parameter factory functions.
"""
if not parser:
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
parser.add_argument(
@@ -252,9 +404,16 @@ def main(
parser.add_argument(
"--proxy", "-x", help="A public proxy host name (no protocol, e.g. proxy.example.com)"
)
parser.add_argument(
"--esp32", action="store_true", default=False, help="Perform SDP munging for the ESP32"
)
parser.add_argument("--verbose", "-v", action="count", default=0)
args = parser.parse_args()
if args.esp32 and args.host == "localhost":
logger.error("For ESP32, you need to specify `--host IP` so we can do SDP munging.")
return
# Log level
logger.remove(0)
logger.add(sys.stderr, level="TRACE" if args.verbose else "DEBUG")

725
src/pipecat/frames/frames.py
View File

File diff suppressed because it is too large Load Diff

65
src/pipecat/metrics/metrics.py
View File

@@ -1,39 +1,102 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Metrics data models for Pipecat framework.
This module defines Pydantic models for various types of metrics data
collected throughout the pipeline, including timing, token usage, and
processing statistics.
"""
from typing import Optional
from pydantic import BaseModel
class MetricsData(BaseModel):
"""Base class for all metrics data.
Parameters:
processor: Name of the processor generating the metrics.
model: Optional model name associated with the metrics.
"""
processor: str
model: Optional[str] = None
class TTFBMetricsData(MetricsData):
"""Time To First Byte (TTFB) metrics data.
Parameters:
value: TTFB measurement in seconds.
"""
value: float
class ProcessingMetricsData(MetricsData):
"""General processing time metrics data.
Parameters:
value: Processing time measurement in seconds.
"""
value: float
class LLMTokenUsage(BaseModel):
"""Token usage statistics for LLM operations.
Parameters:
prompt_tokens: Number of tokens in the input prompt.
completion_tokens: Number of tokens in the generated completion.
total_tokens: Total number of tokens used (prompt + completion).
cache_read_input_tokens: Number of tokens read from cache, if applicable.
cache_creation_input_tokens: Number of tokens used to create cache entries, if applicable.
"""
prompt_tokens: int
completion_tokens: int
total_tokens: int
cache_read_input_tokens: Optional[int] = None
cache_creation_input_tokens: Optional[int] = None
reasoning_tokens: Optional[int] = None
class LLMUsageMetricsData(MetricsData):
"""LLM token usage metrics data.
Parameters:
value: Token usage statistics for the LLM operation.
"""
value: LLMTokenUsage
class TTSUsageMetricsData(MetricsData):
"""Text-to-Speech usage metrics data.
Parameters:
value: Number of characters processed by TTS.
"""
value: int
class SmartTurnMetricsData(MetricsData):
"""Metrics data for smart turn predictions."""
"""Metrics data for smart turn predictions.
Parameters:
is_complete: Whether the turn is predicted to be complete.
probability: Confidence probability of the turn completion prediction.
inference_time_ms: Time taken for inference in milliseconds.
server_total_time_ms: Total server processing time in milliseconds.
e2e_processing_time_ms: End-to-end processing time in milliseconds.
"""
is_complete: bool
probability: float

41
src/pipecat/observers/base_observer.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base observer classes for monitoring frame flow in the Pipecat pipeline.
This module provides the foundation for observing frame transfers between
processors without modifying the pipeline structure. Observers can be used
for logging, debugging, analytics, and monitoring pipeline behavior.
"""
from abc import abstractmethod
from dataclasses import dataclass
@@ -18,19 +25,19 @@ if TYPE_CHECKING:
@dataclass
class FramePushed:
"""Represents an event where a frame is pushed from one processor to another
within the pipeline.
"""Event data for frame transfers between processors in the pipeline.
This data structure is typically used by observers to track the flow of
frames through the pipeline for logging, debugging, or analytics purposes.
Attributes:
source (FrameProcessor): The processor sending the frame.
destination (FrameProcessor): The processor receiving the frame.
frame (Frame): The frame being transferred.
direction (FrameDirection): The direction of the transfer (e.g., downstream or upstream).
timestamp (int): The time when the frame was pushed, based on the pipeline clock.
Represents an event where a frame is pushed from one processor to another
within the pipeline. This data structure is typically used by observers
to track the flow of frames through the pipeline for logging, debugging,
or analytics purposes.
Parameters:
source: The processor sending the frame.
destination: The processor receiving the frame.
frame: The frame being transferred.
direction: The direction of the transfer (e.g., downstream or upstream).
timestamp: The time when the frame was pushed, based on the pipeline clock.
"""
source: "FrameProcessor"
@@ -41,11 +48,12 @@ class FramePushed:
class BaseObserver(BaseObject):
"""This is the base class for pipeline frame observers. Observers can view
all the frames that go through the pipeline without the need to inject
processors in the pipeline. This can be useful, for example, to implement
frame loggers or debuggers among other things.
"""Base class for pipeline frame observers.
Observers can view all frames that flow through the pipeline without
needing to inject processors into the pipeline structure. This enables
non-intrusive monitoring capabilities such as frame logging, debugging,
performance analysis, and analytics collection.
"""
@abstractmethod
@@ -57,7 +65,6 @@ class BaseObserver(BaseObject):
transferred through the pipeline.
Args:
data (FramePushed): The event data containing details about the frame transfer.
data: The event data containing details about the frame transfer.
"""
pass

114
src/pipecat/observers/loggers/debug_log_observer.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Debug logging observer for frame activity monitoring.
This module provides a debug observer that logs detailed frame activity
to the console, making it useful for debugging pipeline behavior and
understanding frame flow between processors.
"""
from dataclasses import fields, is_dataclass
from enum import Enum, auto
from typing import Dict, Optional, Set, Tuple, Type, Union
@@ -16,7 +23,12 @@ from pipecat.processors.frame_processor import FrameDirection
class FrameEndpoint(Enum):
"""Specifies which endpoint (source or destination) to filter on."""
"""Specifies which endpoint (source or destination) to filter on.
Parameters:
SOURCE: Filter on the source component that is pushing the frame.
DESTINATION: Filter on the destination component receiving the frame.
"""
SOURCE = auto()
DESTINATION = auto()
@@ -28,44 +40,37 @@ class DebugLogObserver(BaseObserver):
Automatically extracts and formats data from any frame type, making it useful
for debugging pipeline behavior without needing frame-specific observers.
Args:
frame_types: Optional tuple of frame types to log, or a dict with frame type
filters. If None, logs all frame types.
exclude_fields: Optional set of field names to exclude from logging.
Examples:
Log all frames from all services:
```python
observers = DebugLogObserver()
```
Log all frames from all services::
Log specific frame types from any source/destination:
```python
from pipecat.frames.frames import TranscriptionFrame, InterimTranscriptionFrame
observers=[
DebugLogObserver(frame_types=(LLMTextFrame,TranscriptionFrame,)),
],
```
observers = DebugLogObserver()
Log frames with specific source/destination filters:
```python
from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
from pipecat.transports.base_output_transport import BaseOutputTransport
from pipecat.services.stt_service import STTService
Log specific frame types from any source/destination::
observers=[
DebugLogObserver(
frame_types={
# Only log StartInterruptionFrame when source is BaseOutputTransport
StartInterruptionFrame: (BaseOutputTransport, FrameEndpoint.SOURCE),
# Only log UserStartedSpeakingFrame when destination is STTService
UserStartedSpeakingFrame: (STTService, FrameEndpoint.DESTINATION),
# Log LLMTextFrame regardless of source or destination type
LLMTextFrame: None,
}
),
],
```
from pipecat.frames.frames import LLMTextFrame, TranscriptionFrame
observers=[
DebugLogObserver(frame_types=(LLMTextFrame,TranscriptionFrame,)),
]
Log frames with specific source/destination filters::
from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.services.stt_service import STTService
observers=[
DebugLogObserver(
frame_types={
# Only log StartInterruptionFrame when source is BaseOutputTransport
StartInterruptionFrame: (BaseOutputTransport, FrameEndpoint.SOURCE),
# Only log UserStartedSpeakingFrame when destination is STTService
UserStartedSpeakingFrame: (STTService, FrameEndpoint.DESTINATION),
# Log LLMTextFrame regardless of source or destination type
LLMTextFrame: None,
}
),
]
"""
def __init__(
@@ -79,14 +84,17 @@ class DebugLogObserver(BaseObserver):
"""Initialize the debug log observer.
Args:
frame_types: Tuple of frame types to log, or a dict mapping frame types to
filter configurations. Filter configs can be:
- None to log all instances of the frame type
- A tuple of (service_type, endpoint) to filter on a specific service
and endpoint (SOURCE or DESTINATION)
If None is provided instead of a tuple/dict, log all frames.
exclude_fields: Set of field names to exclude from logging. If None, only binary
data fields are excluded.
frame_types: Frame types to log. Can be:
- Tuple of frame types to log all instances
- Dict mapping frame types to filter configurations
- None to log all frames
Filter configurations can be None (log all instances) or a tuple
of (service_type, endpoint) to filter on specific services.
exclude_fields: Field names to exclude from logging. Defaults to
excluding binary data fields like 'audio', 'image', 'images'.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
@@ -113,14 +121,7 @@ class DebugLogObserver(BaseObserver):
)
def _format_value(self, value):
"""Format a value for logging.
Args:
value: The value to format.
Returns:
str: A string representation of the value suitable for logging.
"""
"""Format a value for logging."""
if value is None:
return "None"
elif isinstance(value, str):
@@ -143,16 +144,7 @@ class DebugLogObserver(BaseObserver):
return str(value)
def _should_log_frame(self, frame, src, dst):
"""Determine if a frame should be logged based on filters.
Args:
frame: The frame being processed
src: The source component
dst: The destination component
Returns:
bool: True if the frame should be logged, False otherwise
"""
"""Determine if a frame should be logged based on filters."""
# If no filters, log all frames
if not self.frame_filters:
return True

9
src/pipecat/observers/loggers/llm_log_observer.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""LLM logging observer for Pipecat."""
from loguru import logger
from pipecat.frames.frames import (
@@ -34,10 +36,15 @@ class LLMLogObserver(BaseObserver):
This allows you to track when the LLM starts responding, what it generates,
and when it finishes.
"""
async def on_push_frame(self, data: FramePushed):
"""Handle frame push events and log LLM-related activities.
Args:
data: The frame push event data containing source, destination,
frame, direction, and timestamp information.
"""
src = data.source
dst = data.destination
frame = data.frame

26
src/pipecat/observers/loggers/transcription_log_observer.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Transcription logging observer for Pipecat.
This module provides an observer that logs transcription frames to the console,
allowing developers to monitor speech-to-text activity in real-time.
"""
from loguru import logger
from pipecat.frames.frames import (
@@ -17,17 +23,23 @@ from pipecat.services.stt_service import STTService
class TranscriptionLogObserver(BaseObserver):
"""Observer to log transcription activity to the console.
Logs all frame instances (only from STT service) of:
- TranscriptionFrame
- InterimTranscriptionFrame
This allows you to track when the LLM starts responding, what it generates,
and when it finishes.
Monitors and logs all transcription frames from STT services, including
both final transcriptions and interim results. This allows developers
to track speech recognition activity and debug transcription issues.
Only processes frames from STTService instances to avoid logging
unrelated transcription frames from other sources.
"""
async def on_push_frame(self, data: FramePushed):
"""Handle frame push events and log transcription frames.
Logs TranscriptionFrame and InterimTranscriptionFrame instances
with timestamps and user information for debugging purposes.
Args:
data: Frame push event data containing source, frame, and timestamp.
"""
src = data.source
frame = data.frame
timestamp = data.timestamp

19
src/pipecat/observers/loggers/user_bot_latency_log_observer.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Observer for measuring user-to-bot response latency."""
import time
from loguru import logger
@@ -18,19 +20,28 @@ from pipecat.processors.frame_processor import FrameDirection
class UserBotLatencyLogObserver(BaseObserver):
"""Observer that logs the latency between when the user stops speaking and
when the bot starts speaking.
This helps measure how quickly the AI services respond.
"""Observer that measures time between user stopping speech and bot starting speech.
This helps measure how quickly the AI services respond by tracking
conversation turn timing and logging latency metrics.
"""
def __init__(self):
"""Initialize the latency observer.
Sets up tracking for processed frames and user speech timing
to calculate response latencies.
"""
super().__init__()
self._processed_frames = set()
self._user_stopped_time = 0
async def on_push_frame(self, data: FramePushed):
"""Process frames to track speech timing and calculate latency.
Args:
data: Frame push event containing the frame and direction information.
"""
# Only process downstream frames
if data.direction != FrameDirection.DOWNSTREAM:
return

39
src/pipecat/observers/turn_tracking_observer.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Turn tracking observer for conversation flow monitoring.
This module provides an observer that monitors conversation turns in a pipeline,
tracking when turns start and end based on user and bot speech patterns.
"""
import asyncio
from collections import deque
@@ -12,6 +18,8 @@ from loguru import logger
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
CancelFrame,
EndFrame,
StartFrame,
UserStartedSpeakingFrame,
)
@@ -21,15 +29,30 @@ from pipecat.observers.base_observer import BaseObserver, FramePushed
class TurnTrackingObserver(BaseObserver):
"""Observer that tracks conversation turns in a pipeline.
This observer monitors the flow of conversation by tracking when turns
start and end based on user and bot speaking patterns. It handles
interruptions, timeouts, and maintains turn state throughout the pipeline.
Turn tracking logic:
- The first turn starts immediately when the pipeline starts (StartFrame)
- Subsequent turns start when the user starts speaking
- A turn ends when the bot stops speaking and either:
- The user starts speaking again
- A timeout period elapses with no more bot speech
"""
def __init__(self, max_frames=100, turn_end_timeout_secs=2.5, **kwargs):
"""Initialize the turn tracking observer.
Args:
max_frames: Maximum number of frame IDs to keep in history for
duplicate detection. Defaults to 100.
turn_end_timeout_secs: Timeout in seconds after bot stops speaking
before automatically ending the turn. Defaults to 2.5.
**kwargs: Additional arguments passed to the parent observer.
"""
super().__init__(**kwargs)
self._turn_count = 0
self._is_turn_active = False
@@ -47,7 +70,11 @@ class TurnTrackingObserver(BaseObserver):
self._register_event_handler("on_turn_ended")
async def on_push_frame(self, data: FramePushed):
"""Process frame events for turn tracking."""
"""Process frame events for turn tracking.
Args:
data: Frame push event data containing the frame and metadata.
"""
# Skip already processed frames
if data.frame.id in self._processed_frames:
return
@@ -73,6 +100,8 @@ class TurnTrackingObserver(BaseObserver):
# We only want to end the turn if the bot was previously speaking
elif isinstance(data.frame, BotStoppedSpeakingFrame) and self._is_bot_speaking:
await self._handle_bot_stopped_speaking(data)
elif isinstance(data.frame, (EndFrame, CancelFrame)):
await self._handle_pipeline_end(data)
def _schedule_turn_end(self, data: FramePushed):
"""Schedule turn end with a timeout."""
@@ -134,6 +163,14 @@ class TurnTrackingObserver(BaseObserver):
# This can happen with HTTP TTS services or function calls
self._schedule_turn_end(data)
async def _handle_pipeline_end(self, data: FramePushed):
"""Handle pipeline end or cancellation by flushing any active turn."""
if self._is_turn_active:
# Cancel any pending turn end timer
self._cancel_turn_end_timer()
# End the current turn
await self._end_turn(data, was_interrupted=True)
async def _start_turn(self, data: FramePushed):
"""Start a new turn."""
self._is_turn_active = True

17
src/pipecat/pipeline/base_pipeline.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base pipeline implementation for frame processing."""
from abc import abstractmethod
from typing import List
@@ -11,9 +13,24 @@ from pipecat.processors.frame_processor import FrameProcessor
class BasePipeline(FrameProcessor):
"""Base class for all pipeline implementations.
Provides the foundation for pipeline processors that need to support
metrics collection from their contained processors.
"""
def __init__(self):
"""Initialize the base pipeline."""
super().__init__()
@abstractmethod
def processors_with_metrics(self) -> List[FrameProcessor]:
"""Return processors that can generate metrics.
Implementing classes should collect and return all processors within
their pipeline that support metrics generation.
Returns:
List of frame processors that support metrics collection.
"""
pass

74
src/pipecat/pipeline/base_task.py
View File

@@ -4,52 +4,98 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base pipeline task implementation for managing pipeline execution.
This module provides the abstract base class and configuration for pipeline
tasks that manage the lifecycle and execution of frame processing pipelines.
"""
import asyncio
from abc import abstractmethod
from dataclasses import dataclass
from typing import AsyncIterable, Iterable
from pipecat.frames.frames import Frame
from pipecat.utils.base_object import BaseObject
class BaseTask(BaseObject):
@abstractmethod
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
"""Sets the event loop that this task will run on."""
pass
@dataclass
class PipelineTaskParams:
"""Configuration parameters for pipeline task execution.
Parameters:
loop: The asyncio event loop to use for task execution.
"""
loop: asyncio.AbstractEventLoop
class BasePipelineTask(BaseObject):
"""Abstract base class for pipeline task implementations.
Defines the interface for managing pipeline execution lifecycle,
including starting, stopping, and frame queuing operations.
"""
@abstractmethod
def has_finished(self) -> bool:
"""Indicates whether the tasks has finished. That is, all processors
have stopped.
"""Check if the pipeline task has finished execution.
Returns:
True if all processors have stopped and the task is complete.
"""
pass
@abstractmethod
async def stop_when_done(self):
"""This is a helper function that sends an EndFrame to the pipeline in
order to stop the task after everything in it has been processed.
"""Schedule the pipeline to stop after processing all queued frames.
Implementing classes should send an EndFrame or equivalent signal to
gracefully terminate the pipeline once all current processing is complete.
"""
pass
@abstractmethod
async def cancel(self):
"""Stops the running pipeline immediately."""
"""Immediately stop the running pipeline.
Implementing classes should cancel all running tasks and stop frame
processing without waiting for completion.
"""
pass
@abstractmethod
async def run(self):
"""Starts running the given pipeline."""
async def run(self, params: PipelineTaskParams):
"""Start and run the pipeline with the given parameters.
Implementing classes should initialize and execute the pipeline using
the provided configuration parameters.
Args:
params: Configuration parameters for pipeline execution.
"""
pass
@abstractmethod
async def queue_frame(self, frame: Frame):
"""Queue a frame to be pushed down the pipeline."""
"""Queue a single frame for processing by the pipeline.
Implementing classes should add the frame to their processing queue
for downstream handling.
Args:
frame: The frame to be processed.
"""
pass
@abstractmethod
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
"""Queues multiple frames to be pushed down the pipeline."""
"""Queue multiple frames for processing by the pipeline.
Implementing classes should process the iterable/async iterable and
add all frames to their processing queue.
Args:
frames: An iterable or async iterable of frames to be processed.
"""
pass

137
src/pipecat/pipeline/parallel_pipeline.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Parallel pipeline implementation for concurrent frame processing.
This module provides a parallel pipeline that processes frames through multiple
sub-pipelines concurrently, with coordination for system frames and proper
handling of pipeline lifecycle events.
"""
import asyncio
from itertools import chain
from typing import Awaitable, Callable, Dict, List
@@ -21,19 +28,38 @@ from pipecat.frames.frames import (
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
class ParallelPipelineSource(FrameProcessor):
"""Source processor for parallel pipeline branches.
Handles frame routing for parallel pipeline inputs, directing system frames
to the parent push function and other upstream frames to a queue for processing.
"""
def __init__(
self,
upstream_queue: asyncio.Queue,
push_frame_func: Callable[[Frame, FrameDirection], Awaitable[None]],
):
"""Initialize the parallel pipeline source.
Args:
upstream_queue: Queue for collecting upstream frames from this branch.
push_frame_func: Function to push frames to the parent parallel pipeline.
"""
super().__init__()
self._up_queue = upstream_queue
self._push_frame_func = push_frame_func
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with special handling for system frames.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
match direction:
@@ -47,16 +73,34 @@ class ParallelPipelineSource(FrameProcessor):
class ParallelPipelineSink(FrameProcessor):
"""Sink processor for parallel pipeline branches.
Handles frame routing for parallel pipeline outputs, directing system frames
to the parent push function and other downstream frames to a queue for coordination.
"""
def __init__(
self,
downstream_queue: asyncio.Queue,
push_frame_func: Callable[[Frame, FrameDirection], Awaitable[None]],
):
"""Initialize the parallel pipeline sink.
Args:
downstream_queue: Queue for collecting downstream frames from this branch.
push_frame_func: Function to push frames to the parent parallel pipeline.
"""
super().__init__()
self._down_queue = downstream_queue
self._push_frame_func = push_frame_func
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with special handling for system frames.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
match direction:
@@ -70,26 +114,72 @@ class ParallelPipelineSink(FrameProcessor):
class ParallelPipeline(BasePipeline):
"""Pipeline that processes frames through multiple sub-pipelines concurrently.
Creates multiple parallel processing branches from the provided processor lists,
coordinating frame flow and ensuring proper synchronization of lifecycle events
like EndFrames. Each branch runs independently while system frames are handled
specially to maintain pipeline coordination.
"""
def __init__(self, *args):
"""Initialize the parallel pipeline with processor lists.
Args:
*args: Variable number of processor lists, each becoming a parallel branch.
Raises:
Exception: If no processor lists are provided.
TypeError: If any argument is not a list of processors.
"""
super().__init__()
if len(args) == 0:
raise Exception(f"ParallelPipeline needs at least one argument")
self._args = args
self._sources = []
self._sinks = []
self._pipelines = []
self._seen_ids = set()
self._endframe_counter: Dict[int, int] = {}
self._up_task = None
self._down_task = None
self._up_queue = asyncio.Queue()
self._down_queue = asyncio.Queue()
self._pipelines = []
#
# BasePipeline
#
def processors_with_metrics(self) -> List[FrameProcessor]:
"""Collect processors that can generate metrics from all parallel branches.
Returns:
List of frame processors that support metrics collection from all branches.
"""
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
#
# Frame processor
#
async def setup(self, setup: FrameProcessorSetup):
"""Set up the parallel pipeline and all its branches.
Args:
setup: Configuration for frame processor setup.
Raises:
TypeError: If any processor list argument is not actually a list.
"""
await super().setup(setup)
self._up_queue = WatchdogQueue(setup.task_manager)
self._down_queue = WatchdogQueue(setup.task_manager)
logger.debug(f"Creating {self} pipelines")
for processors in args:
for processors in self._args:
if not isinstance(processors, list):
raise TypeError(f"ParallelPipeline argument {processors} is not a list")
@@ -107,34 +197,28 @@ class ParallelPipeline(BasePipeline):
logger.debug(f"Finished creating {self} pipelines")
#
# BasePipeline
#
def processors_with_metrics(self) -> List[FrameProcessor]:
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
#
# Frame processor
#
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await asyncio.gather(*[s.setup(setup) for s in self._sources])
await asyncio.gather(*[p.setup(setup) for p in self._pipelines])
await asyncio.gather(*[s.setup(setup) for s in self._sinks])
async def cleanup(self):
"""Clean up the parallel pipeline and all its branches."""
await super().cleanup()
await asyncio.gather(*[s.cleanup() for s in self._sources])
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
await asyncio.gather(*[s.cleanup() for s in self._sinks])
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames through all parallel branches with lifecycle coordination.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
await self._start()
await self._start(frame)
elif isinstance(frame, EndFrame):
self._endframe_counter[frame.id] = len(self._pipelines)
elif isinstance(frame, CancelFrame):
@@ -154,10 +238,12 @@ class ParallelPipeline(BasePipeline):
elif isinstance(frame, EndFrame):
await self._stop()
async def _start(self):
async def _start(self, frame: StartFrame):
"""Start the parallel pipeline processing tasks."""
await self._create_tasks()
async def _stop(self):
"""Stop all parallel pipeline processing tasks."""
if self._up_task:
# The up task doesn't receive an EndFrame, so we just cancel it.
await self.cancel_task(self._up_task)
@@ -170,42 +256,55 @@ class ParallelPipeline(BasePipeline):
self._down_task = None
async def _cancel(self):
"""Cancel all parallel pipeline processing tasks."""
if self._up_task:
self._up_queue.cancel()
await self.cancel_task(self._up_task)
self._up_task = None
if self._down_task:
self._down_queue.cancel()
await self.cancel_task(self._down_task)
self._down_task = None
async def _create_tasks(self):
"""Create upstream and downstream processing tasks if not already running."""
if not self._up_task:
self._up_task = self.create_task(self._process_up_queue())
if not self._down_task:
self._down_task = self.create_task(self._process_down_queue())
async def _drain_queues(self):
"""Drain all frames from upstream and downstream queues."""
while not self._up_queue.empty:
await self._up_queue.get()
while not self._down_queue.empty:
await self._down_queue.get()
async def _handle_interruption(self):
"""Handle interruption by cancelling tasks, draining queues, and restarting."""
await self._cancel()
await self._drain_queues()
await self._create_tasks()
async def _parallel_push_frame(self, frame: Frame, direction: FrameDirection):
"""Push frames while avoiding duplicates using frame ID tracking."""
if frame.id not in self._seen_ids:
self._seen_ids.add(frame.id)
await self.push_frame(frame, direction)
async def _process_up_queue(self):
"""Process upstream frames from all parallel branches."""
while True:
frame = await self._up_queue.get()
await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
self._up_queue.task_done()
async def _process_down_queue(self):
"""Process downstream frames with EndFrame coordination.
Coordinates EndFrames to ensure they are only pushed upstream once
all parallel branches have completed processing them.
"""
running = True
while running:
frame = await self._down_queue.get()

78
src/pipecat/pipeline/pipeline.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Pipeline implementation for connecting and managing frame processors.
This module provides the main Pipeline class that connects frame processors
in sequence and manages frame flow between them, along with helper classes
for pipeline source and sink operations.
"""
from typing import Callable, Coroutine, List
from pipecat.frames.frames import Frame
@@ -12,11 +19,29 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, F
class PipelineSource(FrameProcessor):
"""Source processor that forwards frames to an upstream handler.
This processor acts as the entry point for a pipeline, forwarding
downstream frames to the next processor and upstream frames to a
provided upstream handler function.
"""
def __init__(self, upstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
"""Initialize the pipeline source.
Args:
upstream_push_frame: Coroutine function to handle upstream frames.
"""
super().__init__()
self._upstream_push_frame = upstream_push_frame
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and route them based on direction.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
match direction:
@@ -27,11 +52,29 @@ class PipelineSource(FrameProcessor):
class PipelineSink(FrameProcessor):
"""Sink processor that forwards frames to a downstream handler.
This processor acts as the exit point for a pipeline, forwarding
upstream frames to the previous processor and downstream frames to a
provided downstream handler function.
"""
def __init__(self, downstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
"""Initialize the pipeline sink.
Args:
downstream_push_frame: Coroutine function to handle downstream frames.
"""
super().__init__()
self._downstream_push_frame = downstream_push_frame
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and route them based on direction.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
match direction:
@@ -42,7 +85,19 @@ class PipelineSink(FrameProcessor):
class Pipeline(BasePipeline):
"""Main pipeline implementation that connects frame processors in sequence.
Creates a linear chain of frame processors with automatic source and sink
processors for external frame handling. Manages processor lifecycle and
provides metrics collection from contained processors.
"""
def __init__(self, processors: List[FrameProcessor]):
"""Initialize the pipeline with a list of processors.
Args:
processors: List of frame processors to connect in sequence.
"""
super().__init__()
# Add a source and a sink queue so we can forward frames upstream and
@@ -58,6 +113,14 @@ class Pipeline(BasePipeline):
#
def processors_with_metrics(self):
"""Return processors that can generate metrics.
Recursively collects all processors that support metrics generation,
including those from nested pipelines.
Returns:
List of frame processors that can generate metrics.
"""
services = []
for p in self._processors:
if isinstance(p, BasePipeline):
@@ -71,14 +134,26 @@ class Pipeline(BasePipeline):
#
async def setup(self, setup: FrameProcessorSetup):
"""Set up the pipeline and all contained processors.
Args:
setup: Configuration for frame processor setup.
"""
await super().setup(setup)
await self._setup_processors(setup)
async def cleanup(self):
"""Clean up the pipeline and all contained processors."""
await super().cleanup()
await self._cleanup_processors()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames by routing them through the pipeline.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
if direction == FrameDirection.DOWNSTREAM:
@@ -87,14 +162,17 @@ class Pipeline(BasePipeline):
await self._sink.queue_frame(frame, FrameDirection.UPSTREAM)
async def _setup_processors(self, setup: FrameProcessorSetup):
"""Set up all processors in the pipeline."""
for p in self._processors:
await p.setup(setup)
async def _cleanup_processors(self):
"""Clean up all processors in the pipeline."""
for p in self._processors:
await p.cleanup()
def _link_processors(self):
"""Link all processors in sequence and set their parent."""
prev = self._processors[0]
for curr in self._processors[1:]:
prev.set_parent(self)

38
src/pipecat/pipeline/runner.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Pipeline runner for managing pipeline task execution.
This module provides the PipelineRunner class that handles the execution
of pipeline tasks with signal handling, garbage collection, and lifecycle
management.
"""
import asyncio
import gc
import signal
@@ -11,11 +18,19 @@ from typing import Optional
from loguru import logger
from pipecat.pipeline.base_task import PipelineTaskParams
from pipecat.pipeline.task import PipelineTask
from pipecat.utils.base_object import BaseObject
class PipelineRunner(BaseObject):
"""Manages the execution of pipeline tasks with lifecycle and signal handling.
Provides a high-level interface for running pipeline tasks with automatic
signal handling (SIGINT/SIGTERM), optional garbage collection, and proper
cleanup of resources.
"""
def __init__(
self,
*,
@@ -24,6 +39,14 @@ class PipelineRunner(BaseObject):
force_gc: bool = False,
loop: Optional[asyncio.AbstractEventLoop] = None,
):
"""Initialize the pipeline runner.
Args:
name: Optional name for the runner instance.
handle_sigint: Whether to automatically handle SIGINT/SIGTERM signals.
force_gc: Whether to force garbage collection after task completion.
loop: Event loop to use. If None, uses the current running loop.
"""
super().__init__(name=name)
self._tasks = {}
@@ -35,10 +58,15 @@ class PipelineRunner(BaseObject):
self._setup_sigint()
async def run(self, task: PipelineTask):
"""Run a pipeline task to completion.
Args:
task: The pipeline task to execute.
"""
logger.debug(f"Runner {self} started running {task}")
self._tasks[task.name] = task
task.set_event_loop(self._loop)
await task.run()
params = PipelineTaskParams(loop=self._loop)
await task.run(params)
del self._tasks[task.name]
# Cleanup base object.
@@ -55,27 +83,33 @@ class PipelineRunner(BaseObject):
logger.debug(f"Runner {self} finished running {task}")
async def stop_when_done(self):
"""Schedule all running tasks to stop when their current processing is complete."""
logger.debug(f"Runner {self} scheduled to stop when all tasks are done")
await asyncio.gather(*[t.stop_when_done() for t in self._tasks.values()])
async def cancel(self):
"""Cancel all running tasks immediately."""
logger.debug(f"Cancelling runner {self}")
await asyncio.gather(*[t.cancel() for t in self._tasks.values()])
def _setup_sigint(self):
"""Set up signal handlers for graceful shutdown."""
loop = asyncio.get_running_loop()
loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler())
loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler())
def _sig_handler(self):
"""Handle interrupt signals by cancelling all tasks."""
if not self._sig_task:
self._sig_task = asyncio.create_task(self._sig_cancel())
async def _sig_cancel(self):
"""Cancel all running tasks due to signal interruption."""
logger.warning(f"Interruption detected. Cancelling runner {self}")
await self.cancel()
def _gc_collect(self):
"""Force garbage collection and log results."""
collected = gc.collect()
logger.debug(f"Garbage collector: collected {collected} objects.")
logger.debug(f"Garbage collector: uncollectable objects {gc.garbage}")

124
src/pipecat/pipeline/sync_parallel_pipeline.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Synchronous parallel pipeline implementation for concurrent frame processing.
This module provides a pipeline that processes frames through multiple parallel
pipelines simultaneously, synchronizing their output to maintain frame ordering
and prevent duplicate processing.
"""
import asyncio
from dataclasses import dataclass
from itertools import chain
@@ -15,21 +22,43 @@ from pipecat.frames.frames import ControlFrame, EndFrame, Frame, SystemFrame
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
@dataclass
class SyncFrame(ControlFrame):
"""This frame is used to know when the internal pipelines have finished."""
"""Control frame used to synchronize parallel pipeline processing.
This frame is sent through parallel pipelines to determine when the
internal pipelines have finished processing a batch of frames.
"""
pass
class SyncParallelPipelineSource(FrameProcessor):
"""Source processor for synchronous parallel pipeline processing.
Routes frames to parallel pipelines and collects upstream responses
for synchronization purposes.
"""
def __init__(self, upstream_queue: asyncio.Queue):
"""Initialize the sync parallel pipeline source.
Args:
upstream_queue: Queue for collecting upstream frames from the pipeline.
"""
super().__init__()
self._up_queue = upstream_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and route them based on direction.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
match direction:
@@ -40,11 +69,28 @@ class SyncParallelPipelineSource(FrameProcessor):
class SyncParallelPipelineSink(FrameProcessor):
"""Sink processor for synchronous parallel pipeline processing.
Collects downstream frames from parallel pipelines and routes
upstream frames back through the pipeline.
"""
def __init__(self, downstream_queue: asyncio.Queue):
"""Initialize the sync parallel pipeline sink.
Args:
downstream_queue: Queue for collecting downstream frames from the pipeline.
"""
super().__init__()
self._down_queue = downstream_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and route them based on direction.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
match direction:
@@ -55,21 +101,67 @@ class SyncParallelPipelineSink(FrameProcessor):
class SyncParallelPipeline(BasePipeline):
"""Pipeline that processes frames through multiple parallel pipelines synchronously.
Creates multiple parallel processing paths that all receive the same input frames
and produces synchronized output. Each parallel path is a separate pipeline that
processes frames independently, with synchronization points to ensure consistent
ordering and prevent duplicate frame processing.
The pipeline uses SyncFrame control frames to coordinate between parallel paths
and ensure all paths have completed processing before moving to the next frame.
"""
def __init__(self, *args):
"""Initialize the synchronous parallel pipeline.
Args:
*args: Variable number of processor lists, each representing a parallel pipeline path.
Each argument should be a list of FrameProcessor instances.
Raises:
Exception: If no arguments are provided.
TypeError: If any argument is not a list of processors.
"""
super().__init__()
if len(args) == 0:
raise Exception(f"SyncParallelPipeline needs at least one argument")
self._args = args
self._sinks = []
self._sources = []
self._pipelines = []
self._up_queue = asyncio.Queue()
self._down_queue = asyncio.Queue()
#
# BasePipeline
#
def processors_with_metrics(self) -> List[FrameProcessor]:
"""Collect processors that can generate metrics from all parallel pipelines.
Returns:
List of frame processors that support metrics collection from all parallel paths.
"""
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
#
# Frame processor
#
async def setup(self, setup: FrameProcessorSetup):
"""Set up the parallel pipeline and all contained processors.
Args:
setup: Configuration for frame processor setup.
"""
await super().setup(setup)
self._up_queue = WatchdogQueue(setup.task_manager)
self._down_queue = WatchdogQueue(setup.task_manager)
logger.debug(f"Creating {self} pipelines")
for processors in args:
for processors in self._args:
if not isinstance(processors, list):
raise TypeError(f"SyncParallelPipeline argument {processors} is not a list")
@@ -92,30 +184,28 @@ class SyncParallelPipeline(BasePipeline):
logger.debug(f"Finished creating {self} pipelines")
#
# BasePipeline
#
def processors_with_metrics(self) -> List[FrameProcessor]:
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
#
# Frame processor
#
async def setup(self, setup: FrameProcessorSetup):
await super().setup(setup)
await asyncio.gather(*[s["processor"].setup(setup) for s in self._sources])
await asyncio.gather(*[p.setup(setup) for p in self._pipelines])
await asyncio.gather(*[s["processor"].setup(setup) for s in self._sinks])
async def cleanup(self):
"""Clean up the parallel pipeline and all contained processors."""
await super().cleanup()
await asyncio.gather(*[s["processor"].cleanup() for s in self._sources])
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
await asyncio.gather(*[s["processor"].cleanup() for s in self._sinks])
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames through all parallel pipelines with synchronization.
Distributes frames to all parallel pipelines and synchronizes their output
to maintain proper ordering and prevent duplicate processing. Uses SyncFrame
control frames to coordinate between parallel paths.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
# The last processor of each pipeline needs to be synchronous otherwise

418
src/pipecat/pipeline/task.py
View File

@@ -4,9 +4,17 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Pipeline task implementation for managing frame processing pipelines.
This module provides the main PipelineTask class that orchestrates pipeline
execution, frame routing, lifecycle management, and monitoring capabilities
including heartbeats, idle detection, and observer integration.
"""
import asyncio
import time
from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Sequence, Tuple, Type
from collections import deque
from typing import Any, AsyncIterable, Deque, Dict, Iterable, List, Optional, Tuple, Type
from loguru import logger
from pydantic import BaseModel, ConfigDict, Field
@@ -23,6 +31,7 @@ from pipecat.frames.frames import (
ErrorFrame,
Frame,
HeartbeatFrame,
InputAudioRawFrame,
LLMFullResponseEndFrame,
MetricsFrame,
StartFrame,
@@ -33,21 +42,31 @@ from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
from pipecat.observers.base_observer import BaseObserver
from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.base_task import BaseTask
from pipecat.pipeline.base_task import BasePipelineTask, PipelineTaskParams
from pipecat.pipeline.task_observer import TaskObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
from pipecat.utils.asyncio import BaseTaskManager, TaskManager
from pipecat.utils.asyncio.task_manager import (
WATCHDOG_TIMEOUT,
BaseTaskManager,
TaskManager,
TaskManagerParams,
)
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
from pipecat.utils.tracing.setup import is_tracing_available
from pipecat.utils.tracing.turn_trace_observer import TurnTraceObserver
HEARTBEAT_SECONDS = 1.0
HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5
HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 10
class PipelineParams(BaseModel):
"""Configuration parameters for pipeline execution.
Attributes:
These parameters are usually passed to all frame processors through
StartFrame. For other generic pipeline task parameters use PipelineTask
constructor arguments instead.
Parameters:
allow_interruptions: Whether to allow pipeline interruptions.
audio_in_sample_rate: Input audio sample rate in Hz.
audio_out_sample_rate: Output audio sample rate in Hz.
@@ -55,27 +74,31 @@ class PipelineParams(BaseModel):
enable_metrics: Whether to enable metrics collection.
enable_usage_metrics: Whether to enable usage metrics.
heartbeats_period_secs: Period between heartbeats in seconds.
interruption_strategies: Strategies for bot interruption behavior.
observers: [deprecated] Use `observers` arg in `PipelineTask` class.
.. deprecated:: 0.0.58
Use the `observers` argument in the `PipelineTask` class instead.
report_only_initial_ttfb: Whether to report only initial time to first byte.
send_initial_empty_metrics: Whether to send initial empty metrics.
start_metadata: Additional metadata for pipeline start.
interruption_strategies: Strategies for bot interruption behavior.
"""
model_config = ConfigDict(arbitrary_types_allowed=True)
allow_interruptions: bool = False
allow_interruptions: bool = True
audio_in_sample_rate: int = 16000
audio_out_sample_rate: int = 24000
enable_heartbeats: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
heartbeats_period_secs: float = HEARTBEAT_SECONDS
interruption_strategies: List[BaseInterruptionStrategy] = Field(default_factory=list)
observers: List[BaseObserver] = Field(default_factory=list)
report_only_initial_ttfb: bool = False
send_initial_empty_metrics: bool = True
start_metadata: Dict[str, Any] = Field(default_factory=dict)
interruption_strategies: List[BaseInterruptionStrategy] = Field(default_factory=list)
class PipelineTaskSource(FrameProcessor):
@@ -85,17 +108,25 @@ class PipelineTaskSource(FrameProcessor):
pipeline given to the pipeline task. It allows us to easily push frames
downstream to the pipeline and also receive upstream frames coming from the
pipeline.
Args:
up_queue: Queue for upstream frame processing.
"""
def __init__(self, up_queue: asyncio.Queue, **kwargs):
"""Initialize the pipeline task source.
Args:
up_queue: Queue for upstream frame processing.
**kwargs: Additional arguments passed to the parent class.
"""
super().__init__(**kwargs)
self._up_queue = up_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and route them based on direction.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
match direction:
@@ -111,82 +142,56 @@ class PipelineTaskSink(FrameProcessor):
This is the sink processor that is linked at the end of the pipeline
given to the pipeline task. It allows us to receive downstream frames and
act on them, for example, waiting to receive an EndFrame.
Args:
down_queue: Queue for downstream frame processing.
"""
def __init__(self, down_queue: asyncio.Queue, **kwargs):
"""Initialize the pipeline task sink.
Args:
down_queue: Queue for downstream frame processing.
**kwargs: Additional arguments passed to the parent class.
"""
super().__init__(**kwargs)
self._down_queue = down_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and route them to the downstream queue.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction)
await self._down_queue.put(frame)
class PipelineTask(BaseTask):
class PipelineTask(BasePipelineTask):
"""Manages the execution of a pipeline, handling frame processing and task lifecycle.
It has a couple of event handlers `on_frame_reached_upstream` and
`on_frame_reached_downstream` that are called when upstream frames or
downstream frames reach both ends of pipeline. By default, the events
handlers will not be called unless some filters are set using
`set_reached_upstream_filter` and `set_reached_downstream_filter`.
This class orchestrates pipeline execution with comprehensive monitoring,
event handling, and lifecycle management. It provides event handlers for
various pipeline states and frame types, idle detection, heartbeat monitoring,
and observer integration.
@task.event_handler("on_frame_reached_upstream")
async def on_frame_reached_upstream(task, frame):
...
Event handlers available:
@task.event_handler("on_frame_reached_downstream")
async def on_frame_reached_downstream(task, frame):
...
- on_frame_reached_upstream: Called when upstream frames reach the source
- on_frame_reached_downstream: Called when downstream frames reach the sink
- on_idle_timeout: Called when pipeline is idle beyond timeout threshold
- on_pipeline_started: Called when pipeline starts with StartFrame
- on_pipeline_stopped: Called when pipeline stops with StopFrame
- on_pipeline_ended: Called when pipeline ends with EndFrame
- on_pipeline_cancelled: Called when pipeline is cancelled
It also has an event handler that detects when the pipeline is idle. By
default, a pipeline is idle if no `BotSpeakingFrame` or
`LLMFullResponseEndFrame` are received within `idle_timeout_secs`.
Example::
@task.event_handler("on_idle_timeout")
async def on_pipeline_idle_timeout(task):
...
@task.event_handler("on_frame_reached_upstream")
async def on_frame_reached_upstream(task, frame):
...
There are also events to know if a pipeline has been started, stopped, ended
or cancelled.
@task.event_handler("on_pipeline_started")
async def on_pipeline_started(task, frame: StartFrame):
...
@task.event_handler("on_pipeline_stopped")
async def on_pipeline_stopped(task, frame: StopFrame):
...
@task.event_handler("on_pipeline_ended")
async def on_pipeline_ended(task, frame: EndFrame):
...
@task.event_handler("on_pipeline_cancelled")
async def on_pipeline_cancelled(task, frame: CancelFrame):
...
Args:
pipeline: The pipeline to execute.
params: Configuration parameters for the pipeline.
observers: List of observers for monitoring pipeline execution.
clock: Clock implementation for timing operations.
check_dangling_tasks: Whether to check for processors' tasks finishing properly.
idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or
None. If a pipeline is idle the pipeline task will be cancelled
automatically.
idle_timeout_frames: A tuple with the frames that should trigger an idle
timeout if not received withing `idle_timeout_seconds`.
cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
the idle timeout is reached.
enable_turn_tracking: Whether to enable turn tracking.
enable_turn_tracing: Whether to enable turn tracing.
conversation_id: Optional custom ID for the conversation.
additional_span_attributes: Optional dictionary of attributes to propagate as
OpenTelemetry conversation span attributes.
@task.event_handler("on_idle_timeout")
async def on_pipeline_idle_timeout(task):
...
"""
def __init__(
@@ -194,33 +199,65 @@ class PipelineTask(BaseTask):
pipeline: BasePipeline,
*,
params: Optional[PipelineParams] = None,
observers: Optional[List[BaseObserver]] = None,
clock: Optional[BaseClock] = None,
task_manager: Optional[BaseTaskManager] = None,
additional_span_attributes: Optional[dict] = None,
cancel_on_idle_timeout: bool = True,
check_dangling_tasks: bool = True,
idle_timeout_secs: Optional[float] = 300,
clock: Optional[BaseClock] = None,
conversation_id: Optional[str] = None,
enable_tracing: bool = False,
enable_turn_tracking: bool = True,
enable_watchdog_logging: bool = False,
enable_watchdog_timers: bool = False,
idle_timeout_frames: Tuple[Type[Frame], ...] = (
BotSpeakingFrame,
LLMFullResponseEndFrame,
),
cancel_on_idle_timeout: bool = True,
enable_turn_tracking: bool = True,
enable_tracing: bool = False,
conversation_id: Optional[str] = None,
additional_span_attributes: Optional[dict] = None,
idle_timeout_secs: Optional[float] = 300,
observers: Optional[List[BaseObserver]] = None,
task_manager: Optional[BaseTaskManager] = None,
watchdog_timeout_secs: float = WATCHDOG_TIMEOUT,
):
"""Initialize the PipelineTask.
Args:
pipeline: The pipeline to execute.
params: Configuration parameters for the pipeline.
additional_span_attributes: Optional dictionary of attributes to propagate as
OpenTelemetry conversation span attributes.
cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
the idle timeout is reached.
check_dangling_tasks: Whether to check for processors' tasks finishing properly.
clock: Clock implementation for timing operations.
conversation_id: Optional custom ID for the conversation.
enable_tracing: Whether to enable tracing.
enable_turn_tracking: Whether to enable turn tracking.
enable_watchdog_logging: Whether to print task processing times.
enable_watchdog_timers: Whether to enable task watchdog timers.
idle_timeout_frames: A tuple with the frames that should trigger an idle
timeout if not received within `idle_timeout_seconds`.
idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or
None. If a pipeline is idle the pipeline task will be cancelled
automatically.
observers: List of observers for monitoring pipeline execution.
task_manager: Optional task manager for handling asyncio tasks.
watchdog_timeout_secs: Watchdog timer timeout (in seconds). A warning
will be logged if the watchdog timer is not reset before this timeout.
"""
super().__init__()
self._pipeline = pipeline
self._clock = clock or SystemClock()
self._params = params or PipelineParams()
self._check_dangling_tasks = check_dangling_tasks
self._idle_timeout_secs = idle_timeout_secs
self._idle_timeout_frames = idle_timeout_frames
self._cancel_on_idle_timeout = cancel_on_idle_timeout
self._enable_turn_tracking = enable_turn_tracking
self._enable_tracing = enable_tracing and is_tracing_available()
self._conversation_id = conversation_id
self._additional_span_attributes = additional_span_attributes or {}
self._cancel_on_idle_timeout = cancel_on_idle_timeout
self._check_dangling_tasks = check_dangling_tasks
self._clock = clock or SystemClock()
self._conversation_id = conversation_id
self._enable_tracing = enable_tracing and is_tracing_available()
self._enable_turn_tracking = enable_turn_tracking
self._enable_watchdog_logging = enable_watchdog_logging
self._enable_watchdog_timers = enable_watchdog_timers
self._idle_timeout_frames = idle_timeout_frames
self._idle_timeout_secs = idle_timeout_secs
self._watchdog_timeout_secs = watchdog_timeout_secs
if self._params.observers:
import warnings
@@ -247,19 +284,29 @@ class PipelineTask(BaseTask):
self._finished = False
self._cancelled = False
# This task maneger will handle all the asyncio tasks created by this
# PipelineTask and its frame processors.
self._task_manager = task_manager or TaskManager()
# This queue receives frames coming from the pipeline upstream.
self._up_queue = asyncio.Queue()
self._up_queue = WatchdogQueue(self._task_manager)
self._process_up_task: Optional[asyncio.Task] = None
# This queue receives frames coming from the pipeline downstream.
self._down_queue = asyncio.Queue()
self._down_queue = WatchdogQueue(self._task_manager)
self._process_down_task: Optional[asyncio.Task] = None
# This queue is the queue used to push frames to the pipeline.
self._push_queue = asyncio.Queue()
self._push_queue = WatchdogQueue(self._task_manager)
self._process_push_task: Optional[asyncio.Task] = None
# This is the heartbeat queue. When a heartbeat frame is received in the
# down queue we add it to the heartbeat queue for processing.
self._heartbeat_queue = asyncio.Queue()
self._heartbeat_queue = WatchdogQueue(self._task_manager)
self._heartbeat_push_task: Optional[asyncio.Task] = None
self._heartbeat_monitor_task: Optional[asyncio.Task] = None
# This is the idle queue. When frames are received downstream they are
# put in the queue. If no frame is received the pipeline is considered
# idle.
self._idle_queue = asyncio.Queue()
self._idle_queue = WatchdogQueue(self._task_manager)
self._idle_monitor_task: Optional[asyncio.Task] = None
# This event is used to indicate a finalize frame (e.g. EndFrame,
# StopFrame) has been received in the down queue.
self._pipeline_end_event = asyncio.Event()
@@ -276,10 +323,6 @@ class PipelineTask(BaseTask):
self._sink = PipelineTaskSink(self._down_queue)
pipeline.link(self._sink)
# This task maneger will handle all the asyncio tasks created by this
# PipelineTask and its frame processors.
self._task_manager = task_manager or TaskManager()
# The task observer acts as a proxy to the provided observers. This way,
# we only need to pass a single observer (using the StartFrame) which
# then just acts as a proxy.
@@ -303,69 +346,103 @@ class PipelineTask(BaseTask):
@property
def params(self) -> PipelineParams:
"""Returns the pipeline parameters of this task."""
"""Get the pipeline parameters for this task.
Returns:
The pipeline parameters configuration.
"""
return self._params
@property
def turn_tracking_observer(self) -> Optional[TurnTrackingObserver]:
"""Return the turn tracking observer if enabled."""
"""Get the turn tracking observer if enabled.
Returns:
The turn tracking observer instance or None if not enabled.
"""
return self._turn_tracking_observer
@property
def turn_trace_observer(self) -> Optional[TurnTraceObserver]:
"""Return the turn trace observer if enabled."""
"""Get the turn trace observer if enabled.
Returns:
The turn trace observer instance or None if not enabled.
"""
return self._turn_trace_observer
def add_observer(self, observer: BaseObserver):
"""Add an observer to monitor pipeline execution.
Args:
observer: The observer to add to the pipeline monitoring.
"""
self._observer.add_observer(observer)
async def remove_observer(self, observer: BaseObserver):
"""Remove an observer from pipeline monitoring.
Args:
observer: The observer to remove from pipeline monitoring.
"""
await self._observer.remove_observer(observer)
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
self._task_manager.set_event_loop(loop)
def set_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]):
"""Sets which frames will be checked before calling the
on_frame_reached_upstream event handler.
"""Set which frame types trigger the on_frame_reached_upstream event.
Args:
types: Tuple of frame types to monitor for upstream events.
"""
self._reached_upstream_types = types
def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
"""Sets which frames will be checked before calling the
on_frame_reached_downstream event handler.
"""Set which frame types trigger the on_frame_reached_downstream event.
Args:
types: Tuple of frame types to monitor for downstream events.
"""
self._reached_downstream_types = types
def has_finished(self) -> bool:
"""Indicates whether the tasks has finished. That is, all processors
"""Check if the pipeline task has finished execution.
This indicates whether the tasks has finished, meaninig all processors
have stopped.
Returns:
True if all processors have stopped and the task is complete.
"""
return self._finished
async def stop_when_done(self):
"""This is a helper function that sends an EndFrame to the pipeline in
order to stop the task after everything in it has been processed.
"""Schedule the pipeline to stop after processing all queued frames.
Sends an EndFrame to gracefully terminate the pipeline once all
current processing is complete.
"""
logger.debug(f"Task {self} scheduled to stop when done")
await self.queue_frame(EndFrame())
async def cancel(self):
"""Stops the running pipeline immediately."""
"""Immediately stop the running pipeline.
Cancels all running tasks and stops frame processing without
waiting for completion.
"""
await self._cancel()
async def run(self):
"""Starts and manages the pipeline execution until completion or cancellation."""
async def run(self, params: PipelineTaskParams):
"""Start and manage the pipeline execution until completion or cancellation.
Args:
params: Configuration parameters for pipeline execution.
"""
if self.has_finished():
return
cleanup_pipeline = True
try:
# Setup processors.
await self._setup()
await self._setup(params)
# Create all main tasks and wait of the main push task. This is the
# task that pushes frames to the very beginning of our pipeline (our
@@ -415,6 +492,7 @@ class PipelineTask(BaseTask):
await self.queue_frame(frame)
async def _cancel(self):
"""Internal cancellation logic for the pipeline task."""
if not self._cancelled:
logger.debug(f"Canceling pipeline task {self}")
self._cancelled = True
@@ -423,9 +501,12 @@ class PipelineTask(BaseTask):
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
# Only cancel the push task. Everything else will be cancelled in run().
await self._task_manager.cancel_task(self._process_push_task)
if self._process_push_task:
await self._task_manager.cancel_task(self._process_push_task)
self._process_push_task = None
async def _create_tasks(self):
"""Create and start all pipeline processing tasks."""
self._process_up_task = self._task_manager.create_task(
self._process_up_queue(), f"{self}::_process_up_queue"
)
@@ -441,7 +522,8 @@ class PipelineTask(BaseTask):
return self._process_push_task
def _maybe_start_heartbeat_tasks(self):
if self._params.enable_heartbeats:
"""Start heartbeat tasks if heartbeats are enabled and not already running."""
if self._params.enable_heartbeats and self._heartbeat_push_task is None:
self._heartbeat_push_task = self._task_manager.create_task(
self._heartbeat_push_handler(), f"{self}::_heartbeat_push_handler"
)
@@ -450,30 +532,49 @@ class PipelineTask(BaseTask):
)
def _maybe_start_idle_task(self):
"""Start idle monitoring task if idle timeout is configured."""
if self._idle_timeout_secs:
self._idle_monitor_task = self._task_manager.create_task(
self._idle_monitor_handler(), f"{self}::_idle_monitor_handler"
)
async def _cancel_tasks(self):
"""Cancel all running pipeline tasks."""
await self._observer.stop()
await self._task_manager.cancel_task(self._process_up_task)
await self._task_manager.cancel_task(self._process_down_task)
if self._process_up_task:
await self._task_manager.cancel_task(self._process_up_task)
self._process_up_task = None
if self._process_down_task:
await self._task_manager.cancel_task(self._process_down_task)
self._process_down_task = None
await self._maybe_cancel_heartbeat_tasks()
await self._maybe_cancel_idle_task()
async def _maybe_cancel_heartbeat_tasks(self):
if self._params.enable_heartbeats:
"""Cancel heartbeat tasks if they are running."""
if not self._params.enable_heartbeats:
return
if self._heartbeat_push_task:
await self._task_manager.cancel_task(self._heartbeat_push_task)
self._heartbeat_push_task = None
if self._heartbeat_monitor_task:
await self._task_manager.cancel_task(self._heartbeat_monitor_task)
self._heartbeat_monitor_task = None
async def _maybe_cancel_idle_task(self):
if self._idle_timeout_secs:
"""Cancel idle monitoring task if it is running."""
if self._idle_timeout_secs and self._idle_monitor_task:
self._idle_queue.cancel()
await self._task_manager.cancel_task(self._idle_monitor_task)
self._idle_monitor_task = None
def _initial_metrics_frame(self) -> MetricsFrame:
"""Create an initial metrics frame with zero values for all processors."""
processors = self._pipeline.processors_with_metrics()
data = []
for p in processors:
@@ -482,20 +583,32 @@ class PipelineTask(BaseTask):
return MetricsFrame(data=data)
async def _wait_for_pipeline_end(self):
"""Wait for the pipeline to signal completion."""
await self._pipeline_end_event.wait()
self._pipeline_end_event.clear()
async def _setup(self):
async def _setup(self, params: PipelineTaskParams):
"""Set up the pipeline task and all processors."""
mgr_params = TaskManagerParams(
loop=params.loop,
enable_watchdog_logging=self._enable_watchdog_logging,
enable_watchdog_timers=self._enable_watchdog_timers,
watchdog_timeout=self._watchdog_timeout_secs,
)
self._task_manager.setup(mgr_params)
setup = FrameProcessorSetup(
clock=self._clock,
task_manager=self._task_manager,
observer=self._observer,
watchdog_timers_enabled=self._enable_watchdog_timers,
)
await self._source.setup(setup)
await self._pipeline.setup(setup)
await self._sink.setup(setup)
async def _cleanup(self, cleanup_pipeline: bool):
"""Clean up the pipeline task and processors."""
# Cleanup base object.
await self.cleanup()
@@ -510,14 +623,14 @@ class PipelineTask(BaseTask):
await self._sink.cleanup()
async def _process_push_queue(self):
"""This is the task that runs the pipeline for the first time by sending
"""Process frames from the push queue and send them through the pipeline.
This is the task that runs the pipeline for the first time by sending
a StartFrame and by pushing any other frames queued by the user. It runs
until the tasks is cancelled or stopped (e.g. with an EndFrame).
"""
self._clock.start()
self._maybe_start_heartbeat_tasks()
self._maybe_start_idle_task()
start_frame = StartFrame(
@@ -548,11 +661,12 @@ class PipelineTask(BaseTask):
await self._cleanup(cleanup_pipeline)
async def _process_up_queue(self):
"""This is the task that processes frames coming upstream from the
"""Process frames coming upstream from the pipeline.
This is the task that processes frames coming upstream from the
pipeline. These frames might indicate, for example, that we want the
pipeline to be stopped (e.g. EndTaskFrame) in which case we would send
an EndFrame down the pipeline.
"""
while True:
frame = await self._up_queue.get()
@@ -581,11 +695,12 @@ class PipelineTask(BaseTask):
self._up_queue.task_done()
async def _process_down_queue(self):
"""This tasks process frames coming downstream from the pipeline. For
"""Process frames coming downstream from the pipeline.
This tasks process frames coming downstream from the pipeline. For
example, heartbeat frames or an EndFrame which would indicate all
processors have handled the EndFrame and therefore we can exit the task
cleanly.
"""
while True:
frame = await self._down_queue.get()
@@ -599,6 +714,10 @@ class PipelineTask(BaseTask):
if isinstance(frame, StartFrame):
await self._call_event_handler("on_pipeline_started", frame)
# Start heartbeat tasks now that StartFrame has been processed
# by all processors in the pipeline
self._maybe_start_heartbeat_tasks()
elif isinstance(frame, EndFrame):
await self._call_event_handler("on_pipeline_ended", frame)
self._pipeline_end_event.set()
@@ -612,7 +731,7 @@ class PipelineTask(BaseTask):
self._down_queue.task_done()
async def _heartbeat_push_handler(self):
"""This tasks pushes a heartbeat frame every heartbeat period."""
"""Push heartbeat frames at regular intervals."""
while True:
# Don't use `queue_frame()` because if an EndFrame is queued the
# task will just stop waiting for the pipeline to finish not
@@ -621,11 +740,12 @@ class PipelineTask(BaseTask):
await asyncio.sleep(self._params.heartbeats_period_secs)
async def _heartbeat_monitor_handler(self):
"""This tasks monitors heartbeat frames. If a heartbeat frame has not
"""Monitor heartbeat frames for processing time and timeout detection.
This task monitors heartbeat frames. If a heartbeat frame has not
been received for a long period a warning will be logged. It also logs
the time that a heartbeat frame takes to processes, that is how long it
takes for the heartbeat frame to traverse all the pipeline.
"""
wait_time = HEARTBEAT_MONITOR_SECONDS
while True:
@@ -640,18 +760,26 @@ class PipelineTask(BaseTask):
)
async def _idle_monitor_handler(self):
"""This tasks monitors activity in the pipeline. If no frames are
received (heartbeats don't count) the pipeline is considered idle.
"""Monitor pipeline activity and detect idle conditions.
Tracks frame activity and triggers idle timeout events when the
pipeline hasn't received relevant frames within the timeout period.
Note: Heartbeats are excluded from idle detection.
"""
running = True
last_frame_time = 0
frame_buffer = deque(maxlen=10) # Store last 10 frames
while running:
try:
frame = await asyncio.wait_for(
self._idle_queue.get(), timeout=self._idle_timeout_secs
)
if not isinstance(frame, InputAudioRawFrame):
frame_buffer.append(frame)
if isinstance(frame, StartFrame) or isinstance(frame, self._idle_timeout_frames):
# If we find a StartFrame or one of the frames that prevents a
# time out we update the time.
@@ -662,18 +790,31 @@ class PipelineTask(BaseTask):
# valid frames.
diff_time = time.time() - last_frame_time
if diff_time >= self._idle_timeout_secs:
running = await self._idle_timeout_detected()
running = await self._idle_timeout_detected(frame_buffer)
# Reset `last_frame_time` so we don't trigger another
# immediate idle timeout if we are not cancelling. For
# example, we might want to force the bot to say goodbye
# and then clean nicely with an `EndFrame`.
last_frame_time = time.time()
self._idle_queue.task_done()
except asyncio.TimeoutError:
running = await self._idle_timeout_detected()
async def _idle_timeout_detected(self) -> bool:
"""Logic for when the pipeline is idle.
except asyncio.TimeoutError:
running = await self._idle_timeout_detected(frame_buffer)
async def _idle_timeout_detected(self, last_frames: Deque[Frame]) -> bool:
"""Handle idle timeout detection and optional cancellation.
Args:
last_frames: Recent frames received before timeout for debugging.
Returns:
bool: Whther the pipeline task is being cancelled or not.
Whether the pipeline task should continue running.
"""
logger.warning("Idle timeout detected. Last 10 frames received:")
for i, frame in enumerate(last_frames, 1):
logger.warning(f"Frame {i}: {frame}")
await self._call_event_handler("on_idle_timeout")
if self._cancel_on_idle_timeout:
logger.warning(f"Idle pipeline detected, cancelling pipeline task...")
@@ -682,6 +823,7 @@ class PipelineTask(BaseTask):
return True
def _print_dangling_tasks(self):
"""Log any dangling tasks that haven't been properly cleaned up."""
tasks = [t.get_name() for t in self._task_manager.current_tasks()]
if tasks:
logger.warning(f"Dangling tasks detected: {tasks}")

60
src/pipecat/pipeline/task_observer.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Task observer for managing pipeline frame observers.
This module provides a proxy observer system that manages multiple observers
for pipeline frame events, ensuring that observer processing doesn't block
the main pipeline execution.
"""
import asyncio
import inspect
from typing import Dict, List, Optional
@@ -11,14 +18,21 @@ from typing import Dict, List, Optional
from attr import dataclass
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.utils.asyncio import BaseTaskManager
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
@dataclass
class Proxy:
"""This is the data we receive from the main observer and that we put into
a queue for later processing.
"""Proxy data for managing observer tasks and queues.
This represents is the data received from the main observer that
is queued for later processing.
Parameters:
queue: Queue for frame data awaiting observer processing.
task: Asyncio task running the observer's frame processing loop.
observer: The actual observer instance being proxied.
"""
queue: asyncio.Queue
@@ -27,7 +41,9 @@ class Proxy:
class TaskObserver(BaseObserver):
"""This is a pipeline frame observer that is meant to be used as a proxy to
"""Proxy observer that manages multiple observers without blocking the pipeline.
This is a pipeline frame observer that is meant to be used as a proxy to
the user provided observers. That is, this is the observer that should be
passed to the frame processors. Then, every time a frame is pushed this
observer will call all the observers registered to the pipeline task.
@@ -36,7 +52,6 @@ class TaskObserver(BaseObserver):
pipeline by creating a queue and a task for each user observer. When a frame
is received, it will be put in a queue for efficiency and later processed by
each task.
"""
def __init__(
@@ -46,6 +61,13 @@ class TaskObserver(BaseObserver):
task_manager: BaseTaskManager,
**kwargs,
):
"""Initialize the TaskObserver.
Args:
observers: List of observers to manage. Defaults to empty list.
task_manager: Task manager for creating and managing observer tasks.
**kwargs: Additional arguments passed to the base observer.
"""
super().__init__(**kwargs)
self._observers = observers or []
self._task_manager = task_manager
@@ -54,6 +76,11 @@ class TaskObserver(BaseObserver):
)
def add_observer(self, observer: BaseObserver):
"""Add a new observer to the managed list.
Args:
observer: The observer to add.
"""
# Add the observer to the list.
self._observers.append(observer)
@@ -64,6 +91,11 @@ class TaskObserver(BaseObserver):
self._proxies[observer] = proxy
async def remove_observer(self, observer: BaseObserver):
"""Remove an observer and clean up its resources.
Args:
observer: The observer to remove.
"""
# If the observer has a proxy, remove it.
if observer in self._proxies:
proxy = self._proxies[observer]
@@ -77,23 +109,33 @@ class TaskObserver(BaseObserver):
self._observers.remove(observer)
async def start(self):
"""Starts all proxy observer tasks."""
"""Start all proxy observer tasks."""
self._proxies = self._create_proxies(self._observers)
async def stop(self):
"""Stops all proxy observer tasks."""
"""Stop all proxy observer tasks."""
if not self._proxies:
return
for proxy in self._proxies.values():
await self._task_manager.cancel_task(proxy.task)
async def on_push_frame(self, data: FramePushed):
"""Queue frame data for all managed observers.
Args:
data: The frame push event data to distribute to observers.
"""
for proxy in self._proxies.values():
await proxy.queue.put(data)
def _started(self) -> bool:
"""Check if the task observer has been started."""
return self._proxies is not None
def _create_proxy(self, observer: BaseObserver) -> Proxy:
queue = asyncio.Queue()
"""Create a proxy for a single observer."""
queue = WatchdogQueue(self._task_manager)
task = self._task_manager.create_task(
self._proxy_task_handler(queue, observer),
f"TaskObserver::{observer}::_proxy_task_handler",
@@ -102,6 +144,7 @@ class TaskObserver(BaseObserver):
return proxy
def _create_proxies(self, observers: List[BaseObserver]) -> Dict[BaseObserver, Proxy]:
"""Create proxies for all observers."""
proxies = {}
for observer in observers:
proxy = self._create_proxy(observer)
@@ -109,6 +152,7 @@ class TaskObserver(BaseObserver):
return proxies
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):
"""Handle frame processing for a single observer."""
warning_reported = False
while True:
data = await queue.get()

34
src/pipecat/pipeline/to_be_updated/merge_pipeline.py
View File

@@ -1,3 +1,16 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Sequential pipeline merging for Pipecat.
This module provides a pipeline implementation that sequentially merges
the output from multiple pipelines, processing them one after another
in a specified order.
"""
from typing import List
from pipecat.frames.frames import EndFrame, EndPipeFrame
@@ -5,14 +18,31 @@ from pipecat.pipeline.pipeline import Pipeline
class SequentialMergePipeline(Pipeline):
"""This class merges the sink queues from a list of pipelines. Frames from
each pipeline's sink are merged in the order of pipelines in the list."""
"""Pipeline that sequentially merges output from multiple pipelines.
This pipeline merges the sink queues from a list of pipelines by processing
frames from each pipeline's sink sequentially in the order specified. Each
pipeline runs to completion before the next one begins processing.
"""
def __init__(self, pipelines: List[Pipeline]):
"""Initialize the sequential merge pipeline.
Args:
pipelines: List of pipelines to merge sequentially. Pipelines will
be processed in the order they appear in this list.
"""
super().__init__([])
self.pipelines = pipelines
async def run_pipeline(self):
"""Run all pipelines sequentially and merge their output.
Processes each pipeline in order, consuming all frames from each
pipeline's sink until an EndFrame or EndPipeFrame is encountered,
then moves to the next pipeline. After all pipelines complete,
sends a final EndFrame to signal completion.
"""
for idx, pipeline in enumerate(self.pipelines):
while True:
frame = await pipeline.sink.get()

45
src/pipecat/processors/aggregators/dtmf_aggregator.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""DTMF aggregation processor for converting keypad input to transcription.
This module provides a frame processor that aggregates DTMF (Dual-Tone Multi-Frequency)
keypad inputs into meaningful sequences and converts them to transcription frames
for downstream processing by LLM context aggregators.
"""
import asyncio
from typing import Optional
@@ -26,16 +33,12 @@ class DTMFAggregator(FrameProcessor):
The aggregator accumulates digits from InputDTMFFrame instances and flushes
when:
- Timeout occurs (configurable idle period)
- Termination digit is received (default: '#')
- EndFrame or CancelFrame is received
Emits TranscriptionFrame for compatibility with existing LLM context aggregators.
Args:
timeout: Idle timeout in seconds before flushing
termination_digit: Digit that triggers immediate flush
prefix: Prefix added to DTMF sequence in transcription
"""
def __init__(
@@ -45,6 +48,14 @@ class DTMFAggregator(FrameProcessor):
prefix: str = "DTMF: ",
**kwargs,
):
"""Initialize the DTMF aggregator.
Args:
timeout: Idle timeout in seconds before flushing
termination_digit: Digit that triggers immediate flush
prefix: Prefix added to DTMF sequence in transcription
**kwargs: Additional arguments passed to FrameProcessor
"""
super().__init__(**kwargs)
self._aggregation = ""
self._idle_timeout = timeout
@@ -53,8 +64,15 @@ class DTMFAggregator(FrameProcessor):
self._digit_event = asyncio.Event()
self._aggregation_task: Optional[asyncio.Task] = None
self._interruption_task: Optional[asyncio.Task] = None
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
"""Process incoming frames and handle DTMF aggregation.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
@@ -64,6 +82,7 @@ class DTMFAggregator(FrameProcessor):
if self._aggregation:
await self._flush_aggregation()
await self._stop_aggregation_task()
await self._stop_interruption_task()
await self.push_frame(frame, direction)
elif isinstance(frame, InputDTMFFrame):
# Push the DTMF frame downstream first
@@ -83,7 +102,7 @@ class DTMFAggregator(FrameProcessor):
# For first digit, schedule interruption in separate task
if is_first_digit:
asyncio.create_task(self._send_interruption_task())
self._interruption_task = self.create_task(self._send_interruption_task())
# Check for immediate flush conditions
if frame.button == self._termination_digit:
@@ -94,12 +113,13 @@ class DTMFAggregator(FrameProcessor):
async def _send_interruption_task(self):
"""Send interruption frame safely in a separate task."""
try:
# Send the interruption frame
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
except Exception as e:
# Log error but don't propagate
print(f"Error sending interruption: {e}")
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
async def _stop_interruption_task(self) -> None:
"""Stops the interruption task."""
if self._interruption_task:
await self.cancel_task(self._interruption_task)
self._interruption_task = None
def _create_aggregation_task(self) -> None:
"""Creates the aggregation task if it hasn't been created yet."""
@@ -119,6 +139,7 @@ class DTMFAggregator(FrameProcessor):
await asyncio.wait_for(self._digit_event.wait(), timeout=self._idle_timeout)
self._digit_event.clear()
except asyncio.TimeoutError:
self.reset_watchdog()
if self._aggregation:
await self._flush_aggregation()

49
src/pipecat/processors/aggregators/gated.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Gated frame aggregator for conditional frame accumulation.
This module provides a gated aggregator that accumulates frames based on
custom gate open/close functions, allowing for conditional frame buffering
and release in frame processing pipelines.
"""
from typing import List, Tuple
from loguru import logger
@@ -14,31 +21,11 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class GatedAggregator(FrameProcessor):
"""Accumulate frames, with custom functions to start and stop accumulation.
Yields gate-opening frame before any accumulated frames, then ensuing frames
until and not including the gate-closed frame.
Doctest: FIXME to work with asyncio
>>> from pipecat.frames.frames import ImageRawFrame
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... if isinstance(frame, TextFrame):
... print(frame.text)
... else:
... print(frame.__class__.__name__)
>>> aggregator = GatedAggregator(
... gate_close_fn=lambda x: isinstance(x, LLMResponseStartFrame),
... gate_open_fn=lambda x: isinstance(x, ImageRawFrame),
... start_open=False)
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello again.")))
>>> asyncio.run(print_frames(aggregator, ImageRawFrame(image=bytes([]), size=(0, 0))))
ImageRawFrame
Hello
Hello again.
>>> asyncio.run(print_frames(aggregator, TextFrame("Goodbye.")))
Goodbye.
until and not including the gate-closed frame. The aggregator maintains an
internal gate state that controls whether frames are passed through immediately
or accumulated for later release.
"""
def __init__(
@@ -48,6 +35,14 @@ class GatedAggregator(FrameProcessor):
start_open,
direction: FrameDirection = FrameDirection.DOWNSTREAM,
):
"""Initialize the gated aggregator.
Args:
gate_open_fn: Function that returns True when a frame should open the gate.
gate_close_fn: Function that returns True when a frame should close the gate.
start_open: Whether the gate should start in the open state.
direction: The frame direction this aggregator operates on.
"""
super().__init__()
self._gate_open_fn = gate_open_fn
self._gate_close_fn = gate_close_fn
@@ -56,6 +51,12 @@ class GatedAggregator(FrameProcessor):
self._accumulator: List[Tuple[Frame, FrameDirection]] = []
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames with gated accumulation logic.
Args:
frame: The frame to process.
direction: The direction of the frame flow.
"""
await super().process_frame(frame, direction)
# We must not block system frames.

24
src/pipecat/processors/aggregators/gated_openai_llm_context.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Gated OpenAI LLM context aggregator for controlled message flow."""
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -11,12 +13,21 @@ from pipecat.sync.base_notifier import BaseNotifier
class GatedOpenAILLMContextAggregator(FrameProcessor):
"""This aggregator keeps the last received OpenAI LLM context frame and it
doesn't let it through until the notifier is notified.
"""Aggregator that gates OpenAI LLM context frames until notified.
This aggregator captures OpenAI LLM context frames and holds them until
a notifier signals that they can be released. This is useful for controlling
the flow of context frames based on external conditions or timing.
"""
def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs):
"""Initialize the gated context aggregator.
Args:
notifier: The notifier that controls when frames are released.
start_open: If True, the first context frame passes through immediately.
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
self._notifier = notifier
self._start_open = start_open
@@ -24,6 +35,12 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
self._gate_task = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames, gating OpenAI LLM context frames.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
@@ -42,15 +59,18 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
await self.push_frame(frame, direction)
async def _start(self):
"""Start the gate task handler."""
if not self._gate_task:
self._gate_task = self.create_task(self._gate_task_handler())
async def _stop(self):
"""Stop the gate task handler."""
if self._gate_task:
await self.cancel_task(self._gate_task)
self._gate_task = None
async def _gate_task_handler(self):
"""Handle the gating logic by waiting for notifications and releasing frames."""
while True:
await self._notifier.wait()
if self._last_context_frame:

384
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""LLM response aggregators for handling conversation context and message aggregation.
This module provides aggregators that process and accumulate LLM responses, user inputs,
and conversation context. These aggregators handle the flow between speech-to-text,
LLM processing, and text-to-speech components in conversational AI pipelines.
"""
import asyncio
from abc import abstractmethod
from dataclasses import dataclass
@@ -54,30 +61,55 @@ from pipecat.utils.time import time_now_iso8601
@dataclass
class LLMUserAggregatorParams:
"""Parameters for configuring LLM user aggregation behavior.
Parameters:
aggregation_timeout: Maximum time in seconds to wait for additional
transcription content before pushing aggregated result. This
timeout is used only when the transcription is slow to arrive.
"""
aggregation_timeout: float = 0.5
@dataclass
class LLMAssistantAggregatorParams:
"""Parameters for configuring LLM assistant aggregation behavior.
Parameters:
expect_stripped_words: Whether to expect and handle stripped words
in text frames by adding spaces between tokens.
"""
expect_stripped_words: bool = True
class LLMFullResponseAggregator(FrameProcessor):
"""This is an LLM aggregator that aggregates a full LLM completion. It
aggregates LLM text frames (tokens) received between
`LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. Every full
completion is returned via the "on_completion" event handler:
"""Aggregates complete LLM responses between start and end frames.
@aggregator.event_handler("on_completion")
async def on_completion(
aggregator: LLMFullResponseAggregator,
completion: str,
completed: bool,
)
This aggregator collects LLM text frames (tokens) received between
`LLMFullResponseStartFrame` and `LLMFullResponseEndFrame` and provides
the complete response via an event handler.
The aggregator provides an "on_completion" event that fires when a full
completion is available::
@aggregator.event_handler("on_completion")
async def on_completion(
aggregator: LLMFullResponseAggregator,
completion: str,
completed: bool,
):
# Handle the completion
pass
"""
def __init__(self, **kwargs):
"""Initialize the LLM full response aggregator.
Args:
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs)
self._aggregation = ""
@@ -86,6 +118,12 @@ class LLMFullResponseAggregator(FrameProcessor):
self._register_event_handler("on_completion")
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and aggregate LLM text content.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame):
@@ -116,83 +154,123 @@ class LLMFullResponseAggregator(FrameProcessor):
class BaseLLMResponseAggregator(FrameProcessor):
"""This is the base class for all LLM response aggregators. These
aggregators process incoming frames and aggregate content until they are
ready to push the aggregation. In the case of a user, an aggregation might
be a full transcription received from the STT service.
"""Base class for all LLM response aggregators.
The LLM response aggregators also keep a store (e.g. a message list or an
LLM context) of the current conversation, that is, it stores the messages
said by the user or by the bot.
These aggregators process incoming frames and aggregate content until they are
ready to push the aggregation downstream. They maintain conversation state
and handle message flow between different components in the pipeline.
The aggregators keep a store (e.g. message list or LLM context) of the current
conversation, storing messages from both users and the bot.
"""
def __init__(self, **kwargs):
"""Initialize the base LLM response aggregator.
Args:
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs)
@property
@abstractmethod
def messages(self) -> List[dict]:
"""Returns the messages from the current conversation."""
"""Get the messages from the current conversation.
Returns:
List of message dictionaries representing the conversation history.
"""
pass
@property
@abstractmethod
def role(self) -> str:
"""Returns the role (e.g. user, assistant...) for this aggregator."""
"""Get the role for this aggregator.
Returns:
The role string (e.g. "user", "assistant") for this aggregator.
"""
pass
@abstractmethod
def add_messages(self, messages):
"""Add the given messages to the conversation."""
"""Add the given messages to the conversation.
Args:
messages: Messages to append to the conversation history.
"""
pass
@abstractmethod
def set_messages(self, messages):
"""Reset the conversation with the given messages."""
"""Reset the conversation with the given messages.
Args:
messages: Messages to replace the current conversation history.
"""
pass
@abstractmethod
def set_tools(self, tools):
"""Set LLM tools to be used in the current conversation."""
"""Set LLM tools to be used in the current conversation.
Args:
tools: List of tool definitions for the LLM to use.
"""
pass
@abstractmethod
def set_tool_choice(self, tool_choice):
"""Set the tool choice. This should modify the LLM context."""
"""Set the tool choice for the LLM.
Args:
tool_choice: Tool choice configuration for the LLM context.
"""
pass
@abstractmethod
async def reset(self):
"""Reset the internals of this aggregator. This should not modify the
internal messages.
"""Reset the internal state of this aggregator.
This should clear aggregation state but not modify the conversation messages.
"""
pass
@abstractmethod
async def handle_aggregation(self, aggregation: str):
"""Adds the given aggregation to the aggregator. The aggregator can use
a simple list of message or a context. It doesn't not push any frames.
"""Add the given aggregation to the conversation store.
Args:
aggregation: The aggregated text content to add to the conversation.
"""
pass
@abstractmethod
async def push_aggregation(self):
"""Pushes the current aggregation. For example, iN the case of context
aggregation this might push a new context frame.
"""Push the current aggregation downstream.
The specific frame type pushed depends on the aggregator implementation
(e.g. context frame, messages frame).
"""
pass
class LLMContextResponseAggregator(BaseLLMResponseAggregator):
"""This is a base LLM aggregator that uses an LLM context to store the
conversation. It pushes `OpenAILLMContextFrame` as an aggregation frame.
"""Base LLM aggregator that uses an OpenAI LLM context for conversation storage.
This aggregator maintains conversation state using an OpenAILLMContext and
pushes OpenAILLMContextFrame objects as aggregation frames. It provides
common functionality for context-based conversation management.
"""
def __init__(self, *, context: OpenAILLMContext, role: str, **kwargs):
"""Initialize the context response aggregator.
Args:
context: The OpenAI LLM context to use for conversation storage.
role: The role this aggregator represents (e.g. "user", "assistant").
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._context = context
self._role = role
@@ -201,46 +279,99 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator):
@property
def messages(self) -> List[dict]:
"""Get messages from the LLM context.
Returns:
List of message dictionaries from the context.
"""
return self._context.get_messages()
@property
def role(self) -> str:
"""Get the role for this aggregator.
Returns:
The role string for this aggregator.
"""
return self._role
@property
def context(self):
"""Get the OpenAI LLM context.
Returns:
The OpenAILLMContext instance used by this aggregator.
"""
return self._context
def get_context_frame(self) -> OpenAILLMContextFrame:
"""Create a context frame with the current context.
Returns:
OpenAILLMContextFrame containing the current context.
"""
return OpenAILLMContextFrame(context=self._context)
async def push_context_frame(self, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a context frame in the specified direction.
Args:
direction: The direction to push the frame (upstream or downstream).
"""
frame = self.get_context_frame()
await self.push_frame(frame, direction)
def add_messages(self, messages):
"""Add messages to the context.
Args:
messages: Messages to add to the conversation context.
"""
self._context.add_messages(messages)
def set_messages(self, messages):
"""Set the context messages.
Args:
messages: Messages to replace the current context messages.
"""
self._context.set_messages(messages)
def set_tools(self, tools: List):
"""Set tools in the context.
Args:
tools: List of tool definitions to set in the context.
"""
self._context.set_tools(tools)
def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict):
"""Set tool choice in the context.
Args:
tool_choice: Tool choice configuration for the context.
"""
self._context.set_tool_choice(tool_choice)
async def reset(self):
"""Reset the aggregation state."""
self._aggregation = ""
class LLMUserContextAggregator(LLMContextResponseAggregator):
"""This is a user LLM aggregator that uses an LLM context to store the
conversation. It aggregates transcriptions from the STT service and it has
logic to handle multiple scenarios where transcriptions are received between
VAD events (`UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame`) or
even outside or no VAD events at all.
"""User LLM aggregator that processes speech-to-text transcriptions.
This aggregator handles the complex logic of aggregating user speech transcriptions
from STT services. It manages multiple scenarios including:
- Transcriptions received between VAD events
- Transcriptions received outside VAD events
- Interim vs final transcriptions
- User interruptions during bot speech
- Emulated VAD for whispered or short utterances
The aggregator uses timeouts to handle cases where transcriptions arrive
after VAD events or when no VAD is available.
"""
def __init__(
@@ -250,6 +381,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
params: Optional[LLMUserAggregatorParams] = None,
**kwargs,
):
"""Initialize the user context aggregator.
Args:
context: The OpenAI LLM context for conversation storage.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments. Supports deprecated 'aggregation_timeout'.
"""
super().__init__(context=context, role="user", **kwargs)
self._params = params or LLMUserAggregatorParams()
if "aggregation_timeout" in kwargs:
@@ -266,6 +404,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
self._user_speaking = False
self._bot_speaking = False
self._was_bot_speaking = False
self._emulating_vad = False
self._seen_interim_results = False
self._waiting_for_aggregation = False
@@ -274,15 +413,28 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
self._aggregation_task = None
async def reset(self):
"""Reset the aggregation state and interruption strategies."""
await super().reset()
self._was_bot_speaking = False
self._seen_interim_results = False
self._waiting_for_aggregation = False
[await s.reset() for s in self._interruption_strategies]
async def handle_aggregation(self, aggregation: str):
"""Add the aggregated user text to the context.
Args:
aggregation: The aggregated user text to add as a user message.
"""
self._context.add_message({"role": self.role, "content": aggregation})
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for user speech aggregation and context management.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
@@ -318,9 +470,9 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
elif isinstance(frame, InterimTranscriptionFrame):
await self._handle_interim_transcription(frame)
elif isinstance(frame, LLMMessagesAppendFrame):
self.add_messages(frame.messages)
await self._handle_llm_messages_append(frame)
elif isinstance(frame, LLMMessagesUpdateFrame):
self.set_messages(frame.messages)
await self._handle_llm_messages_update(frame)
elif isinstance(frame, LLMSetToolsFrame):
self.set_tools(frame.tools)
elif isinstance(frame, LLMSetToolChoiceFrame):
@@ -337,7 +489,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
await self.push_frame(frame)
async def push_aggregation(self):
"""Pushes the current aggregation based on interruption strategies and conditions."""
"""Push the current aggregation based on interruption strategies and conditions."""
if len(self._aggregation) > 0:
if self.interruption_strategies and self._bot_speaking:
should_interrupt = await self._should_interrupt_based_on_strategies()
@@ -355,9 +507,27 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
else:
# No interruption config - normal behavior (always push aggregation)
await self._process_aggregation()
# Handles the case where both the user and the bot are not speaking,
# and the bot was previously speaking before the user interruption.
# Normally, when the user stops speaking, new text is expected,
# which triggers the bot to respond. However, if no new text
# is received, this safeguard ensures
# the bot doesn't hang indefinitely while waiting to speak again.
elif not self._seen_interim_results and self._was_bot_speaking and not self._bot_speaking:
logger.warning("User stopped speaking but no new aggregation received.")
# Resetting it so we don't trigger this twice
self._was_bot_speaking = False
# TODO: we are not enabling this for now, due to some STT services which can take as long as 2 seconds two return a transcription
# So we need more tests and probably make this feature configurable, disabled it by default.
# We are just pushing the same previous context to be processed again in this case
# await self.push_frame(OpenAILLMContextFrame(self._context))
async def _should_interrupt_based_on_strategies(self) -> bool:
"""Check if interruption should occur based on configured strategies."""
"""Check if interruption should occur based on configured strategies.
Returns:
True if any interruption strategy indicates interruption should occur.
"""
async def should_interrupt(strategy: BaseInterruptionStrategy):
await strategy.append_text(self._aggregation)
@@ -374,6 +544,16 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
async def _cancel(self, frame: CancelFrame):
await self._cancel_aggregation_task()
async def _handle_llm_messages_append(self, frame: LLMMessagesAppendFrame):
self.add_messages(frame.messages)
if frame.run_llm:
await self.push_context_frame()
async def _handle_llm_messages_update(self, frame: LLMMessagesUpdateFrame):
self.set_messages(frame.messages)
if frame.run_llm:
await self.push_context_frame()
async def _handle_input_audio(self, frame: InputAudioRawFrame):
for s in self.interruption_strategies:
await s.append_audio(frame.audio, frame.sample_rate)
@@ -381,6 +561,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame):
self._user_speaking = True
self._waiting_for_aggregation = True
self._was_bot_speaking = self._bot_speaking
# If we get a non-emulated UserStartedSpeakingFrame but we are in the
# middle of emulating VAD, let's stop emulating VAD (i.e. don't send the
@@ -393,8 +574,15 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
# We just stopped speaking. Let's see if there's some aggregation to
# push. If the last thing we saw is an interim transcription, let's wait
# pushing the aggregation as we will probably get a final transcription.
if not self._seen_interim_results:
await self.push_aggregation()
if len(self._aggregation) > 0:
if not self._seen_interim_results:
await self.push_aggregation()
# Handles the case where both the user and the bot are not speaking,
# and the bot was previously speaking before the user interruption.
# So in this case we are resetting the aggregation timer
elif not self._seen_interim_results and self._was_bot_speaking and not self._bot_speaking:
# Reset aggregation timer.
self._aggregation_event.set()
async def _handle_bot_started_speaking(self, _: BotStartedSpeakingFrame):
self._bot_speaking = True
@@ -446,12 +634,14 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
)
self._emulating_vad = False
finally:
self.reset_watchdog()
self._aggregation_event.clear()
async def _maybe_emulate_user_speaking(self):
"""Emulate user speaking if we got a transcription but it was not
detected by VAD. Only do that if the bot is not speaking.
"""Maybe emulate user speaking based on transcription.
Emulate user speaking if we got a transcription but it was not
detected by VAD. Only do that if the bot is not speaking.
"""
# Check if we received a transcription but VAD was not able to detect
# voice (e.g. when you whisper a short utterance). In that case, we need
@@ -472,10 +662,18 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
class LLMAssistantContextAggregator(LLMContextResponseAggregator):
"""This is an assistant LLM aggregator that uses an LLM context to store the
conversation. It aggregates text frames received between
`LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`.
"""Assistant LLM aggregator that processes bot responses and function calls.
This aggregator handles the complex logic of processing assistant responses including:
- Text frame aggregation between response start/end markers
- Function call lifecycle management
- Context updates with timestamps
- Tool execution and result handling
- Interruption handling during responses
The aggregator manages function calls in progress and coordinates between
text generation and tool execution phases of LLM responses.
"""
def __init__(
@@ -485,6 +683,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
params: Optional[LLMAssistantAggregatorParams] = None,
**kwargs,
):
"""Initialize the assistant context aggregator.
Args:
context: The OpenAI LLM context for conversation storage.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments. Supports deprecated 'expect_stripped_words'.
"""
super().__init__(context=context, role="assistant", **kwargs)
self._params = params or LLMAssistantAggregatorParams()
@@ -504,22 +709,62 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {}
self._context_updated_tasks: Set[asyncio.Task] = set()
@property
def has_function_calls_in_progress(self) -> bool:
"""Check if there are any function calls currently in progress.
Returns:
True if function calls are in progress, False otherwise.
"""
return bool(self._function_calls_in_progress)
async def handle_aggregation(self, aggregation: str):
"""Add the aggregated assistant text to the context.
Args:
aggregation: The aggregated assistant text to add as an assistant message.
"""
self._context.add_message({"role": "assistant", "content": aggregation})
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
"""Handle a function call that is in progress.
Args:
frame: The function call in progress frame to handle.
"""
pass
async def handle_function_call_result(self, frame: FunctionCallResultFrame):
"""Handle the result of a completed function call.
Args:
frame: The function call result frame to handle.
"""
pass
async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
"""Handle cancellation of a function call.
Args:
frame: The function call cancel frame to handle.
"""
pass
async def handle_user_image_frame(self, frame: UserImageRawFrame):
"""Handle a user image frame associated with a function call.
Args:
frame: The user image frame to handle.
"""
pass
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for assistant response aggregation and function call management.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame):
@@ -532,9 +777,9 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
elif isinstance(frame, TextFrame):
await self._handle_text(frame)
elif isinstance(frame, LLMMessagesAppendFrame):
self.add_messages(frame.messages)
await self._handle_llm_messages_append(frame)
elif isinstance(frame, LLMMessagesUpdateFrame):
self.set_messages(frame.messages)
await self._handle_llm_messages_update(frame)
elif isinstance(frame, LLMSetToolsFrame):
self.set_tools(frame.tools)
elif isinstance(frame, LLMSetToolChoiceFrame):
@@ -556,6 +801,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
await self.push_frame(frame, direction)
async def push_aggregation(self):
"""Push the current assistant aggregation with timestamp."""
if not self._aggregation:
return
@@ -572,6 +818,16 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601())
await self.push_frame(timestamp_frame)
async def _handle_llm_messages_append(self, frame: LLMMessagesAppendFrame):
self.add_messages(frame.messages)
if frame.run_llm:
await self.push_context_frame(FrameDirection.UPSTREAM)
async def _handle_llm_messages_update(self, frame: LLMMessagesUpdateFrame):
self.set_messages(frame.messages)
if frame.run_llm:
await self.push_context_frame(FrameDirection.UPSTREAM)
async def _handle_interruptions(self, frame: StartInterruptionFrame):
await self.push_aggregation()
self._started = 0
@@ -685,6 +941,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
class LLMUserResponseAggregator(LLMUserContextAggregator):
"""User response aggregator that outputs LLMMessagesFrame instead of context frames.
This aggregator extends LLMUserContextAggregator but pushes LLMMessagesFrame
objects downstream instead of OpenAILLMContextFrame objects. This is useful
when you need message-based output rather than context-based output.
"""
def __init__(
self,
messages: Optional[List[dict]] = None,
@@ -692,9 +955,17 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
params: Optional[LLMUserAggregatorParams] = None,
**kwargs,
):
"""Initialize the user response aggregator.
Args:
messages: Initial messages for the conversation context.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)
async def push_aggregation(self):
"""Push the aggregated user input as an LLMMessagesFrame."""
if len(self._aggregation) > 0:
await self.handle_aggregation(self._aggregation)
@@ -707,6 +978,13 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
"""Assistant response aggregator that outputs LLMMessagesFrame instead of context frames.
This aggregator extends LLMAssistantContextAggregator but pushes LLMMessagesFrame
objects downstream instead of OpenAILLMContextFrame objects. This is useful
when you need message-based output rather than context-based output.
"""
def __init__(
self,
messages: Optional[List[dict]] = None,
@@ -714,9 +992,17 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
params: Optional[LLMAssistantAggregatorParams] = None,
**kwargs,
):
"""Initialize the assistant response aggregator.
Args:
messages: Initial messages for the conversation context.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)
async def push_aggregation(self):
"""Push the aggregated assistant response as an LLMMessagesFrame."""
if len(self._aggregation) > 0:
await self.handle_aggregation(self._aggregation)

172
src/pipecat/processors/aggregators/openai_llm_context.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""OpenAI LLM context management for Pipecat.
This module provides classes for managing OpenAI-specific conversation contexts,
including message handling, tool management, and image/audio processing capabilities.
"""
import base64
import copy
import io
@@ -29,7 +35,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class CustomEncoder(json.JSONEncoder):
"""Custom JSON encoder for handling special data types in logging.
Provides specialized encoding for io.BytesIO objects to display
readable representations in log output instead of raw binary data.
"""
def default(self, obj):
"""Encode special objects for JSON serialization.
Args:
obj: The object to encode.
Returns:
Encoded representation of the object.
"""
if isinstance(obj, io.BytesIO):
# Convert the first 8 bytes to an ASCII hex string
return f"{obj.getbuffer()[0:8].hex()}..."
@@ -37,25 +57,57 @@ class CustomEncoder(json.JSONEncoder):
class OpenAILLMContext:
"""Manages conversation context for OpenAI LLM interactions.
Handles message history, tool definitions, tool choices, and multimedia content
for OpenAI API conversations. Provides methods for message manipulation,
content formatting, and integration with various LLM adapters.
"""
def __init__(
self,
messages: Optional[List[ChatCompletionMessageParam]] = None,
tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN,
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
):
"""Initialize the OpenAI LLM context.
Args:
messages: Initial list of conversation messages.
tools: Available tools for the LLM to use.
tool_choice: Tool selection strategy for the LLM.
"""
self._messages: List[ChatCompletionMessageParam] = messages if messages else []
self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice
self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools
self._llm_adapter: Optional[BaseLLMAdapter] = None
def get_llm_adapter(self) -> Optional[BaseLLMAdapter]:
"""Get the current LLM adapter.
Returns:
The currently set LLM adapter, or None if not set.
"""
return self._llm_adapter
def set_llm_adapter(self, llm_adapter: BaseLLMAdapter):
"""Set the LLM adapter for context processing.
Args:
llm_adapter: The LLM adapter to use for tool conversion.
"""
self._llm_adapter = llm_adapter
@staticmethod
def from_messages(messages: List[dict]) -> "OpenAILLMContext":
"""Create a context from a list of message dictionaries.
Args:
messages: List of message dictionaries to convert to context.
Returns:
New OpenAILLMContext instance with the provided messages.
"""
context = OpenAILLMContext()
for message in messages:
@@ -66,34 +118,81 @@ class OpenAILLMContext:
@property
def messages(self) -> List[ChatCompletionMessageParam]:
"""Get the current messages list.
Returns:
List of conversation messages.
"""
return self._messages
@property
def tools(self) -> List[ChatCompletionToolParam] | NotGiven | List[Any]:
"""Get the tools list, converting through adapter if available.
Returns:
Tools list, potentially converted by the LLM adapter.
"""
if self._llm_adapter:
return self._llm_adapter.from_standard_tools(self._tools)
return self._tools
@property
def tool_choice(self) -> ChatCompletionToolChoiceOptionParam | NotGiven:
"""Get the current tool choice setting.
Returns:
The tool choice configuration.
"""
return self._tool_choice
def add_message(self, message: ChatCompletionMessageParam):
"""Add a single message to the context.
Args:
message: The message to add to the conversation history.
"""
self._messages.append(message)
def add_messages(self, messages: List[ChatCompletionMessageParam]):
"""Add multiple messages to the context.
Args:
messages: List of messages to add to the conversation history.
"""
self._messages.extend(messages)
def set_messages(self, messages: List[ChatCompletionMessageParam]):
"""Replace all messages in the context.
Args:
messages: New list of messages to replace the current history.
"""
self._messages[:] = messages
def get_messages(self) -> List[ChatCompletionMessageParam]:
"""Get a copy of the current messages list.
Returns:
List of all messages in the conversation history.
"""
return self._messages
def get_messages_json(self) -> str:
"""Get messages as a formatted JSON string.
Returns:
JSON string representation of all messages with custom encoding.
"""
return json.dumps(self._messages, cls=CustomEncoder, ensure_ascii=False, indent=2)
def get_messages_for_logging(self) -> str:
"""Get sanitized messages suitable for logging.
Removes or truncates sensitive data like image content for safe logging.
Returns:
JSON string with sanitized message content for logging.
"""
msgs = []
for message in self.messages:
msg = copy.deepcopy(message)
@@ -111,17 +210,18 @@ class OpenAILLMContext:
def from_standard_message(self, message):
"""Convert from OpenAI message format to OpenAI message format (passthrough).
OpenAI's format allows both simple string content and structured content:
- Simple: {"role": "user", "content": "Hello"}
- Structured: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
OpenAI's format allows both simple string content and structured content::
Simple: {"role": "user", "content": "Hello"}
Structured: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
Since OpenAI is our standard format, this is a passthrough function.
Args:
message (dict): Message in OpenAI format
message: Message in OpenAI format.
Returns:
dict: Same message, unchanged
Same message, unchanged.
"""
return message
@@ -133,20 +233,28 @@ class OpenAILLMContext:
other LLM services that may need to return multiple messages.
Args:
obj (dict): Message in OpenAI format with either:
- Simple content: {"role": "user", "content": "Hello"}
- List content: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
obj: Message in OpenAI format with either simple string content
or structured list content.
Returns:
list: List containing the original messages, preserving whether
the content was in simple string or structured list format
List containing the original messages, preserving the content format.
"""
return [obj]
def get_messages_for_initializing_history(self):
"""Get messages for initializing conversation history.
Returns:
List of messages suitable for history initialization.
"""
return self._messages
def get_messages_for_persistent_storage(self):
"""Get messages formatted for persistent storage.
Returns:
List of messages converted to standard format for storage.
"""
messages = []
for m in self._messages:
standard_messages = self.to_standard_messages(m)
@@ -154,9 +262,19 @@ class OpenAILLMContext:
return messages
def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven):
"""Set the tool choice configuration.
Args:
tool_choice: Tool selection strategy for the LLM.
"""
self._tool_choice = tool_choice
def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN):
"""Set the available tools for the LLM.
Args:
tools: List of tools available to the LLM, or NOT_GIVEN to disable tools.
"""
if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0:
tools = NOT_GIVEN
self._tools = tools
@@ -164,6 +282,14 @@ class OpenAILLMContext:
def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
):
"""Add a message containing an image frame.
Args:
format: Image format (e.g., 'RGB', 'RGBA').
size: Image dimensions as (width, height) tuple.
image: Raw image bytes.
text: Optional text to include with the image.
"""
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
@@ -177,10 +303,30 @@ class OpenAILLMContext:
self.add_message({"role": "user", "content": content})
def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None):
"""Add a message containing audio frames.
Args:
audio_frames: List of audio frame objects to include.
text: Optional text to include with the audio.
Note:
This method is currently a placeholder for future implementation.
"""
# todo: implement for OpenAI models and others
pass
def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size):
"""Create a WAV file header for audio data.
Args:
sample_rate: Audio sample rate in Hz.
num_channels: Number of audio channels.
bits_per_sample: Bits per audio sample.
data_size: Size of audio data in bytes.
Returns:
WAV header as a bytearray.
"""
# RIFF chunk descriptor
header = bytearray()
header.extend(b"RIFF") # ChunkID
@@ -206,10 +352,14 @@ class OpenAILLMContext:
@dataclass
class OpenAILLMContextFrame(Frame):
"""Like an LLMMessagesFrame, but with extra context specific to the OpenAI
"""Frame containing OpenAI-specific LLM context.
Like an LLMMessagesFrame, but with extra context specific to the OpenAI
API. The context in this message is also mutable, and will be changed by the
OpenAIContextAggregator frame processor.
Parameters:
context: The OpenAI LLM context containing messages, tools, and configuration.
"""
context: OpenAILLMContext

39
src/pipecat/processors/aggregators/sentence.py
View File

@@ -4,35 +4,46 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Text sentence aggregation processor for Pipecat.
This module provides a frame processor that accumulates text frames into
complete sentences, only outputting when a sentence-ending pattern is detected.
"""
from pipecat.frames.frames import EndFrame, Frame, InterimTranscriptionFrame, TextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.string import match_endofsentence
class SentenceAggregator(FrameProcessor):
"""This frame processor aggregates text frames into complete sentences.
"""Aggregates text frames into complete sentences.
This processor accumulates incoming text frames until a sentence-ending
pattern is detected, then outputs the complete sentence as a single frame.
Useful for ensuring downstream processors receive coherent, complete sentences
rather than fragmented text.
Frame input/output::
Frame input/output:
TextFrame("Hello,") -> None
TextFrame(" world.") -> TextFrame("Hello world.")
Doctest: FIXME to work with asyncio
>>> import asyncio
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... print(frame.text)
>>> aggregator = SentenceAggregator()
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello,")))
>>> asyncio.run(print_frames(aggregator, TextFrame(" world.")))
Hello, world.
TextFrame(" world.") -> TextFrame("Hello, world.")
"""
def __init__(self):
"""Initialize the sentence aggregator.
Sets up internal state for accumulating text frames into complete sentences.
"""
super().__init__()
self._aggregation = ""
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and aggregate text into complete sentences.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
# We ignore interim description at this point.

24
src/pipecat/processors/aggregators/user_response.py
View File

@@ -4,15 +4,39 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""User response aggregation for text frames.
This module provides an aggregator that collects user responses and outputs
them as TextFrame objects, useful for capturing and processing user input
in conversational pipelines.
"""
from pipecat.frames.frames import TextFrame
from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator
class UserResponseAggregator(LLMUserResponseAggregator):
"""Aggregates user responses into TextFrame objects.
This aggregator extends LLMUserResponseAggregator to specifically handle
user input by collecting text responses and outputting them as TextFrame
objects when the aggregation is complete.
"""
def __init__(self, **kwargs):
"""Initialize the user response aggregator.
Args:
**kwargs: Additional arguments passed to parent LLMUserResponseAggregator.
"""
super().__init__(**kwargs)
async def push_aggregation(self):
"""Push the aggregated user response as a TextFrame.
Creates a TextFrame from the current aggregation if it contains content,
resets the aggregation state, and pushes the frame downstream.
"""
if len(self._aggregation) > 0:
frame = TextFrame(self._aggregation.strip())

38
src/pipecat/processors/aggregators/vision_image_frame.py
View File

@@ -4,33 +4,43 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Vision image frame aggregation for Pipecat.
This module provides frame aggregation functionality to combine text and image
frames into vision frames for multimodal processing.
"""
from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class VisionImageFrameAggregator(FrameProcessor):
"""This aggregator waits for a consecutive TextFrame and an
InputImageRawFrame. After the InputImageRawFrame arrives it will output a
VisionImageRawFrame.
>>> from pipecat.frames.frames import ImageFrame
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... print(frame)
>>> aggregator = VisionImageFrameAggregator()
>>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?")))
>>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0))))
VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B
"""Aggregates consecutive text and image frames into vision frames.
This aggregator waits for a consecutive TextFrame and an InputImageRawFrame.
After the InputImageRawFrame arrives it will output a VisionImageRawFrame
combining both the text and image data for multimodal processing.
"""
def __init__(self):
"""Initialize the vision image frame aggregator.
The aggregator starts with no cached text, waiting for the first
TextFrame to arrive before it can create vision frames.
"""
super().__init__()
self._describe_text = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and aggregate text with images.
Caches TextFrames and combines them with subsequent InputImageRawFrames
to create VisionImageRawFrames. Other frames are passed through unchanged.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame):

28
src/pipecat/processors/async_generator.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Async generator processor for frame serialization and streaming."""
import asyncio
from typing import Any, AsyncGenerator
@@ -17,12 +19,32 @@ from pipecat.serializers.base_serializer import FrameSerializer
class AsyncGeneratorProcessor(FrameProcessor):
"""A frame processor that serializes frames and provides them via async generator.
This processor passes frames through unchanged while simultaneously serializing
them and making the serialized data available through an async generator interface.
Useful for streaming frame data to external consumers while maintaining the
normal frame processing pipeline.
"""
def __init__(self, *, serializer: FrameSerializer, **kwargs):
"""Initialize the async generator processor.
Args:
serializer: The frame serializer to use for converting frames to data.
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
self._serializer = serializer
self._data_queue = asyncio.Queue()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames by passing them through and queuing serialized data.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
@@ -35,6 +57,12 @@ class AsyncGeneratorProcessor(FrameProcessor):
await self._data_queue.put(data)
async def generator(self) -> AsyncGenerator[Any, None]:
"""Generate serialized frame data asynchronously.
Yields:
Serialized frame data from the internal queue until a termination
signal (None) is received.
"""
running = True
while running:
data = await self._data_queue.get()

165
src/pipecat/processors/audio/audio_buffer_processor.py
View File

@@ -4,12 +4,18 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Audio buffer processor for managing and synchronizing audio streams.
This module provides an AudioBufferProcessor that handles buffering and synchronization
of audio from both user input and bot output sources, with support for various audio
configurations and event-driven processing.
"""
import time
from typing import Optional
from pipecat.audio.utils import create_default_resampler, interleave_stereo_audio, mix_audio
from pipecat.audio.utils import create_stream_resampler, interleave_stereo_audio, mix_audio
from pipecat.frames.frames import (
AudioRawFrame,
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
CancelFrame,
@@ -32,28 +38,19 @@ class AudioBufferProcessor(FrameProcessor):
including sample rate conversion and mono/stereo output.
Events:
on_audio_data: Triggered when buffer_size is reached, providing merged audio
on_track_audio_data: Triggered when buffer_size is reached, providing separate tracks
on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio
on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio
Args:
sample_rate (Optional[int]): Desired output sample rate. If None, uses source rate
num_channels (int): Number of channels (1 for mono, 2 for stereo). Defaults to 1
buffer_size (int): Size of buffer before triggering events. 0 for no buffering
user_continuous_stream (bool): Whether user audio is continuous or speech-only
enable_turn_audio (bool): Whether turn audio event handlers should be triggered
- on_audio_data: Triggered when buffer_size is reached, providing merged audio
- on_track_audio_data: Triggered when buffer_size is reached, providing separate tracks
- on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio
- on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio
Audio handling:
- Mono output (num_channels=1): User and bot audio are mixed
- Stereo output (num_channels=2): User audio on left, bot audio on right
- Automatic resampling of incoming audio to match desired sample_rate
- Silence insertion for non-continuous audio streams
- Buffer synchronization between user and bot audio
Note:
When user_continuous_stream is False, the processor expects only speech
segments and will handle silence insertion between segments automatically.
- Mono output (num_channels=1): User and bot audio are mixed
- Stereo output (num_channels=2): User audio on left, bot audio on right
- Automatic resampling of incoming audio to match desired sample_rate
- Silence insertion for non-continuous audio streams
- Buffer synchronization between user and bot audio
"""
def __init__(
@@ -62,19 +59,43 @@ class AudioBufferProcessor(FrameProcessor):
sample_rate: Optional[int] = None,
num_channels: int = 1,
buffer_size: int = 0,
user_continuous_stream: bool = True,
user_continuous_stream: Optional[bool] = None,
enable_turn_audio: bool = False,
**kwargs,
):
"""Initialize the audio buffer processor.
Args:
sample_rate: Desired output sample rate. If None, uses source rate.
num_channels: Number of channels (1 for mono, 2 for stereo). Defaults to 1.
buffer_size: Size of buffer before triggering events. 0 for no buffering.
user_continuous_stream: Controls whether user audio is treated as a continuous
stream for buffering purposes.
.. deprecated:: 0.0.72
This parameter no longer has any effect and will be removed in a future version.
enable_turn_audio: Whether turn audio event handlers should be triggered.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._audio_buffer_size_1s = 0
self._num_channels = num_channels
self._buffer_size = buffer_size
self._user_continuous_stream = user_continuous_stream
self._enable_turn_audio = enable_turn_audio
if user_continuous_stream is not None:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter `user_continuous_stream` is deprecated.",
DeprecationWarning,
)
self._user_audio_buffer = bytearray()
self._bot_audio_buffer = bytearray()
@@ -89,7 +110,8 @@ class AudioBufferProcessor(FrameProcessor):
self._recording = False
self._resampler = create_default_resampler()
self._input_resampler = create_stream_resampler()
self._output_resampler = create_stream_resampler()
self._register_event_handler("on_audio_data")
self._register_event_handler("on_track_audio_data")
@@ -101,7 +123,7 @@ class AudioBufferProcessor(FrameProcessor):
"""Current sample rate of the audio processor.
Returns:
int: The sample rate in Hz
The sample rate in Hz.
"""
return self._sample_rate
@@ -110,7 +132,7 @@ class AudioBufferProcessor(FrameProcessor):
"""Number of channels in the audio output.
Returns:
int: Number of channels (1 for mono, 2 for stereo)
Number of channels (1 for mono, 2 for stereo).
"""
return self._num_channels
@@ -118,7 +140,7 @@ class AudioBufferProcessor(FrameProcessor):
"""Check if both user and bot audio buffers contain data.
Returns:
bool: True if both buffers contain audio data
True if both buffers contain audio data.
"""
return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio(
self._bot_audio_buffer
@@ -131,7 +153,7 @@ class AudioBufferProcessor(FrameProcessor):
on the left channel and bot audio on the right channel.
Returns:
bytes: Mixed audio data
Mixed audio data as bytes.
"""
if self._num_channels == 1:
return mix_audio(bytes(self._user_audio_buffer), bytes(self._bot_audio_buffer))
@@ -159,7 +181,12 @@ class AudioBufferProcessor(FrameProcessor):
self._recording = False
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming audio frames and manage audio buffers."""
"""Process incoming audio frames and manage audio buffers.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
# Update output sample rate if necessary.
@@ -177,19 +204,36 @@ class AudioBufferProcessor(FrameProcessor):
await self.push_frame(frame, direction)
def _update_sample_rate(self, frame: StartFrame):
"""Update the sample rate from the start frame."""
self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
self._audio_buffer_size_1s = self._sample_rate * 2
async def _process_recording(self, frame: Frame):
if self._user_continuous_stream:
await self._handle_continuous_stream(frame)
else:
await self._handle_intermittent_stream(frame)
"""Process audio frames for recording."""
if isinstance(frame, InputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_user_frame_at)
self._user_audio_buffer.extend(silence)
# Add user audio.
resampled = await self._resample_input_audio(frame)
self._user_audio_buffer.extend(resampled)
# Save time of frame so we can compute silence.
self._last_user_frame_at = time.time()
elif self._recording and isinstance(frame, OutputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_bot_frame_at)
self._bot_audio_buffer.extend(silence)
# Add bot audio.
resampled = await self._resample_output_audio(frame)
self._bot_audio_buffer.extend(resampled)
# Save time of frame so we can compute silence.
self._last_bot_frame_at = time.time()
if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size:
await self._call_on_audio_data_handler()
async def _process_turn_recording(self, frame: Frame):
"""Process frames for turn-based audio recording."""
if isinstance(frame, UserStartedSpeakingFrame):
self._user_speaking = True
elif isinstance(frame, UserStoppedSpeakingFrame):
@@ -208,7 +252,7 @@ class AudioBufferProcessor(FrameProcessor):
self._bot_turn_audio_buffer = bytearray()
if isinstance(frame, InputAudioRawFrame):
resampled = await self._resample_audio(frame)
resampled = await self._resample_input_audio(frame)
self._user_turn_audio_buffer += resampled
# In the case of the user, we need to keep a short buffer of audio
# since VAD notification of when the user starts speaking comes
@@ -220,45 +264,11 @@ class AudioBufferProcessor(FrameProcessor):
discarded = len(self._user_turn_audio_buffer) - self._audio_buffer_size_1s
self._user_turn_audio_buffer = self._user_turn_audio_buffer[discarded:]
elif self._bot_speaking and isinstance(frame, OutputAudioRawFrame):
resampled = await self._resample_audio(frame)
resampled = await self._resample_output_audio(frame)
self._bot_turn_audio_buffer += resampled
async def _handle_continuous_stream(self, frame: Frame):
if isinstance(frame, InputAudioRawFrame):
# Add user audio.
resampled = await self._resample_audio(frame)
self._user_audio_buffer.extend(resampled)
# Sync the bot's buffer to the user's buffer by adding silence if needed
if len(self._user_audio_buffer) > len(self._bot_audio_buffer):
silence_size = len(self._user_audio_buffer) - len(self._bot_audio_buffer)
silence = b"\x00" * silence_size
self._bot_audio_buffer.extend(silence)
elif self._recording and isinstance(frame, OutputAudioRawFrame):
# Add bot audio.
resampled = await self._resample_audio(frame)
self._bot_audio_buffer.extend(resampled)
async def _handle_intermittent_stream(self, frame: Frame):
if isinstance(frame, InputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_user_frame_at)
self._user_audio_buffer.extend(silence)
# Add user audio.
resampled = await self._resample_audio(frame)
self._user_audio_buffer.extend(resampled)
# Save time of frame so we can compute silence.
self._last_user_frame_at = time.time()
elif self._recording and isinstance(frame, OutputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_bot_frame_at)
self._bot_audio_buffer.extend(silence)
# Add bot audio.
resampled = await self._resample_audio(frame)
self._bot_audio_buffer.extend(resampled)
# Save time of frame so we can compute silence.
self._last_bot_frame_at = time.time()
async def _call_on_audio_data_handler(self):
"""Call the audio data event handlers with buffered audio."""
if not self.has_audio() or not self._recording:
return
@@ -280,23 +290,36 @@ class AudioBufferProcessor(FrameProcessor):
self._reset_audio_buffers()
def _buffer_has_audio(self, buffer: bytearray) -> bool:
"""Check if a buffer contains audio data."""
return buffer is not None and len(buffer) > 0
def _reset_recording(self):
"""Reset recording state and buffers."""
self._reset_audio_buffers()
self._last_user_frame_at = time.time()
self._last_bot_frame_at = time.time()
def _reset_audio_buffers(self):
"""Reset all audio buffers to empty state."""
self._user_audio_buffer = bytearray()
self._bot_audio_buffer = bytearray()
self._user_turn_audio_buffer = bytearray()
self._bot_turn_audio_buffer = bytearray()
async def _resample_audio(self, frame: AudioRawFrame) -> bytes:
return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate)
async def _resample_input_audio(self, frame: InputAudioRawFrame) -> bytes:
"""Resample audio frame to the target sample rate."""
return await self._input_resampler.resample(
frame.audio, frame.sample_rate, self._sample_rate
)
async def _resample_output_audio(self, frame: OutputAudioRawFrame) -> bytes:
"""Resample audio frame to the target sample rate."""
return await self._output_resampler.resample(
frame.audio, frame.sample_rate, self._sample_rate
)
def _compute_silence(self, from_time: float) -> bytes:
"""Compute silence to insert based on time gap."""
quiet_time = time.time() - from_time
# We should get audio frames very frequently. We introduce silence only
# if there's a big enough gap of 1s.

33
src/pipecat/processors/consumer_processor.py
View File

@@ -4,20 +4,23 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Consumer processor for consuming frames from ProducerProcessor queues."""
import asyncio
from typing import Awaitable, Callable, Optional
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.producer_processor import ProducerProcessor, identity_transformer
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
class ConsumerProcessor(FrameProcessor):
"""This class passes-through frames and also consumes frames from a
producer's queue. When a frame from a producer queue is received it will be
pushed to the specified direction. The frames can be transformed into a
different type of frame before being pushed.
"""Frame processor that consumes frames from a ProducerProcessor's queue.
This processor passes through frames normally while also consuming frames
from a ProducerProcessor's queue. When frames are received from the producer
queue, they are optionally transformed and pushed in the specified direction.
"""
def __init__(
@@ -28,13 +31,27 @@ class ConsumerProcessor(FrameProcessor):
direction: FrameDirection = FrameDirection.DOWNSTREAM,
**kwargs,
):
"""Initialize the consumer processor.
Args:
producer: The producer processor to consume frames from.
transformer: Function to transform frames before pushing. Defaults to identity_transformer.
direction: Direction to push consumed frames. Defaults to DOWNSTREAM.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._transformer = transformer
self._direction = direction
self._queue: asyncio.Queue = producer.add_consumer()
self._producer = producer
self._consumer_task: Optional[asyncio.Task] = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle lifecycle events.
Args:
frame: The frame to process.
direction: The direction the frame is traveling.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
@@ -47,18 +64,24 @@ class ConsumerProcessor(FrameProcessor):
await self.push_frame(frame, direction)
async def _start(self, _: StartFrame):
"""Start the consumer task and register with the producer."""
if not self._consumer_task:
self._queue: WatchdogQueue = self._producer.add_consumer()
self._consumer_task = self.create_task(self._consumer_task_handler())
async def _stop(self, _: EndFrame):
"""Stop the consumer task."""
if self._consumer_task:
await self.cancel_task(self._consumer_task)
async def _cancel(self, _: CancelFrame):
"""Cancel the consumer task."""
if self._consumer_task:
self._queue.cancel()
await self.cancel_task(self._consumer_task)
async def _consumer_task_handler(self):
"""Handle consuming frames from the producer queue."""
while True:
frame = await self._queue.get()
new_frame = await self._transformer(frame)

23
src/pipecat/processors/filters/frame_filter.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Frame filtering processor for the Pipecat framework."""
from typing import Tuple, Type
from pipecat.frames.frames import EndFrame, Frame, SystemFrame
@@ -11,7 +13,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FrameFilter(FrameProcessor):
"""A frame processor that filters frames based on their types.
This processor acts as a selective gate in the pipeline, allowing only
frames of specified types to pass through. System and end frames are
automatically allowed to pass through to maintain pipeline integrity.
"""
def __init__(self, types: Tuple[Type[Frame], ...]):
"""Initialize the frame filter.
Args:
types: Tuple of frame types that should be allowed to pass through
the filter. All other frame types (except SystemFrame and
EndFrame) will be blocked.
"""
super().__init__()
self._types = types
@@ -20,12 +36,19 @@ class FrameFilter(FrameProcessor):
#
def _should_passthrough_frame(self, frame):
"""Determine if a frame should pass through the filter."""
if isinstance(frame, self._types):
return True
return isinstance(frame, (EndFrame, SystemFrame))
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process an incoming frame and conditionally pass it through.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if self._should_passthrough_frame(frame):

30
src/pipecat/processors/filters/function_filter.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Function-based frame filtering for Pipecat pipelines.
This module provides a processor that filters frames based on a custom function,
allowing for flexible frame filtering logic in processing pipelines.
"""
from typing import Awaitable, Callable
from pipecat.frames.frames import EndFrame, Frame, SystemFrame
@@ -11,11 +17,26 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FunctionFilter(FrameProcessor):
"""A frame processor that filters frames using a custom function.
This processor allows frames to pass through based on the result of a
user-provided filter function. System and end frames always pass through
regardless of the filter result.
"""
def __init__(
self,
filter: Callable[[Frame], Awaitable[bool]],
direction: FrameDirection = FrameDirection.DOWNSTREAM,
):
"""Initialize the function filter.
Args:
filter: An async function that takes a Frame and returns True if the
frame should pass through, False otherwise.
direction: The direction to apply filtering. Only frames moving in
this direction will be filtered. Defaults to DOWNSTREAM.
"""
super().__init__()
self._filter = filter
self._direction = direction
@@ -27,9 +48,18 @@ class FunctionFilter(FrameProcessor):
# Ignore system frames, end frames and frames that are not following the
# direction of this gate
def _should_passthrough_frame(self, frame, direction):
"""Check if a frame should pass through without filtering."""
# Ignore system frames, end frames and frames that are not following the
# direction of this gate
return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame through the filter.
Args:
frame: The frame to process.
direction: The direction the frame is moving in the pipeline.
"""
await super().process_frame(frame, direction)
passthrough = self._should_passthrough_frame(frame, direction)

19
src/pipecat/processors/filters/identity_filter.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Identity filter for transparent frame passthrough.
This module provides a simple passthrough filter that forwards all frames
without modification, useful for testing and pipeline composition.
"""
from pipecat.frames.frames import Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -14,10 +20,14 @@ class IdentityFilter(FrameProcessor):
This filter acts as a transparent passthrough, allowing all frames to flow
through unchanged. It can be useful when testing `ParallelPipeline` to
create pipelines that pass through frames (no frames should be repeated).
"""
def __init__(self, **kwargs):
"""Initialize the identity filter.
Args:
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
#
@@ -25,6 +35,11 @@ class IdentityFilter(FrameProcessor):
#
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process an incoming frame by passing it through unchanged."""
"""Process an incoming frame by passing it through unchanged.
Args:
frame: The frame to process and forward.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)

25
src/pipecat/processors/filters/null_filter.py
View File

@@ -4,14 +4,31 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Null filter processor for blocking frame transmission.
This module provides a frame processor that blocks all frames except
system and end frames, useful for testing or temporarily stopping
frame flow in a pipeline.
"""
from pipecat.frames.frames import EndFrame, Frame, SystemFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class NullFilter(FrameProcessor):
"""This filter doesn't allow passing any frames up or downstream."""
"""A filter that blocks all frames except system and end frames.
This processor acts as a null filter, preventing frames from passing
through the pipeline while still allowing essential system and end
frames to maintain proper pipeline operation.
"""
def __init__(self, **kwargs):
"""Initialize the null filter.
Args:
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs)
#
@@ -19,6 +36,12 @@ class NullFilter(FrameProcessor):
#
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames, only allowing system and end frames through.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, (SystemFrame, EndFrame)):

70
src/pipecat/processors/filters/stt_mute_filter.py
View File

@@ -39,12 +39,17 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class STTMuteStrategy(Enum):
"""Strategies determining when STT should be muted.
Attributes:
FIRST_SPEECH: Mute only during first detected bot speech
MUTE_UNTIL_FIRST_BOT_COMPLETE: Start muted and remain muted until first bot speech completes
FUNCTION_CALL: Mute during function calls
ALWAYS: Mute during all bot speech
CUSTOM: Allow custom logic via callback
Each strategy defines different conditions under which speech-to-text
processing should be temporarily disabled to prevent unwanted audio
processing during specific conversation states.
Parameters:
FIRST_SPEECH: Mute STT until the first bot speech is detected.
MUTE_UNTIL_FIRST_BOT_COMPLETE: Mute STT until the first bot completes speaking,
regardless of whether it is the first speech.
FUNCTION_CALL: Mute STT during function calls to prevent interruptions.
ALWAYS: Always mute STT when the bot is speaking.
CUSTOM: Use a custom callback to determine muting logic dynamically.
"""
FIRST_SPEECH = "first_speech"
@@ -58,10 +63,15 @@ class STTMuteStrategy(Enum):
class STTMuteConfig:
"""Configuration for STT muting behavior.
Args:
strategies: Set of muting strategies to apply
Defines which muting strategies to apply and provides optional custom
callback for advanced muting logic. Multiple strategies can be combined
to create sophisticated muting behavior.
Parameters:
strategies: Set of muting strategies to apply simultaneously.
should_mute_callback: Optional callback for custom muting logic.
Only required when using STTMuteStrategy.CUSTOM
Only required when using STTMuteStrategy.CUSTOM. Called with
the STTMuteFilter instance to determine muting state.
Note:
MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together
@@ -69,10 +79,14 @@ class STTMuteConfig:
"""
strategies: set[STTMuteStrategy]
# Optional callback for custom muting logic
should_mute_callback: Optional[Callable[["STTMuteFilter"], Awaitable[bool]]] = None
def __post_init__(self):
"""Validate configuration after initialization.
Raises:
ValueError: If incompatible strategies are used together.
"""
if (
STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE in self.strategies
and STTMuteStrategy.FIRST_SPEECH in self.strategies
@@ -86,15 +100,18 @@ class STTMuteFilter(FrameProcessor):
"""A processor that handles STT muting and interruption control.
This processor combines STT muting and interruption control as a coordinated
feature. When STT is muted, interruptions are automatically disabled.
Args:
config: Configuration specifying muting strategies
stt_service: STT service instance (deprecated, will be removed in future version)
**kwargs: Additional arguments passed to parent class
feature. When STT is muted, interruptions are automatically disabled by
suppressing VAD-related frames. This prevents unwanted speech detection
during bot speech, function calls, or other specified conditions.
"""
def __init__(self, *, config: STTMuteConfig, **kwargs):
"""Initialize the STT mute filter.
Args:
config: Configuration specifying muting strategies and behavior.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._config = config
self._first_speech_handled = False
@@ -104,18 +121,22 @@ class STTMuteFilter(FrameProcessor):
@property
def is_muted(self) -> bool:
"""Returns whether STT is currently muted."""
"""Check if STT is currently muted.
Returns:
True if STT is currently muted and audio frames are being suppressed.
"""
return self._is_muted
async def _handle_mute_state(self, should_mute: bool):
"""Handles both STT muting and interruption control."""
"""Handle STT muting and interruption control state changes."""
if should_mute != self.is_muted:
logger.debug(f"STTMuteFilter {'muting' if should_mute else 'unmuting'}")
self._is_muted = should_mute
await self.push_frame(STTMuteFrame(mute=should_mute))
async def _should_mute(self) -> bool:
"""Determines if STT should be muted based on current state and strategy."""
"""Determine if STT should be muted based on current state and strategies."""
for strategy in self._config.strategies:
match strategy:
case STTMuteStrategy.FUNCTION_CALL:
@@ -144,7 +165,16 @@ class STTMuteFilter(FrameProcessor):
return False
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Processes incoming frames and manages muting state."""
"""Process incoming frames and manage muting state.
Monitors conversation state through frame types and applies muting
strategies accordingly. Suppresses VAD-related frames when muted
while allowing other frames to pass through.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
# Determine if we need to change mute state based on frame type

49
src/pipecat/processors/filters/wake_check_filter.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Wake phrase detection filter for Pipecat transcription processing.
This module provides a frame processor that filters transcription frames,
only allowing them through after wake phrases have been detected. Includes
keepalive functionality to maintain conversation flow after wake detection.
"""
import re
import time
from enum import Enum
@@ -16,23 +23,53 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class WakeCheckFilter(FrameProcessor):
"""This filter looks for wake phrases in the transcription frames and only passes through frames
after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief
period of continued conversation after a wake phrase has been detected.
"""Frame processor that filters transcription frames based on wake phrase detection.
This filter monitors transcription frames for configured wake phrases and only
passes frames through after a wake phrase has been detected. Maintains a
keepalive timeout to allow continued conversation after wake detection.
"""
class WakeState(Enum):
"""Enumeration of wake detection states.
Parameters:
IDLE: No wake phrase detected, filtering active.
AWAKE: Wake phrase detected, allowing frames through.
"""
IDLE = 1
AWAKE = 2
class ParticipantState:
"""State tracking for individual participants.
Parameters:
participant_id: Unique identifier for the participant.
state: Current wake state (IDLE or AWAKE).
wake_timer: Timestamp of last wake phrase detection.
accumulator: Accumulated text for wake phrase matching.
"""
def __init__(self, participant_id: str):
"""Initialize participant state.
Args:
participant_id: Unique identifier for the participant.
"""
self.participant_id = participant_id
self.state = WakeCheckFilter.WakeState.IDLE
self.wake_timer = 0.0
self.accumulator = ""
def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3):
"""Initialize the wake phrase filter.
Args:
wake_phrases: List of wake phrases to detect in transcriptions.
keepalive_timeout: Duration in seconds to keep passing frames after
wake detection. Defaults to 3 seconds.
"""
super().__init__()
self._participant_states = {}
self._keepalive_timeout = keepalive_timeout
@@ -44,6 +81,12 @@ class WakeCheckFilter(FrameProcessor):
self._wake_patterns.append(pattern)
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames, filtering transcriptions based on wake detection.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
try:

24
src/pipecat/processors/filters/wake_notifier_filter.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Wake notifier filter for conditional frame-based notifications."""
from typing import Awaitable, Callable, Tuple, Type
from pipecat.frames.frames import Frame
@@ -12,10 +14,11 @@ from pipecat.sync.base_notifier import BaseNotifier
class WakeNotifierFilter(FrameProcessor):
"""This processor expects a list of frame types and will execute a given
callback predicate when a frame of any of those type is being processed. If
the callback returns true the notifier will be notified.
"""Frame processor that conditionally triggers notifications based on frame types and filters.
This processor monitors frames of specified types and executes a callback predicate
when such frames are processed. If the callback returns True, the associated
notifier is triggered, allowing for conditional wake-up or notification scenarios.
"""
def __init__(
@@ -26,12 +29,27 @@ class WakeNotifierFilter(FrameProcessor):
filter: Callable[[Frame], Awaitable[bool]],
**kwargs,
):
"""Initialize the wake notifier filter.
Args:
notifier: The notifier to trigger when conditions are met.
types: Tuple of frame types to monitor for potential notifications.
filter: Async callback that determines whether to trigger notification.
Should return True to trigger notification, False otherwise.
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs)
self._notifier = notifier
self._types = types
self._filter = filter
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and conditionally trigger notifications.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, self._types) and await self._filter(frame):

402
src/pipecat/processors/frame_processor.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Frame processing pipeline infrastructure for Pipecat.
This module provides the core frame processing system that enables building
audio/video processing pipelines. It includes frame processors, pipeline
management, and frame flow control mechanisms.
"""
import asyncio
from dataclasses import dataclass
from enum import Enum
@@ -17,6 +24,10 @@ from pipecat.frames.frames import (
CancelFrame,
ErrorFrame,
Frame,
FrameProcessorPauseFrame,
FrameProcessorPauseUrgentFrame,
FrameProcessorResumeFrame,
FrameProcessorResumeUrgentFrame,
StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
@@ -25,35 +36,84 @@ from pipecat.frames.frames import (
from pipecat.metrics.metrics import LLMTokenUsage, MetricsData
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
from pipecat.utils.asyncio import BaseTaskManager
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.asyncio.watchdog_event import WatchdogEvent
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
from pipecat.utils.base_object import BaseObject
class FrameDirection(Enum):
"""Direction of frame flow in the processing pipeline.
Parameters:
DOWNSTREAM: Frames flowing from input to output.
UPSTREAM: Frames flowing back from output to input.
"""
DOWNSTREAM = 1
UPSTREAM = 2
@dataclass
class FrameProcessorSetup:
"""Configuration parameters for frame processor initialization.
Parameters:
clock: The clock instance for timing operations.
task_manager: The task manager for handling async operations.
observer: Optional observer for monitoring frame processing events.
watchdog_timers_enabled: Whether to enable watchdog timers by default.
"""
clock: BaseClock
task_manager: BaseTaskManager
observer: Optional[BaseObserver] = None
watchdog_timers_enabled: bool = False
class FrameProcessor(BaseObject):
"""Base class for all frame processors in the pipeline.
Frame processors are the building blocks of Pipecat pipelines. They receive
frames, process them, and pass them to the next processor in the chain.
Each processor runs in its own task and can be linked to form complex
processing pipelines.
"""
def __init__(
self,
*,
name: Optional[str] = None,
enable_watchdog_logging: Optional[bool] = None,
enable_watchdog_timers: Optional[bool] = None,
metrics: Optional[FrameProcessorMetrics] = None,
watchdog_timeout_secs: Optional[float] = None,
**kwargs,
):
"""Initialize the frame processor.
Args:
name: Optional name for this processor instance.
enable_watchdog_logging: Whether to enable watchdog logging for tasks.
enable_watchdog_timers: Whether to enable watchdog timers for tasks.
metrics: Optional metrics collector for this processor.
watchdog_timeout_secs: Timeout in seconds for watchdog operations.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(name=name)
self._parent: Optional["FrameProcessor"] = None
self._prev: Optional["FrameProcessor"] = None
self._next: Optional["FrameProcessor"] = None
# Enable watchdog timers for all tasks created by this frame processor.
self._enable_watchdog_timers = enable_watchdog_timers
# Enable watchdog logging for all tasks created by this frame processor.
self._enable_watchdog_logging = enable_watchdog_logging
# Allow this frame processor to control their tasks timeout.
self._watchdog_timeout_secs = watchdog_timeout_secs
# Clock
self._clock: Optional[BaseClock] = None
@@ -89,139 +149,287 @@ class FrameProcessor(BaseObject):
# is called. To resume processing frames we need to call
# `resume_processing_frames()` which will wake up the event.
self.__should_block_frames = False
self.__input_event = asyncio.Event()
self.__input_event = None
self.__input_frame_task: Optional[asyncio.Task] = None
# Every processor in Pipecat should only output frames from a single
# task. This avoid problems like audio overlapping. System frames are the
# exception to this rule. This create this task.
self.__push_frame_task: Optional[asyncio.Task] = None
@property
def id(self) -> int:
"""Get the unique identifier for this processor.
Returns:
The unique integer ID of this processor.
"""
return self._id
@property
def name(self) -> str:
"""Get the name of this processor.
Returns:
The name of this processor instance.
"""
return self._name
@property
def interruptions_allowed(self):
"""Check if interruptions are allowed for this processor.
Returns:
True if interruptions are allowed.
"""
return self._allow_interruptions
@property
def metrics_enabled(self):
"""Check if metrics collection is enabled.
Returns:
True if metrics collection is enabled.
"""
return self._enable_metrics
@property
def usage_metrics_enabled(self):
"""Check if usage metrics collection is enabled.
Returns:
True if usage metrics collection is enabled.
"""
return self._enable_usage_metrics
@property
def report_only_initial_ttfb(self):
"""Check if only initial TTFB should be reported.
Returns:
True if only initial time-to-first-byte should be reported.
"""
return self._report_only_initial_ttfb
@property
def interruption_strategies(self) -> Sequence[BaseInterruptionStrategy]:
"""Get the interruption strategies for this processor.
Returns:
Sequence of interruption strategies.
"""
return self._interruption_strategies
@property
def task_manager(self) -> BaseTaskManager:
"""Get the task manager for this processor.
Returns:
The task manager instance.
Raises:
Exception: If the task manager is not initialized.
"""
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
return self._task_manager
def can_generate_metrics(self) -> bool:
"""Check if this processor can generate metrics.
Returns:
True if this processor can generate metrics.
"""
return False
def set_core_metrics_data(self, data: MetricsData):
"""Set core metrics data for this processor.
Args:
data: The metrics data to set.
"""
self._metrics.set_core_metrics_data(data)
async def start_ttfb_metrics(self):
"""Start time-to-first-byte metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb)
async def stop_ttfb_metrics(self):
"""Stop time-to-first-byte metrics collection and push results."""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_ttfb_metrics()
if frame:
await self.push_frame(frame)
async def start_processing_metrics(self):
"""Start processing metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_processing_metrics()
async def stop_processing_metrics(self):
"""Stop processing metrics collection and push results."""
if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_processing_metrics()
if frame:
await self.push_frame(frame)
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
"""Start LLM usage metrics collection.
Args:
tokens: Token usage information for the LLM.
"""
if self.can_generate_metrics() and self.usage_metrics_enabled:
frame = await self._metrics.start_llm_usage_metrics(tokens)
if frame:
await self.push_frame(frame)
async def start_tts_usage_metrics(self, text: str):
"""Start TTS usage metrics collection.
Args:
text: The text being processed by TTS.
"""
if self.can_generate_metrics() and self.usage_metrics_enabled:
frame = await self._metrics.start_tts_usage_metrics(text)
if frame:
await self.push_frame(frame)
async def stop_all_metrics(self):
"""Stop all active metrics collection."""
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
def create_task(self, coroutine: Coroutine, name: Optional[str] = None) -> asyncio.Task:
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
def create_task(
self,
coroutine: Coroutine,
name: Optional[str] = None,
*,
enable_watchdog_logging: Optional[bool] = None,
enable_watchdog_timers: Optional[bool] = None,
watchdog_timeout_secs: Optional[float] = None,
) -> asyncio.Task:
"""Create a new task managed by this processor.
Args:
coroutine: The coroutine to run in the task.
name: Optional name for the task.
enable_watchdog_logging: Whether to enable watchdog logging.
enable_watchdog_timers: Whether to enable watchdog timers.
watchdog_timeout_secs: Timeout in seconds for watchdog operations.
Returns:
The created asyncio task.
"""
if name:
name = f"{self}::{name}"
else:
name = f"{self}::{coroutine.cr_code.co_name}"
return self._task_manager.create_task(coroutine, name)
return self.task_manager.create_task(
coroutine,
name,
enable_watchdog_logging=(
enable_watchdog_logging
if enable_watchdog_logging
else self._enable_watchdog_logging
),
enable_watchdog_timers=(
enable_watchdog_timers if enable_watchdog_timers else self._enable_watchdog_timers
),
watchdog_timeout=(
watchdog_timeout_secs if watchdog_timeout_secs else self._watchdog_timeout_secs
),
)
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
await self._task_manager.cancel_task(task, timeout)
"""Cancel a task managed by this processor.
Args:
task: The task to cancel.
timeout: Optional timeout for task cancellation.
"""
await self.task_manager.cancel_task(task, timeout)
async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
await self._task_manager.wait_for_task(task, timeout)
"""Wait for a task to complete.
Args:
task: The task to wait for.
timeout: Optional timeout for waiting.
"""
await self.task_manager.wait_for_task(task, timeout)
def reset_watchdog(self):
"""Reset the watchdog timer for the current task."""
self.task_manager.task_reset_watchdog()
async def setup(self, setup: FrameProcessorSetup):
"""Set up the processor with required components.
Args:
setup: Configuration object containing setup parameters.
"""
self._clock = setup.clock
self._task_manager = setup.task_manager
self._observer = setup.observer
self._watchdog_timers_enabled = (
self._enable_watchdog_timers
if self._enable_watchdog_timers
else setup.watchdog_timers_enabled
)
if self._metrics is not None:
await self._metrics.setup(self._task_manager)
async def cleanup(self):
"""Clean up processor resources."""
await super().cleanup()
await self.__cancel_input_task()
await self.__cancel_push_task()
if self._metrics is not None:
await self._metrics.cleanup()
def link(self, processor: "FrameProcessor"):
"""Link this processor to the next processor in the pipeline.
Args:
processor: The processor to link to.
"""
self._next = processor
processor._prev = self
logger.debug(f"Linking {self} -> {self._next}")
def get_event_loop(self) -> asyncio.AbstractEventLoop:
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
return self._task_manager.get_event_loop()
"""Get the event loop used by this processor.
Returns:
The asyncio event loop.
"""
return self.task_manager.get_event_loop()
def set_parent(self, parent: "FrameProcessor"):
"""Set the parent processor for this processor.
Args:
parent: The parent processor.
"""
self._parent = parent
def get_parent(self) -> Optional["FrameProcessor"]:
"""Get the parent processor.
Returns:
The parent processor, or None if no parent is set.
"""
return self._parent
def get_clock(self) -> BaseClock:
"""Get the clock used by this processor.
Returns:
The clock instance.
Raises:
Exception: If the clock is not initialized.
"""
if not self._clock:
raise Exception(f"{self} Clock is still not initialized.")
return self._clock
def get_task_manager(self) -> BaseTaskManager:
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
return self._task_manager
async def queue_frame(
self,
frame: Frame,
@@ -230,6 +438,13 @@ class FrameProcessor(BaseObject):
Callable[["FrameProcessor", Frame, FrameDirection], Awaitable[None]]
] = None,
):
"""Queue a frame for processing.
Args:
frame: The frame to queue.
direction: The direction of frame flow.
callback: Optional callback to call after processing.
"""
# If we are cancelling we don't want to process any other frame.
if self._cancelling:
return
@@ -242,14 +457,23 @@ class FrameProcessor(BaseObject):
await self.__input_queue.put((frame, direction, callback))
async def pause_processing_frames(self):
"""Pause processing of queued frames."""
logger.trace(f"{self}: pausing frame processing")
self.__should_block_frames = True
async def resume_processing_frames(self):
"""Resume processing of queued frames."""
logger.trace(f"{self}: resuming frame processing")
self.__input_event.set()
if self.__input_event:
self.__input_event.set()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
if isinstance(frame, StartFrame):
await self.__start(frame)
elif isinstance(frame, StartInterruptionFrame):
@@ -259,61 +483,100 @@ class FrameProcessor(BaseObject):
self._should_report_ttfb = True
elif isinstance(frame, CancelFrame):
await self.__cancel(frame)
elif isinstance(frame, (FrameProcessorPauseFrame, FrameProcessorPauseUrgentFrame)):
await self.__pause(frame)
elif isinstance(frame, (FrameProcessorResumeFrame, FrameProcessorResumeUrgentFrame)):
await self.__resume(frame)
async def push_error(self, error: ErrorFrame):
"""Push an error frame upstream.
Args:
error: The error frame to push.
"""
await self.push_frame(error, FrameDirection.UPSTREAM)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame to the next processor in the pipeline.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
if not self._check_started(frame):
return
if isinstance(frame, SystemFrame):
await self.__internal_push_frame(frame, direction)
else:
await self.__push_queue.put((frame, direction))
await self.__internal_push_frame(frame, direction)
async def __start(self, frame: StartFrame):
"""Handle the start frame to initialize processor state.
Args:
frame: The start frame containing initialization parameters.
"""
self.__started = True
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
self._interruption_strategies = frame.interruption_strategies
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
self.__create_input_task()
self.__create_push_task()
async def __cancel(self, frame: CancelFrame):
"""Handle the cancel frame to stop processor operation.
Args:
frame: The cancel frame.
"""
self._cancelling = True
await self.__cancel_input_task()
await self.__cancel_push_task()
async def __pause(self, frame: FrameProcessorPauseFrame | FrameProcessorPauseUrgentFrame):
"""Handle pause frame to pause processor operation.
Args:
frame: The pause frame.
"""
if frame.processor.name == self.name:
await self.pause_processing_frames()
async def __resume(self, frame: FrameProcessorResumeFrame | FrameProcessorResumeUrgentFrame):
"""Handle resume frame to resume processor operation.
Args:
frame: The resume frame.
"""
if frame.processor.name == self.name:
await self.resume_processing_frames()
#
# Handle interruptions
#
async def _start_interruption(self):
"""Start handling an interruption by canceling current tasks."""
try:
# Cancel the push frame task. This will stop pushing frames downstream.
await self.__cancel_push_task()
# Cancel the input task. This will stop processing queued frames.
await self.__cancel_input_task()
except Exception as e:
logger.exception(f"Uncaught exception in {self}: {e}")
logger.exception(f"Uncaught exception in {self} when handling _start_interruption: {e}")
await self.push_error(ErrorFrame(str(e)))
raise
# Create a new input queue and task.
self.__create_input_task()
# Create a new output queue and task.
self.__create_push_task()
async def _stop_interruption(self):
"""Stop handling an interruption."""
# Nothing to do right now.
pass
async def __internal_push_frame(self, frame: Frame, direction: FrameDirection):
"""Internal method to push frames to adjacent processors.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
try:
timestamp = self._clock.get_time() if self._clock else 0
if direction == FrameDirection.DOWNSTREAM and self._next:
@@ -344,28 +607,41 @@ class FrameProcessor(BaseObject):
except Exception as e:
logger.exception(f"Uncaught exception in {self}: {e}")
await self.push_error(ErrorFrame(str(e)))
raise
def _check_started(self, frame: Frame):
"""Check if the processor has been started.
Args:
frame: The frame being processed.
Returns:
True if the processor has been started.
"""
if not self.__started:
logger.error(f"{self} Trying to process {frame} but StartFrame not received yet")
return self.__started
def __create_input_task(self):
"""Create the input processing task."""
if not self.__input_frame_task:
self.__should_block_frames = False
if not self.__input_event:
self.__input_event = WatchdogEvent(self.task_manager)
self.__input_event.clear()
self.__input_queue = asyncio.Queue()
self.__input_queue = WatchdogQueue(self.task_manager)
self.__input_frame_task = self.create_task(self.__input_frame_task_handler())
async def __cancel_input_task(self):
"""Cancel the input processing task."""
if self.__input_frame_task:
self.__input_queue.cancel()
await self.cancel_task(self.__input_frame_task)
self.__input_frame_task = None
async def __input_frame_task_handler(self):
"""Handle frames from the input queue."""
while True:
if self.__should_block_frames:
if self.__should_block_frames and self.__input_event:
logger.trace(f"{self}: frame processing paused")
await self.__input_event.wait()
self.__input_event.clear()
@@ -373,28 +649,14 @@ class FrameProcessor(BaseObject):
logger.trace(f"{self}: frame processing resumed")
(frame, direction, callback) = await self.__input_queue.get()
# Process the frame.
await self.process_frame(frame, direction)
# If this frame has an associated callback, call it now.
if callback:
await callback(self, frame, direction)
self.__input_queue.task_done()
def __create_push_task(self):
if not self.__push_frame_task:
self.__push_queue = asyncio.Queue()
self.__push_frame_task = self.create_task(self.__push_frame_task_handler())
async def __cancel_push_task(self):
if self.__push_frame_task:
await self.cancel_task(self.__push_frame_task)
self.__push_frame_task = None
async def __push_frame_task_handler(self):
while True:
(frame, direction) = await self.__push_queue.get()
await self.__internal_push_frame(frame, direction)
self.__push_queue.task_done()
try:
# Process the frame.
await self.process_frame(frame, direction)
# If this frame has an associated callback, call it now.
if callback:
await callback(self, frame, direction)
except Exception as e:
logger.exception(f"{self}: error processing frame: {e}")
await self.push_error(ErrorFrame(str(e)))
finally:
self.__input_queue.task_done()

35
src/pipecat/processors/frameworks/langchain.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Langchain integration processor for Pipecat."""
from typing import Optional, Union
from loguru import logger
@@ -26,16 +28,40 @@ except ModuleNotFoundError as e:
class LangchainProcessor(FrameProcessor):
"""Processor that integrates Langchain runnables with Pipecat's frame pipeline.
This processor takes LLM message frames, extracts the latest user message,
and processes it through a Langchain runnable chain. The response is streamed
back as text frames with appropriate response markers.
"""
def __init__(self, chain: Runnable, transcript_key: str = "input"):
"""Initialize the Langchain processor.
Args:
chain: The Langchain runnable to use for processing messages.
transcript_key: The key to use when passing input to the chain.
"""
super().__init__()
self._chain = chain
self._transcript_key = transcript_key
self._participant_id: Optional[str] = None
def set_participant_id(self, participant_id: str):
"""Set the participant ID for session tracking.
Args:
participant_id: The participant ID to use for session configuration.
"""
self._participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle LLM message frames.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, LLMMessagesFrame):
@@ -50,6 +76,14 @@ class LangchainProcessor(FrameProcessor):
@staticmethod
def __get_token_value(text: Union[str, AIMessageChunk]) -> str:
"""Extract token value from various text types.
Args:
text: The text or message chunk to extract value from.
Returns:
The extracted string value.
"""
match text:
case str():
return text
@@ -59,6 +93,7 @@ class LangchainProcessor(FrameProcessor):
return ""
async def _ainvoke(self, text: str):
"""Invoke the Langchain runnable with the provided text."""
logger.debug(f"Invoking chain with {text}")
await self.push_frame(LLMFullResponseStartFrame())
try:

778
src/pipecat/processors/frameworks/rtvi.py
View File

File diff suppressed because it is too large Load Diff

41
src/pipecat/processors/gstreamer/pipeline_source.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""GStreamer pipeline source integration for Pipecat."""
import asyncio
from typing import Optional
@@ -36,7 +38,24 @@ except ModuleNotFoundError as e:
class GStreamerPipelineSource(FrameProcessor):
"""A frame processor that uses GStreamer pipelines as media sources.
This processor creates and manages GStreamer pipelines to generate audio and video
output frames. It handles pipeline lifecycle, decoding, format conversion, and
frame generation with configurable output parameters.
"""
class OutputParams(BaseModel):
"""Output configuration parameters for GStreamer pipeline.
Parameters:
video_width: Width of output video frames in pixels.
video_height: Height of output video frames in pixels.
audio_sample_rate: Sample rate for audio output. If None, uses frame sample rate.
audio_channels: Number of audio channels for output.
clock_sync: Whether to synchronize output with pipeline clock.
"""
video_width: int = 1280
video_height: int = 720
audio_sample_rate: Optional[int] = None
@@ -44,6 +63,13 @@ class GStreamerPipelineSource(FrameProcessor):
clock_sync: bool = True
def __init__(self, *, pipeline: str, out_params: Optional[OutputParams] = None, **kwargs):
"""Initialize the GStreamer pipeline source.
Args:
pipeline: GStreamer pipeline description string for the source.
out_params: Output configuration parameters. If None, uses defaults.
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs)
self._out_params = out_params or GStreamerPipelineSource.OutputParams()
@@ -67,6 +93,12 @@ class GStreamerPipelineSource(FrameProcessor):
bus.connect("message", self._on_gstreamer_message)
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and manage GStreamer pipeline lifecycle.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction)
# Specific system frames
@@ -92,13 +124,16 @@ class GStreamerPipelineSource(FrameProcessor):
await self.push_frame(frame, direction)
async def _start(self, frame: StartFrame):
"""Start the GStreamer pipeline."""
self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate
self._player.set_state(Gst.State.PLAYING)
async def _stop(self, frame: EndFrame):
"""Stop the GStreamer pipeline."""
self._player.set_state(Gst.State.NULL)
async def _cancel(self, frame: CancelFrame):
"""Cancel the GStreamer pipeline."""
self._player.set_state(Gst.State.NULL)
#
@@ -106,6 +141,7 @@ class GStreamerPipelineSource(FrameProcessor):
#
def _on_gstreamer_message(self, bus: Gst.Bus, message: Gst.Message):
"""Handle GStreamer bus messages."""
t = message.type
if t == Gst.MessageType.ERROR:
err, debug = message.parse_error()
@@ -113,6 +149,7 @@ class GStreamerPipelineSource(FrameProcessor):
return True
def _decodebin_callback(self, decodebin: Gst.Element, pad: Gst.Pad):
"""Handle new pads from decodebin element."""
caps_string = pad.get_current_caps().to_string()
if caps_string.startswith("audio"):
self._decodebin_audio(pad)
@@ -120,6 +157,7 @@ class GStreamerPipelineSource(FrameProcessor):
self._decodebin_video(pad)
def _decodebin_audio(self, pad: Gst.Pad):
"""Set up audio processing pipeline from decoded audio pad."""
queue_audio = Gst.ElementFactory.make("queue", None)
audioconvert = Gst.ElementFactory.make("audioconvert", None)
audioresample = Gst.ElementFactory.make("audioresample", None)
@@ -153,6 +191,7 @@ class GStreamerPipelineSource(FrameProcessor):
pad.link(queue_pad)
def _decodebin_video(self, pad: Gst.Pad):
"""Set up video processing pipeline from decoded video pad."""
queue_video = Gst.ElementFactory.make("queue", None)
videoconvert = Gst.ElementFactory.make("videoconvert", None)
videoscale = Gst.ElementFactory.make("videoscale", None)
@@ -187,6 +226,7 @@ class GStreamerPipelineSource(FrameProcessor):
pad.link(queue_pad)
def _appsink_audio_new_sample(self, appsink: GstApp.AppSink):
"""Handle new audio samples from GStreamer appsink."""
buffer = appsink.pull_sample().get_buffer()
(_, info) = buffer.map(Gst.MapFlags.READ)
frame = OutputAudioRawFrame(
@@ -199,6 +239,7 @@ class GStreamerPipelineSource(FrameProcessor):
return Gst.FlowReturn.OK
def _appsink_video_new_sample(self, appsink: GstApp.AppSink):
"""Handle new video samples from GStreamer appsink."""
buffer = appsink.pull_sample().get_buffer()
(_, info) = buffer.map(Gst.MapFlags.READ)
frame = OutputImageRawFrame(

32
src/pipecat/processors/idle_frame_processor.py
View File

@@ -4,17 +4,22 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Idle frame processor for timeout-based callback execution."""
import asyncio
from typing import Awaitable, Callable, List, Optional
from pipecat.frames.frames import Frame, StartFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.asyncio.watchdog_event import WatchdogEvent
class IdleFrameProcessor(FrameProcessor):
"""This class waits to receive any frame or list of desired frames within a
given timeout. If the timeout is reached before receiving any of those
frames the provided callback will be called.
"""Monitors frame activity and triggers callbacks on timeout.
This processor waits to receive any frame or specific frame types within a
given timeout period. If the timeout is reached before receiving the expected
frames, the provided callback will be executed.
"""
def __init__(
@@ -25,6 +30,16 @@ class IdleFrameProcessor(FrameProcessor):
types: Optional[List[type]] = None,
**kwargs,
):
"""Initialize the idle frame processor.
Args:
callback: Async callback function to execute on timeout. Receives
this processor instance as an argument.
timeout: Timeout duration in seconds before triggering the callback.
types: Optional list of frame types to monitor. If None, monitors
all frames.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._callback = callback
@@ -33,6 +48,12 @@ class IdleFrameProcessor(FrameProcessor):
self._idle_task = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and manage idle timeout monitoring.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
@@ -50,15 +71,18 @@ class IdleFrameProcessor(FrameProcessor):
self._idle_event.set()
async def cleanup(self):
"""Clean up resources and cancel pending tasks."""
if self._idle_task:
await self.cancel_task(self._idle_task)
def _create_idle_task(self):
"""Create and start the idle monitoring task."""
if not self._idle_task:
self._idle_event = asyncio.Event()
self._idle_event = WatchdogEvent(self.task_manager)
self._idle_task = self.create_task(self._idle_task_handler())
async def _idle_task_handler(self):
"""Handle idle timeout monitoring and callback execution."""
while True:
try:
await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)

23
src/pipecat/processors/logger.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Frame logging utilities for debugging and monitoring frame flow in Pipecat pipelines."""
from typing import Optional, Tuple, Type
from loguru import logger
@@ -21,6 +23,13 @@ logger = logger.opt(ansi=True)
class FrameLogger(FrameProcessor):
"""A frame processor that logs frame information for debugging purposes.
This processor intercepts frames passing through the pipeline and logs
their details with configurable formatting and filtering. Useful for
debugging frame flow and understanding pipeline behavior.
"""
def __init__(
self,
prefix="Frame",
@@ -32,12 +41,26 @@ class FrameLogger(FrameProcessor):
TransportMessageFrame,
),
):
"""Initialize the frame logger.
Args:
prefix: Text prefix to add to log messages. Defaults to "Frame".
color: ANSI color code for log message formatting. If None, no coloring is applied.
ignored_frame_types: Tuple of frame types to exclude from logging.
Defaults to common high-frequency frames like audio and speaking frames.
"""
super().__init__()
self._prefix = prefix
self._color = color
self._ignored_frame_types = ignored_frame_types
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process and log frame information.
Args:
frame: The frame to process and potentially log.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if self._ignored_frame_types and not isinstance(frame, self._ignored_frame_types):

97
src/pipecat/processors/metrics/frame_processor_metrics.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Frame processor metrics collection and reporting."""
import time
from typing import Optional
@@ -18,21 +20,59 @@ from pipecat.metrics.metrics import (
TTFBMetricsData,
TTSUsageMetricsData,
)
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject
class FrameProcessorMetrics:
class FrameProcessorMetrics(BaseObject):
"""Metrics collection and reporting for frame processors.
Provides comprehensive metrics tracking for frame processing operations,
including timing measurements, resource usage, and performance analytics.
Supports TTFB tracking, processing duration metrics, and usage statistics
for LLM and TTS operations.
"""
def __init__(self):
"""Initialize the frame processor metrics collector.
Sets up internal state for tracking various metrics including TTFB,
processing times, and usage statistics.
"""
super().__init__()
self._task_manager = None
self._start_ttfb_time = 0
self._start_processing_time = 0
self._last_ttfb_time = 0
self._should_report_ttfb = True
async def setup(self, task_manager: BaseTaskManager):
"""Set up the metrics collector with a task manager.
Args:
task_manager: The task manager for handling async operations.
"""
self._task_manager = task_manager
async def cleanup(self):
"""Clean up metrics collection resources."""
await super().cleanup()
@property
def task_manager(self) -> BaseTaskManager:
"""Get the associated task manager.
Returns:
The task manager instance for async operations.
"""
return self._task_manager
@property
def ttfb(self) -> Optional[float]:
"""Get the current TTFB value in seconds.
Returns:
Optional[float]: The TTFB value in seconds, or None if not measured
The TTFB value in seconds, or None if not measured.
"""
if self._last_ttfb_time > 0:
return self._last_ttfb_time
@@ -44,24 +84,46 @@ class FrameProcessorMetrics:
return None
def _processor_name(self):
"""Get the processor name from core metrics data."""
return self._core_metrics_data.processor
def _model_name(self):
"""Get the model name from core metrics data."""
return self._core_metrics_data.model
def set_core_metrics_data(self, data: MetricsData):
"""Set the core metrics data for this collector.
Args:
data: The core metrics data containing processor and model information.
"""
self._core_metrics_data = data
def set_processor_name(self, name: str):
"""Set the processor name for metrics reporting.
Args:
name: The name of the processor to use in metrics.
"""
self._core_metrics_data = MetricsData(processor=name)
async def start_ttfb_metrics(self, report_only_initial_ttfb):
"""Start measuring time-to-first-byte (TTFB).
Args:
report_only_initial_ttfb: Whether to report only the first TTFB measurement.
"""
if self._should_report_ttfb:
self._start_ttfb_time = time.time()
self._last_ttfb_time = 0
self._should_report_ttfb = not report_only_initial_ttfb
async def stop_ttfb_metrics(self):
"""Stop TTFB measurement and generate metrics frame.
Returns:
MetricsFrame containing TTFB data, or None if not measuring.
"""
if self._start_ttfb_time == 0:
return None
@@ -74,9 +136,15 @@ class FrameProcessorMetrics:
return MetricsFrame(data=[ttfb])
async def start_processing_metrics(self):
"""Start measuring processing time."""
self._start_processing_time = time.time()
async def stop_processing_metrics(self):
"""Stop processing time measurement and generate metrics frame.
Returns:
MetricsFrame containing processing duration data, or None if not measuring.
"""
if self._start_processing_time == 0:
return None
@@ -89,15 +157,34 @@ class FrameProcessorMetrics:
return MetricsFrame(data=[processing])
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
logger.debug(
f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}"
)
"""Record LLM token usage metrics.
Args:
tokens: Token usage information including prompt and completion tokens.
Returns:
MetricsFrame containing LLM usage data.
"""
logstr = f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}"
if tokens.cache_read_input_tokens:
logstr += f", cache read input tokens: {tokens.cache_read_input_tokens}"
if tokens.reasoning_tokens:
logstr += f", reasoning tokens: {tokens.reasoning_tokens}"
logger.debug(logstr)
value = LLMUsageMetricsData(
processor=self._processor_name(), model=self._model_name(), value=tokens
)
return MetricsFrame(data=[value])
async def start_tts_usage_metrics(self, text: str):
"""Record TTS character usage metrics.
Args:
text: The text being processed by TTS.
Returns:
MetricsFrame containing TTS usage data.
"""
characters = TTSUsageMetricsData(
processor=self._processor_name(), model=self._model_name(), value=len(text)
)

81
src/pipecat/processors/metrics/sentry.py
View File

@@ -4,8 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Sentry integration for frame processor metrics."""
from loguru import logger
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
try:
import sentry_sdk
except ModuleNotFoundError as e:
@@ -17,15 +22,59 @@ from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMet
class SentryMetrics(FrameProcessorMetrics):
"""Frame processor metrics integration with Sentry monitoring.
Extends FrameProcessorMetrics to send time-to-first-byte (TTFB) and
processing metrics as Sentry transactions for performance monitoring
and debugging.
"""
def __init__(self):
"""Initialize the Sentry metrics collector.
Sets up internal state for tracking transactions and verifies
Sentry SDK initialization status.
"""
super().__init__()
self._ttfb_metrics_tx = None
self._processing_metrics_tx = None
self._sentry_available = sentry_sdk.is_initialized()
if not self._sentry_available:
logger.warning("Sentry SDK not initialized. Sentry features will be disabled.")
self._sentry_task = None
async def setup(self, task_manager: BaseTaskManager):
"""Setup the Sentry metrics system.
Args:
task_manager: The task manager to use for background operations.
"""
await super().setup(task_manager)
if self._sentry_available:
self._sentry_queue = WatchdogQueue(task_manager)
self._sentry_task = self.task_manager.create_task(
self._sentry_task_handler(), name=f"{self}::_sentry_task_handler"
)
async def cleanup(self):
"""Clean up Sentry resources and flush pending transactions.
Ensures all pending transactions are sent to Sentry before shutdown.
"""
await super().cleanup()
if self._sentry_task:
await self._sentry_queue.put(None)
await self.task_manager.wait_for_task(self._sentry_task)
self._sentry_task = None
logger.trace(f"{self} Flushing Sentry metrics")
sentry_sdk.flush(timeout=5.0)
async def start_ttfb_metrics(self, report_only_initial_ttfb):
"""Start tracking time-to-first-byte metrics.
Args:
report_only_initial_ttfb: Whether to report only the initial TTFB measurement.
"""
await super().start_ttfb_metrics(report_only_initial_ttfb)
if self._should_report_ttfb and self._sentry_available:
@@ -34,16 +83,25 @@ class SentryMetrics(FrameProcessorMetrics):
name=f"TTFB for {self._processor_name()}",
)
logger.debug(
f"Sentry transaction started (ID: {self._ttfb_metrics_tx.span_id} Name: {self._ttfb_metrics_tx.name})"
f"{self} Sentry transaction started (ID: {self._ttfb_metrics_tx.span_id} Name: {self._ttfb_metrics_tx.name})"
)
async def stop_ttfb_metrics(self):
"""Stop tracking time-to-first-byte metrics.
Queues the TTFB transaction for completion and transmission to Sentry.
"""
await super().stop_ttfb_metrics()
if self._sentry_available and self._ttfb_metrics_tx:
self._ttfb_metrics_tx.finish()
await self._sentry_queue.put(self._ttfb_metrics_tx)
self._ttfb_metrics_tx = None
async def start_processing_metrics(self):
"""Start tracking frame processing metrics.
Creates a new Sentry transaction to track processing performance.
"""
await super().start_processing_metrics()
if self._sentry_available:
@@ -52,11 +110,26 @@ class SentryMetrics(FrameProcessorMetrics):
name=f"Processing for {self._processor_name()}",
)
logger.debug(
f"Sentry transaction started (ID: {self._processing_metrics_tx.span_id} Name: {self._processing_metrics_tx.name})"
f"{self} Sentry transaction started (ID: {self._processing_metrics_tx.span_id} Name: {self._processing_metrics_tx.name})"
)
async def stop_processing_metrics(self):
"""Stop tracking frame processing metrics.
Queues the processing transaction for completion and transmission to Sentry.
"""
await super().stop_processing_metrics()
if self._sentry_available and self._processing_metrics_tx:
self._processing_metrics_tx.finish()
await self._sentry_queue.put(self._processing_metrics_tx)
self._processing_metrics_tx = None
async def _sentry_task_handler(self):
"""Background task handler for completing Sentry transactions."""
running = True
while running:
tx = await self._sentry_queue.get()
if tx:
await self.task_manager.get_event_loop().run_in_executor(None, tx.finish)
running = tx is not None
self._sentry_queue.task_done()

48
src/pipecat/processors/producer_processor.py
View File

@@ -4,23 +4,35 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Producer processor for frame filtering and distribution."""
import asyncio
from typing import Awaitable, Callable, List
from pipecat.frames.frames import Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
async def identity_transformer(frame: Frame):
"""Default transformer that returns the frame unchanged.
Args:
frame: The frame to transform.
Returns:
The same frame without modifications.
"""
return frame
class ProducerProcessor(FrameProcessor):
"""This class optionally passes-through received frames and decides if those
frames should be sent to consumers based on a user-defined filter. The
frames can be transformed into a different type of frame before being
sending them to the consumers. More than one consumer can be added.
"""A processor that filters frames and distributes them to multiple consumers.
This processor receives frames, applies a filter to determine which frames
should be sent to consumers (ConsumerProcessor), optionally transforms those
frames, and distributes them to registered consumer queues. It can also pass
frames through to the next processor in the pipeline.
"""
def __init__(
@@ -30,6 +42,16 @@ class ProducerProcessor(FrameProcessor):
transformer: Callable[[Frame], Awaitable[Frame]] = identity_transformer,
passthrough: bool = True,
):
"""Initialize the producer processor.
Args:
filter: Async function that determines if a frame should be produced.
Must return True for frames to be sent to consumers.
transformer: Async function to transform frames before sending to consumers.
Defaults to identity_transformer which returns frames unchanged.
passthrough: Whether to pass frames through to the next processor.
If True, all frames continue downstream regardless of filter result.
"""
super().__init__()
self._filter = filter
self._transformer = transformer
@@ -37,26 +59,25 @@ class ProducerProcessor(FrameProcessor):
self._consumers: List[asyncio.Queue] = []
def add_consumer(self):
"""
Adds a new consumer and returns its associated queue.
"""Add a new consumer and return its associated queue.
Returns:
asyncio.Queue: The queue for the newly added consumer.
"""
queue = asyncio.Queue()
queue = WatchdogQueue(self.task_manager)
self._consumers.append(queue)
return queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""
Processes an incoming frame and determines whether to produce it as a ProducerItem.
"""Process an incoming frame and determine whether to produce it.
If the frame meets the produce criteria, it will be added to the consumer queues.
If passthrough is enabled, the frame will also be sent to consumers.
If the frame meets the filter criteria, it will be transformed and added
to all consumer queues. If passthrough is enabled, the original frame
will also be sent downstream.
Args:
frame (Frame): The frame to process.
direction (FrameDirection): The direction of the frame.
frame: The frame to process.
direction: The direction of the frame flow.
"""
await super().process_frame(frame, direction)
@@ -68,6 +89,7 @@ class ProducerProcessor(FrameProcessor):
await self.push_frame(frame, direction)
async def _produce(self, frame: Frame):
"""Produce a frame to all consumers."""
for consumer in self._consumers:
new_frame = await self._transformer(frame)
await consumer.put(new_frame)

32
src/pipecat/processors/text_transformer.py
View File

@@ -4,29 +4,41 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Coroutine
"""Stateless text transformation processor for Pipecat."""
from typing import Callable, Coroutine, Union
from pipecat.frames.frames import Frame, TextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class StatelessTextTransformer(FrameProcessor):
"""This processor calls the given function on any text in a text frame.
"""Processor that applies transformation functions to text frames.
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... print(frame.text)
>>> aggregator = StatelessTextTransformer(lambda x: x.upper())
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
HELLO
This processor intercepts TextFrame objects and applies a user-provided
transformation function to the text content. The function can be either
synchronous or asynchronous (coroutine).
"""
def __init__(self, transform_fn):
def __init__(
self, transform_fn: Union[Callable[[str], str], Callable[[str], Coroutine[None, None, str]]]
):
"""Initialize the text transformer.
Args:
transform_fn: Function to apply to text content. Can be synchronous
(str -> str) or asynchronous (str -> Coroutine[None, None, str]).
"""
super().__init__()
self._transform_fn = transform_fn
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames, applying transformation to text frames.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame):

71
src/pipecat/processors/transcript_processor.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Transcript processing utilities for conversation recording and analysis.
This module provides processors that convert speech and text frames into structured
transcript messages with timestamps, enabling conversation history tracking and analysis.
"""
from typing import List, Optional
from loguru import logger
@@ -30,7 +36,11 @@ class BaseTranscriptProcessor(FrameProcessor):
"""
def __init__(self, **kwargs):
"""Initialize processor with empty message store."""
"""Initialize processor with empty message store.
Args:
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._processed_messages: List[TranscriptionMessage] = []
self._register_event_handler("on_transcript_update")
@@ -39,7 +49,7 @@ class BaseTranscriptProcessor(FrameProcessor):
"""Emit transcript updates for new messages.
Args:
messages: New messages to emit in update
messages: New messages to emit in update.
"""
if messages:
self._processed_messages.extend(messages)
@@ -55,8 +65,8 @@ class UserTranscriptProcessor(BaseTranscriptProcessor):
"""Process TranscriptionFrames into user conversation messages.
Args:
frame: Input frame to process
direction: Frame processing direction
frame: Input frame to process.
direction: Frame processing direction.
"""
await super().process_frame(frame, direction)
@@ -74,17 +84,18 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
This processor aggregates TTS text frames into complete utterances and emits them as
transcript messages. Utterances are completed when:
- The bot stops speaking (BotStoppedSpeakingFrame)
- The bot is interrupted (StartInterruptionFrame)
- The pipeline ends (EndFrame)
Attributes:
_current_text_parts: List of text fragments being aggregated for current utterance
_aggregation_start_time: Timestamp when the current utterance began
"""
def __init__(self, **kwargs):
"""Initialize processor with aggregation state."""
"""Initialize processor with aggregation state.
Args:
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._current_text_parts: List[str] = []
self._aggregation_start_time: Optional[str] = None
@@ -98,34 +109,34 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
TTS services with different formatting patterns.
Examples:
Fragments with embedded spacing (concatenated):
```
Fragments with embedded spacing (concatenated)::
TTSTextFrame: ["Hello"]
TTSTextFrame: [" there"] # Leading space
TTSTextFrame: ["!"]
TTSTextFrame: [" How"] # Leading space
TTSTextFrame: ["'s"]
TTSTextFrame: [" it"] # Leading space
```
Result: "Hello there! How's it"
Fragments with trailing spaces (concatenated):
```
Fragments with trailing spaces (concatenated)::
TTSTextFrame: ["Hel"]
TTSTextFrame: ["lo "] # Trailing space
TTSTextFrame: ["to "] # Trailing space
TTSTextFrame: ["you"]
```
Result: "Hello to you"
Word-by-word fragments without spacing (joined with spaces):
```
Word-by-word fragments without spacing (joined with spaces)::
TTSTextFrame: ["Hello"]
TTSTextFrame: ["there"]
TTSTextFrame: ["how"]
TTSTextFrame: ["are"]
TTSTextFrame: ["you"]
```
Result: "Hello there how are you"
"""
if self._current_text_parts and self._aggregation_start_time:
@@ -169,6 +180,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
"""Process frames into assistant conversation messages.
Handles different frame types:
- TTSTextFrame: Aggregates text for current utterance
- BotStoppedSpeakingFrame: Completes current utterance
- StartInterruptionFrame: Completes current utterance due to interruption
@@ -176,8 +188,8 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
- CancelFrame: Completes current utterance due to cancellation
Args:
frame: Input frame to process
direction: Frame processing direction
frame: Input frame to process.
direction: Frame processing direction.
"""
await super().process_frame(frame, direction)
@@ -211,8 +223,8 @@ class TranscriptProcessor:
Provides unified access to user and assistant transcript processors
with shared event handling.
Example:
```python
Example::
transcript = TranscriptProcessor()
pipeline = Pipeline(
@@ -232,7 +244,6 @@ class TranscriptProcessor:
@transcript.event_handler("on_transcript_update")
async def handle_update(processor, frame):
print(f"New messages: {frame.messages}")
```
"""
def __init__(self):
@@ -245,7 +256,10 @@ class TranscriptProcessor:
"""Get the user transcript processor.
Args:
**kwargs: Arguments specific to UserTranscriptProcessor
**kwargs: Arguments specific to UserTranscriptProcessor.
Returns:
The user transcript processor instance.
"""
if self._user_processor is None:
self._user_processor = UserTranscriptProcessor(**kwargs)
@@ -262,7 +276,10 @@ class TranscriptProcessor:
"""Get the assistant transcript processor.
Args:
**kwargs: Arguments specific to AssistantTranscriptProcessor
**kwargs: Arguments specific to AssistantTranscriptProcessor.
Returns:
The assistant transcript processor instance.
"""
if self._assistant_processor is None:
self._assistant_processor = AssistantTranscriptProcessor(**kwargs)
@@ -279,10 +296,10 @@ class TranscriptProcessor:
"""Register event handler for both processors.
Args:
event_name: Name of event to handle
event_name: Name of event to handle.
Returns:
Decorator function that registers handler with both processors
Decorator function that registers handler with both processors.
"""
def decorator(handler):

53
src/pipecat/processors/user_idle_processor.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""User idle detection and timeout handling for Pipecat."""
import asyncio
import inspect
from typing import Awaitable, Callable, Union
@@ -13,28 +15,25 @@ from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
StartFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.asyncio.watchdog_event import WatchdogEvent
class UserIdleProcessor(FrameProcessor):
"""Monitors user inactivity and triggers callbacks after timeout periods.
Starts monitoring only after the first conversation activity (UserStartedSpeaking
or BotSpeaking).
This processor tracks user activity and triggers configurable callbacks when
users become idle. It starts monitoring only after the first conversation
activity and supports both basic and retry-based callback patterns.
Args:
callback: Function to call when user is idle. Can be either:
- Basic callback(processor) -> None
- Retry callback(processor, retry_count) -> bool
Return True to continue monitoring for idle events,
Return False to stop the idle monitoring task
timeout: Seconds to wait before considering user idle
**kwargs: Additional arguments passed to FrameProcessor
Example::
Example:
# Retry callback:
async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool:
if retry_count < 3:
@@ -62,6 +61,16 @@ class UserIdleProcessor(FrameProcessor):
timeout: float,
**kwargs,
):
"""Initialize the user idle processor.
Args:
callback: Function to call when user is idle. Can be either a basic
callback taking only the processor, or a retry callback taking
the processor and retry count. Retry callbacks should return
True to continue monitoring or False to stop.
timeout: Seconds to wait before considering user idle.
**kwargs: Additional arguments passed to FrameProcessor.
"""
super().__init__(**kwargs)
self._callback = self._wrap_callback(callback)
self._timeout = timeout
@@ -69,7 +78,7 @@ class UserIdleProcessor(FrameProcessor):
self._interrupted = False
self._conversation_started = False
self._idle_task = None
self._idle_event = asyncio.Event()
self._idle_event = None
def _wrap_callback(
self,
@@ -107,7 +116,11 @@ class UserIdleProcessor(FrameProcessor):
@property
def retry_count(self) -> int:
"""Returns the current retry count."""
"""Get the current retry count.
Returns:
The number of times the idle callback has been triggered.
"""
return self._retry_count
async def _stop(self) -> None:
@@ -120,11 +133,14 @@ class UserIdleProcessor(FrameProcessor):
"""Processes incoming frames and manages idle monitoring state.
Args:
frame: The frame to process
direction: Direction of the frame flow
frame: The frame to process.
direction: Direction of the frame flow.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
self._idle_event = WatchdogEvent(self.task_manager)
# Check for end frames before processing
if isinstance(frame, (EndFrame, CancelFrame)):
# Stop the idle task, if it exists
@@ -154,6 +170,13 @@ class UserIdleProcessor(FrameProcessor):
self._idle_event.set()
elif isinstance(frame, BotSpeakingFrame):
self._idle_event.set()
elif isinstance(frame, FunctionCallInProgressFrame):
# Function calls can take longer than the timeout, so we want to prevent idle callbacks
self._interrupted = True
self._idle_event.set()
elif isinstance(frame, FunctionCallResultFrame):
self._interrupted = False
self._idle_event.set()
async def cleanup(self) -> None:
"""Cleans up resources when processor is shutting down."""

42
src/pipecat/serializers/base_serializer.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Frame serialization interfaces for Pipecat."""
from abc import ABC, abstractmethod
from enum import Enum
@@ -11,23 +13,63 @@ from pipecat.frames.frames import Frame, StartFrame
class FrameSerializerType(Enum):
"""Enumeration of supported frame serialization formats.
Parameters:
BINARY: Binary serialization format for compact representation.
TEXT: Text-based serialization format for human-readable output.
"""
BINARY = "binary"
TEXT = "text"
class FrameSerializer(ABC):
"""Abstract base class for frame serialization implementations.
Defines the interface for converting frames to/from serialized formats
for transmission or storage. Subclasses must implement serialization
type detection and the core serialize/deserialize methods.
"""
@property
@abstractmethod
def type(self) -> FrameSerializerType:
"""Get the serialization type supported by this serializer.
Returns:
The FrameSerializerType indicating binary or text format.
"""
pass
async def setup(self, frame: StartFrame):
"""Initialize the serializer with startup configuration.
Args:
frame: StartFrame containing initialization parameters.
"""
pass
@abstractmethod
async def serialize(self, frame: Frame) -> str | bytes | None:
"""Convert a frame to its serialized representation.
Args:
frame: The frame to serialize.
Returns:
Serialized frame data as string, bytes, or None if serialization fails.
"""
pass
@abstractmethod
async def deserialize(self, data: str | bytes) -> Frame | None:
"""Convert serialized data back to a frame object.
Args:
data: Serialized frame data as string or bytes.
Returns:
Reconstructed Frame object, or None if deserialization fails.
"""
pass

16
src/pipecat/serializers/exotel.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Exotel Media Streams serializer for Pipecat."""
import base64
import json
from typing import Optional
@@ -11,7 +13,7 @@ from typing import Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import create_default_resampler
from pipecat.audio.utils import create_stream_resampler
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
@@ -33,13 +35,14 @@ class ExotelFrameSerializer(FrameSerializer):
media streams protocol. It supports audio conversion, DTMF events, and automatic
call termination.
Ref Doc for events - https://support.exotel.com/support/solutions/articles/3000108630-working-with-the-stream-and-voicebot-applet
Note: Ref docs for events:
https://support.exotel.com/support/solutions/articles/3000108630-working-with-the-stream-and-voicebot-applet
"""
class InputParams(BaseModel):
"""Configuration parameters for ExotelFrameSerializer.
Attributes:
Parameters:
exotel_sample_rate: Sample rate used by Exotel, defaults to 8000 Hz.
sample_rate: Optional override for pipeline input sample rate.
"""
@@ -64,7 +67,8 @@ class ExotelFrameSerializer(FrameSerializer):
self._exotel_sample_rate = self._params.exotel_sample_rate
self._sample_rate = 0 # Pipeline input rate
self._resampler = create_default_resampler()
self._input_resampler = create_stream_resampler()
self._output_resampler = create_stream_resampler()
@property
def type(self) -> FrameSerializerType:
@@ -101,7 +105,7 @@ class ExotelFrameSerializer(FrameSerializer):
data = frame.audio
# Output: Exotel outputs PCM audio, but we need to resample to match requested sample_rate
serialized_data = await self._resampler.resample(
serialized_data = await self._output_resampler.resample(
data, frame.sample_rate, self._exotel_sample_rate
)
payload = base64.b64encode(serialized_data).decode("ascii")
@@ -135,7 +139,7 @@ class ExotelFrameSerializer(FrameSerializer):
payload_base64 = message["media"]["payload"]
payload = base64.b64decode(payload_base64)
deserialized_data = await self._resampler.resample(
deserialized_data = await self._input_resampler.resample(
payload,
self._exotel_sample_rate,
self._sample_rate,

33
src/pipecat/serializers/livekit.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""LiveKit frame serializer for Pipecat."""
import ctypes
import pickle
@@ -21,11 +23,33 @@ except ModuleNotFoundError as e:
class LivekitFrameSerializer(FrameSerializer):
"""Serializer for converting between Pipecat frames and LiveKit audio frames.
This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects
to LiveKit AudioFrame objects for transmission, and the reverse conversion
for received audio data.
"""
@property
def type(self) -> FrameSerializerType:
"""Get the serializer type.
Returns:
The serializer type indicating binary serialization.
"""
return FrameSerializerType.BINARY
async def serialize(self, frame: Frame) -> str | bytes | None:
"""Serialize a Pipecat frame to LiveKit AudioFrame format.
Args:
frame: The Pipecat frame to serialize. Only OutputAudioRawFrame
instances are supported.
Returns:
Pickled LiveKit AudioFrame bytes if frame is OutputAudioRawFrame,
None otherwise.
"""
if not isinstance(frame, OutputAudioRawFrame):
return None
audio_frame = AudioFrame(
@@ -37,6 +61,15 @@ class LivekitFrameSerializer(FrameSerializer):
return pickle.dumps(audio_frame)
async def deserialize(self, data: str | bytes) -> Frame | None:
"""Deserialize LiveKit AudioFrame data to a Pipecat frame.
Args:
data: Pickled data containing a LiveKit AudioFrame.
Returns:
InputAudioRawFrame containing the deserialized audio data,
or None if deserialization fails.
"""
audio_frame: AudioFrame = pickle.loads(data)["frame"]
return InputAudioRawFrame(
audio=bytes(audio_frame.data),

23
src/pipecat/serializers/plivo.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Plivo WebSocket frame serializer for audio streaming."""
import base64
import json
from typing import Optional
@@ -11,7 +13,7 @@ from typing import Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import create_default_resampler, pcm_to_ulaw, ulaw_to_pcm
from pipecat.audio.utils import create_stream_resampler, pcm_to_ulaw, ulaw_to_pcm
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
@@ -38,22 +40,12 @@ class PlivoFrameSerializer(FrameSerializer):
When auto_hang_up is enabled (default), the serializer will automatically terminate
the Plivo call when an EndFrame or CancelFrame is processed, but requires Plivo
credentials to be provided.
Attributes:
_stream_id: The Plivo Stream ID.
_call_id: The associated Plivo Call ID.
_auth_id: Plivo auth ID for API access.
_auth_token: Plivo authentication token for API access.
_params: Configuration parameters.
_plivo_sample_rate: Sample rate used by Plivo (typically 8kHz).
_sample_rate: Input sample rate for the pipeline.
_resampler: Audio resampler for format conversion.
"""
class InputParams(BaseModel):
"""Configuration parameters for PlivoFrameSerializer.
Attributes:
Parameters:
plivo_sample_rate: Sample rate used by Plivo, defaults to 8000 Hz.
sample_rate: Optional override for pipeline input sample rate.
auto_hang_up: Whether to automatically terminate call on EndFrame.
@@ -89,7 +81,8 @@ class PlivoFrameSerializer(FrameSerializer):
self._plivo_sample_rate = self._params.plivo_sample_rate
self._sample_rate = 0 # Pipeline input rate
self._resampler = create_default_resampler()
self._input_resampler = create_stream_resampler()
self._output_resampler = create_stream_resampler()
self._hangup_attempted = False
@property
@@ -137,7 +130,7 @@ class PlivoFrameSerializer(FrameSerializer):
# Output: Convert PCM at frame's rate to 8kHz μ-law for Plivo
serialized_data = await pcm_to_ulaw(
data, frame.sample_rate, self._plivo_sample_rate, self._resampler
data, frame.sample_rate, self._plivo_sample_rate, self._output_resampler
)
payload = base64.b64encode(serialized_data).decode("utf-8")
answer = {
@@ -232,7 +225,7 @@ class PlivoFrameSerializer(FrameSerializer):
# Input: Convert Plivo's 8kHz μ-law to PCM at pipeline input rate
deserialized_data = await ulaw_to_pcm(
payload, self._plivo_sample_rate, self._sample_rate, self._resampler
payload, self._plivo_sample_rate, self._sample_rate, self._input_resampler
)
audio_frame = InputAudioRawFrame(
audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate

38
src/pipecat/serializers/protobuf.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Protobuf frame serialization for Pipecat."""
import dataclasses
import json
@@ -22,13 +24,25 @@ from pipecat.frames.frames import (
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
# Data class for converting transport messages into Protobuf format.
@dataclasses.dataclass
class MessageFrame:
"""Data class for converting transport messages into Protobuf format.
Parameters:
data: JSON-encoded message data for transport.
"""
data: str
class ProtobufFrameSerializer(FrameSerializer):
"""Serializer for converting Pipecat frames to/from Protocol Buffer format.
Provides efficient binary serialization for frame transport over network
connections. Supports text, audio, transcription, and message frames with
automatic conversion between transport message types.
"""
SERIALIZABLE_TYPES = {
TextFrame: "text",
OutputAudioRawFrame: "audio",
@@ -46,13 +60,27 @@ class ProtobufFrameSerializer(FrameSerializer):
DESERIALIZABLE_FIELDS = {v: k for k, v in DESERIALIZABLE_TYPES.items()}
def __init__(self):
"""Initialize the Protobuf frame serializer."""
pass
@property
def type(self) -> FrameSerializerType:
"""Get the serializer type.
Returns:
FrameSerializerType.BINARY indicating binary serialization format.
"""
return FrameSerializerType.BINARY
async def serialize(self, frame: Frame) -> str | bytes | None:
"""Serialize a frame to Protocol Buffer binary format.
Args:
frame: The frame to serialize.
Returns:
Serialized frame as bytes, or None if frame type is not serializable.
"""
# Wrapping this messages as a JSONFrame to send
if isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
frame = MessageFrame(
@@ -75,6 +103,14 @@ class ProtobufFrameSerializer(FrameSerializer):
return proto_frame.SerializeToString()
async def deserialize(self, data: str | bytes) -> Frame | None:
"""Deserialize Protocol Buffer binary data to a frame.
Args:
data: Binary protobuf data to deserialize.
Returns:
Deserialized frame instance, or None if deserialization fails.
"""
proto = frame_protos.Frame.FromString(data)
which = proto.WhichOneof("frame")
if which not in self.DESERIALIZABLE_FIELDS:

52
src/pipecat/serializers/telnyx.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Telnyx WebSocket frame serializer for Pipecat."""
import base64
import json
from typing import Optional
@@ -14,7 +16,7 @@ from pydantic import BaseModel
from pipecat.audio.utils import (
alaw_to_pcm,
create_default_resampler,
create_stream_resampler,
pcm_to_alaw,
pcm_to_ulaw,
ulaw_to_pcm,
@@ -43,22 +45,12 @@ class TelnyxFrameSerializer(FrameSerializer):
When auto_hang_up is enabled (default), the serializer will automatically terminate
the Telnyx call when an EndFrame or CancelFrame is processed, but requires Telnyx
credentials to be provided.
Attributes:
_stream_id: The Telnyx Stream ID.
_call_control_id: The associated Telnyx Call Control ID.
_api_key: Telnyx API key for API access.
_params: Configuration parameters.
_telnyx_sample_rate: Sample rate used by Telnyx (typically 8kHz).
_sample_rate: Input sample rate for the pipeline.
_resampler: Audio resampler for format conversion.
_hangup_attempted: Flag to track if hang-up has been attempted.
"""
class InputParams(BaseModel):
"""Configuration parameters for TelnyxFrameSerializer.
Attributes:
Parameters:
telnyx_sample_rate: Sample rate used by Telnyx, defaults to 8000 Hz.
sample_rate: Optional override for pipeline input sample rate.
inbound_encoding: Audio encoding for data sent to Telnyx (e.g., "PCMU").
@@ -101,7 +93,8 @@ class TelnyxFrameSerializer(FrameSerializer):
self._telnyx_sample_rate = self._params.telnyx_sample_rate
self._sample_rate = 0 # Pipeline input rate
self._resampler = create_default_resampler()
self._input_resampler = create_stream_resampler()
self._output_resampler = create_stream_resampler()
self._hangup_attempted = False
@property
@@ -153,11 +146,11 @@ class TelnyxFrameSerializer(FrameSerializer):
# Output: Convert PCM at frame's rate to 8kHz encoded for Telnyx
if self._params.inbound_encoding == "PCMU":
serialized_data = await pcm_to_ulaw(
data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
data, frame.sample_rate, self._telnyx_sample_rate, self._output_resampler
)
elif self._params.inbound_encoding == "PCMA":
serialized_data = await pcm_to_alaw(
data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
data, frame.sample_rate, self._telnyx_sample_rate, self._output_resampler
)
else:
raise ValueError(f"Unsupported encoding: {self._params.inbound_encoding}")
@@ -196,8 +189,31 @@ class TelnyxFrameSerializer(FrameSerializer):
async with session.post(endpoint, headers=headers) as response:
if response.status == 200:
logger.info(f"Successfully terminated Telnyx call {call_control_id}")
elif response.status == 422:
# Handle the case where the call has already ended
# Error code 90018: "Call has already ended"
# Source: https://developers.telnyx.com/api/errors/90018
try:
error_data = await response.json()
if any(
error.get("code") == "90018"
for error in error_data.get("errors", [])
):
logger.debug(
f"Telnyx call {call_control_id} was already terminated"
)
return
except:
pass # Fall through to log the raw error
# Log other 422 errors
error_text = await response.text()
logger.error(
f"Failed to terminate Telnyx call {call_control_id}: "
f"Status {response.status}, Response: {error_text}"
)
else:
# Get the error details for better debugging
# Log other errors
error_text = await response.text()
logger.error(
f"Failed to terminate Telnyx call {call_control_id}: "
@@ -234,14 +250,14 @@ class TelnyxFrameSerializer(FrameSerializer):
payload,
self._telnyx_sample_rate,
self._sample_rate,
self._resampler,
self._input_resampler,
)
elif self._params.outbound_encoding == "PCMA":
deserialized_data = await alaw_to_pcm(
payload,
self._telnyx_sample_rate,
self._sample_rate,
self._resampler,
self._input_resampler,
)
else:
raise ValueError(f"Unsupported encoding: {self._params.outbound_encoding}")

43
src/pipecat/serializers/twilio.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Twilio Media Streams WebSocket protocol serializer for Pipecat."""
import base64
import json
from typing import Optional
@@ -11,7 +13,7 @@ from typing import Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import create_default_resampler, pcm_to_ulaw, ulaw_to_pcm
from pipecat.audio.utils import create_stream_resampler, pcm_to_ulaw, ulaw_to_pcm
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
@@ -38,22 +40,12 @@ class TwilioFrameSerializer(FrameSerializer):
When auto_hang_up is enabled (default), the serializer will automatically terminate
the Twilio call when an EndFrame or CancelFrame is processed, but requires Twilio
credentials to be provided.
Attributes:
_stream_sid: The Twilio Media Stream SID.
_call_sid: The associated Twilio Call SID.
_account_sid: Twilio account SID for API access.
_auth_token: Twilio authentication token for API access.
_params: Configuration parameters.
_twilio_sample_rate: Sample rate used by Twilio (typically 8kHz).
_sample_rate: Input sample rate for the pipeline.
_resampler: Audio resampler for format conversion.
"""
class InputParams(BaseModel):
"""Configuration parameters for TwilioFrameSerializer.
Attributes:
Parameters:
twilio_sample_rate: Sample rate used by Twilio, defaults to 8000 Hz.
sample_rate: Optional override for pipeline input sample rate.
auto_hang_up: Whether to automatically terminate call on EndFrame.
@@ -89,7 +81,8 @@ class TwilioFrameSerializer(FrameSerializer):
self._twilio_sample_rate = self._params.twilio_sample_rate
self._sample_rate = 0 # Pipeline input rate
self._resampler = create_default_resampler()
self._input_resampler = create_stream_resampler()
self._output_resampler = create_stream_resampler()
self._hangup_attempted = False
@property
@@ -137,7 +130,7 @@ class TwilioFrameSerializer(FrameSerializer):
# Output: Convert PCM at frame's rate to 8kHz μ-law for Twilio
serialized_data = await pcm_to_ulaw(
data, frame.sample_rate, self._twilio_sample_rate, self._resampler
data, frame.sample_rate, self._twilio_sample_rate, self._output_resampler
)
payload = base64.b64encode(serialized_data).decode("utf-8")
answer = {
@@ -192,8 +185,26 @@ class TwilioFrameSerializer(FrameSerializer):
async with session.post(endpoint, auth=auth, data=params) as response:
if response.status == 200:
logger.info(f"Successfully terminated Twilio call {call_sid}")
elif response.status == 404:
# Handle the case where the call has already ended
# Error code 20404: "The requested resource was not found"
# Source: https://www.twilio.com/docs/errors/20404
try:
error_data = await response.json()
if error_data.get("code") == 20404:
logger.debug(f"Twilio call {call_sid} was already terminated")
return
except:
pass # Fall through to log the raw error
# Log other 404 errors
error_text = await response.text()
logger.error(
f"Failed to terminate Twilio call {call_sid}: "
f"Status {response.status}, Response: {error_text}"
)
else:
# Get the error details for better debugging
# Log other errors
error_text = await response.text()
logger.error(
f"Failed to terminate Twilio call {call_sid}: "
@@ -222,7 +233,7 @@ class TwilioFrameSerializer(FrameSerializer):
# Input: Convert Twilio's 8kHz μ-law to PCM at pipeline input rate
deserialized_data = await ulaw_to_pcm(
payload, self._twilio_sample_rate, self._sample_rate, self._resampler
payload, self._twilio_sample_rate, self._sample_rate, self._input_resampler
)
audio_frame = InputAudioRawFrame(
audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate

70
src/pipecat/services/ai_service.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Base AI service implementation.
Provides the foundation for all AI services in the Pipecat framework, including
model management, settings handling, and frame processing lifecycle methods.
"""
from typing import Any, AsyncGenerator, Dict, Mapping
from loguru import logger
@@ -20,7 +26,20 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class AIService(FrameProcessor):
"""Base class for all AI services.
Provides common functionality for AI services including model management,
settings handling, session properties, and frame processing lifecycle.
Subclasses should implement specific AI functionality while leveraging
this base infrastructure.
"""
def __init__(self, **kwargs):
"""Initialize the AI service.
Args:
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs)
self._model_name: str = ""
self._settings: Dict[str, Any] = {}
@@ -28,19 +47,53 @@ class AIService(FrameProcessor):
@property
def model_name(self) -> str:
"""Get the current model name.
Returns:
The name of the AI model being used.
"""
return self._model_name
def set_model_name(self, model: str):
"""Set the AI model name and update metrics.
Args:
model: The name of the AI model to use.
"""
self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
async def start(self, frame: StartFrame):
"""Start the AI service.
Called when the service should begin processing. Subclasses should
override this method to perform service-specific initialization.
Args:
frame: The start frame containing initialization parameters.
"""
pass
async def stop(self, frame: EndFrame):
"""Stop the AI service.
Called when the service should stop processing. Subclasses should
override this method to perform cleanup operations.
Args:
frame: The end frame.
"""
pass
async def cancel(self, frame: CancelFrame):
"""Cancel the AI service.
Called when the service should cancel all operations. Subclasses should
override this method to handle cancellation logic.
Args:
frame: The cancel frame.
"""
pass
async def _update_settings(self, settings: Mapping[str, Any]):
@@ -87,6 +140,15 @@ class AIService(FrameProcessor):
logger.warning(f"Unknown setting for {self.name} service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and handle service lifecycle.
Automatically handles StartFrame, EndFrame, and CancelFrame by calling
the appropriate lifecycle methods.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
@@ -97,6 +159,14 @@ class AIService(FrameProcessor):
await self.stop(frame)
async def process_generator(self, generator: AsyncGenerator[Frame | None, None]):
"""Process frames from an async generator.
Takes an async generator that yields frames and processes each one,
handling error frames specially by pushing them as errors.
Args:
generator: An async generator that yields Frame objects or None.
"""
async for f in generator:
if f:
if isinstance(f, ErrorFrame):

11
src/pipecat/services/ai_services.py
View File

@@ -4,6 +4,17 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Deprecated AI services module.
This module is deprecated. Import services directly from their respective modules:
- pipecat.services.ai_service
- pipecat.services.image_service
- pipecat.services.llm_service
- pipecat.services.stt_service
- pipecat.services.tts_service
- pipecat.services.vision_service
"""
import sys
from pipecat.services import DeprecatedModuleProxy

314
src/pipecat/services/anthropic/llm.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Anthropic AI service integration for Pipecat.
This module provides LLM services and context management for Anthropic's Claude models,
including support for function calling, vision, and prompt caching features.
"""
import asyncio
import base64
import copy
@@ -46,6 +52,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator
from pipecat.utils.tracing.service_decorators import traced_llm
try:
@@ -58,27 +65,59 @@ except ModuleNotFoundError as e:
@dataclass
class AnthropicContextAggregatorPair:
"""Pair of context aggregators for Anthropic conversations.
Encapsulates both user and assistant context aggregators
to manage conversation flow and message formatting.
Parameters:
_user: The user context aggregator.
_assistant: The assistant context aggregator.
"""
_user: "AnthropicUserContextAggregator"
_assistant: "AnthropicAssistantContextAggregator"
def user(self) -> "AnthropicUserContextAggregator":
"""Get the user context aggregator.
Returns:
The user context aggregator instance.
"""
return self._user
def assistant(self) -> "AnthropicAssistantContextAggregator":
"""Get the assistant context aggregator.
Returns:
The assistant context aggregator instance.
"""
return self._assistant
class AnthropicLLMService(LLMService):
"""This class implements inference with Anthropic's AI models.
"""LLM service for Anthropic's Claude models.
Can provide a custom client via the `client` kwarg, allowing you to
use `AsyncAnthropicBedrock` and `AsyncAnthropicVertex` clients
Provides inference capabilities with Claude models including support for
function calling, vision processing, streaming responses, and prompt caching.
Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex.
"""
# Overriding the default adapter to use the Anthropic one.
adapter_class = AnthropicLLMAdapter
class InputParams(BaseModel):
"""Input parameters for Anthropic model inference.
Parameters:
enable_prompt_caching_beta: Whether to enable beta prompt caching feature.
max_tokens: Maximum tokens to generate. Must be at least 1.
temperature: Sampling temperature between 0.0 and 1.0.
top_k: Top-k sampling parameter.
top_p: Top-p sampling parameter between 0.0 and 1.0.
extra: Additional parameters to pass to the API.
"""
enable_prompt_caching_beta: Optional[bool] = False
max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1)
temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
@@ -95,6 +134,15 @@ class AnthropicLLMService(LLMService):
client=None,
**kwargs,
):
"""Initialize the Anthropic LLM service.
Args:
api_key: Anthropic API key for authentication.
model: Model name to use. Defaults to "claude-sonnet-4-20250514".
params: Optional model parameters for inference.
client: Optional custom Anthropic client instance.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AnthropicLLMService.InputParams()
self._client = client or AsyncAnthropic(
@@ -111,10 +159,20 @@ class AnthropicLLMService(LLMService):
}
def can_generate_metrics(self) -> bool:
"""Check if this service can generate usage metrics.
Returns:
True, as Anthropic provides detailed token usage metrics.
"""
return True
@property
def enable_prompt_caching_beta(self) -> bool:
"""Check if prompt caching beta feature is enabled.
Returns:
True if prompt caching is enabled.
"""
return self._enable_prompt_caching_beta
def create_context_aggregator(
@@ -124,22 +182,19 @@ class AnthropicLLMService(LLMService):
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> AnthropicContextAggregatorPair:
"""Create an instance of AnthropicContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
assistant aggregators can be provided.
"""Create Anthropic-specific context aggregators.
Creates a pair of context aggregators optimized for Anthropic's message format,
including support for function calls, tool usage, and image handling.
Args:
context (OpenAILLMContext): The LLM context.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
context: The LLM context.
user_params: User aggregator parameters.
assistant_params: Assistant aggregator parameters.
Returns:
AnthropicContextAggregatorPair: A pair of context aggregators, one
for the user and one for the assistant, encapsulated in an
AnthropicContextAggregatorPair.
A pair of context aggregators, one for the user and one for the assistant,
encapsulated in an AnthropicContextAggregatorPair.
"""
context.set_llm_adapter(self.get_llm_adapter())
@@ -203,7 +258,7 @@ class AnthropicLLMService(LLMService):
json_accumulator = ""
function_calls = []
async for event in response:
async for event in WatchdogAsyncIterator(response, manager=self.task_manager):
# Aggregate streaming content, create frames, trigger events
if event.type == "content_block_delta":
@@ -307,6 +362,15 @@ class AnthropicLLMService(LLMService):
)
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and route them appropriately.
Handles various frame types including context frames, message frames,
vision frames, and settings updates.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction)
context = None
@@ -358,6 +422,13 @@ class AnthropicLLMService(LLMService):
class AnthropicLLMContext(OpenAILLMContext):
"""LLM context specialized for Anthropic's message format and features.
Extends OpenAILLMContext to handle Anthropic-specific features like
system messages, prompt caching, and message format conversions.
Manages conversation state and message history formatting.
"""
def __init__(
self,
messages: Optional[List[dict]] = None,
@@ -366,6 +437,14 @@ class AnthropicLLMContext(OpenAILLMContext):
*,
system: Union[str, NotGiven] = NOT_GIVEN,
):
"""Initialize the Anthropic LLM context.
Args:
messages: Initial list of conversation messages.
tools: Available function calling tools.
tool_choice: Tool selection preference.
system: System message content.
"""
super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
# For beta prompt caching. This is a counter that tracks the number of turns
@@ -378,6 +457,16 @@ class AnthropicLLMContext(OpenAILLMContext):
@staticmethod
def upgrade_to_anthropic(obj: OpenAILLMContext) -> "AnthropicLLMContext":
"""Upgrade an OpenAI context to Anthropic format.
Converts message format and restructures content for Anthropic compatibility.
Args:
obj: The OpenAI context to upgrade.
Returns:
The upgraded Anthropic context.
"""
logger.debug(f"Upgrading to Anthropic: {obj}")
if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AnthropicLLMContext):
obj.__class__ = AnthropicLLMContext
@@ -386,6 +475,14 @@ class AnthropicLLMContext(OpenAILLMContext):
@classmethod
def from_openai_context(cls, openai_context: OpenAILLMContext):
"""Create Anthropic context from OpenAI context.
Args:
openai_context: The OpenAI context to convert.
Returns:
New Anthropic context with converted messages.
"""
self = cls(
messages=openai_context.messages,
tools=openai_context.tools,
@@ -397,12 +494,28 @@ class AnthropicLLMContext(OpenAILLMContext):
@classmethod
def from_messages(cls, messages: List[dict]) -> "AnthropicLLMContext":
"""Create context from a list of messages.
Args:
messages: List of conversation messages.
Returns:
New Anthropic context with the provided messages.
"""
self = cls(messages=messages)
self._restructure_from_openai_messages()
return self
@classmethod
def from_image_frame(cls, frame: VisionImageRawFrame) -> "AnthropicLLMContext":
"""Create context from a vision image frame.
Args:
frame: The vision image frame to process.
Returns:
New Anthropic context with the image message.
"""
context = cls()
context.add_image_frame_message(
format=frame.format, size=frame.size, image=frame.image, text=frame.text
@@ -410,31 +523,52 @@ class AnthropicLLMContext(OpenAILLMContext):
return context
def set_messages(self, messages: List):
"""Set the messages list and reset cache tracking.
Args:
messages: New list of messages to set.
"""
self.turns_above_cache_threshold = 0
self._messages[:] = messages
self._restructure_from_openai_messages()
# convert a message in Anthropic format into one or more messages in OpenAI format
def to_standard_messages(self, obj):
"""Convert Anthropic message format to standard structured format.
Handles text content and function calls for both user and assistant messages.
Args:
obj: Message in Anthropic format:
{
"role": "user/assistant",
"content": str | [{"type": "text/tool_use/tool_result", ...}]
}
obj: Message in Anthropic format.
Returns:
List of messages in standard format:
[
List of messages in standard format.
Examples:
Input Anthropic format::
{
"role": "user/assistant/tool",
"content": [{"type": "text", "text": str}]
"role": "assistant",
"content": [
{"type": "text", "text": "Hello"},
{"type": "tool_use", "id": "123", "name": "search", "input": {"q": "test"}}
]
}
]
Output standard format::
[
{"role": "assistant", "content": [{"type": "text", "text": "Hello"}]},
{
"role": "assistant",
"tool_calls": [
{
"type": "function",
"id": "123",
"function": {"name": "search", "arguments": '{"q": "test"}'}
}
]
}
]
"""
# todo: image format (?)
# tool_use
@@ -496,23 +630,37 @@ class AnthropicLLMContext(OpenAILLMContext):
Empty text content is converted to "(empty)".
Args:
message: Message in standard format:
{
"role": "user/assistant/tool",
"content": str | [{"type": "text", ...}],
"tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}]
}
message: Message in standard format.
Returns:
Message in Anthropic format:
{
"role": "user/assistant",
"content": str | [
{"type": "text", "text": str} |
{"type": "tool_use", "id": str, "name": str, "input": dict} |
{"type": "tool_result", "tool_use_id": str, "content": str}
]
}
Message in Anthropic format.
Examples:
Input standard format::
{
"role": "assistant",
"tool_calls": [
{
"id": "123",
"function": {"name": "search", "arguments": '{"q": "test"}'}
}
]
}
Output Anthropic format::
{
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": "123",
"name": "search",
"input": {"q": "test"}
}
]
}
"""
# todo: image messages (?)
if message["role"] == "tool":
@@ -555,6 +703,17 @@ class AnthropicLLMContext(OpenAILLMContext):
def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
):
"""Add an image message to the context.
Converts the image to base64 JPEG format and adds it as a user message
with optional accompanying text.
Args:
format: The image format (e.g., 'RGB', 'RGBA').
size: Image dimensions as (width, height).
image: Raw image bytes.
text: Optional text to accompany the image.
"""
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
@@ -575,6 +734,14 @@ class AnthropicLLMContext(OpenAILLMContext):
self.add_message({"role": "user", "content": content})
def add_message(self, message):
"""Add a message to the context, merging with previous message if same role.
Anthropic requires alternating roles, so consecutive messages from the same
role are merged together.
Args:
message: The message to add to the context.
"""
try:
if self.messages:
# Anthropic requires that roles alternate. If this message's role is the same as the
@@ -600,6 +767,14 @@ class AnthropicLLMContext(OpenAILLMContext):
logger.error(f"Error adding message: {e}")
def get_messages_with_cache_control_markers(self) -> List[dict]:
"""Get messages with prompt caching markers applied.
Adds cache control markers to appropriate messages based on the
number of turns above the cache threshold.
Returns:
List of messages with cache control markers added.
"""
try:
messages = copy.deepcopy(self.messages)
if self.turns_above_cache_threshold >= 1 and messages[-1]["role"] == "user":
@@ -667,12 +842,26 @@ class AnthropicLLMContext(OpenAILLMContext):
message["content"] = [{"type": "text", "text": "(empty)"}]
def get_messages_for_persistent_storage(self):
"""Get messages formatted for persistent storage.
Includes system message at the beginning if present.
Returns:
List of messages suitable for storage.
"""
messages = super().get_messages_for_persistent_storage()
if self.system:
messages.insert(0, {"role": "system", "content": self.system})
return messages
def get_messages_for_logging(self) -> str:
"""Get messages formatted for logging with sensitive data redacted.
Replaces image data with placeholder text for cleaner logs.
Returns:
JSON string representation of messages for logging.
"""
msgs = []
for message in self.messages:
msg = copy.deepcopy(message)
@@ -686,6 +875,12 @@ class AnthropicLLMContext(OpenAILLMContext):
class AnthropicUserContextAggregator(LLMUserContextAggregator):
"""Anthropic-specific user context aggregator.
Handles aggregation of user messages for Anthropic LLM services.
Inherits all functionality from the base LLMUserContextAggregator.
"""
pass
@@ -700,7 +895,20 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator):
class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
"""Context aggregator for assistant messages in Anthropic conversations.
Handles function call lifecycle management including in-progress tracking,
result handling, and cancellation for Anthropic's tool use format.
"""
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
"""Handle a function call that is starting.
Creates tool use message and placeholder tool result for tracking.
Args:
frame: Frame containing function call details.
"""
assistant_message = {"role": "assistant", "content": []}
assistant_message["content"].append(
{
@@ -725,6 +933,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
)
async def handle_function_call_result(self, frame: FunctionCallResultFrame):
"""Handle the result of a completed function call.
Updates the tool result with actual return value or completion status.
Args:
frame: Frame containing function call result.
"""
if frame.result:
result = json.dumps(frame.result)
await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
@@ -734,6 +949,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
)
async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
"""Handle cancellation of a function call.
Updates the tool result to indicate cancellation.
Args:
frame: Frame containing function call cancellation details.
"""
await self._update_function_call_result(
frame.function_name, frame.tool_call_id, "CANCELLED"
)
@@ -752,6 +974,14 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
content["content"] = result
async def handle_user_image_frame(self, frame: UserImageRawFrame):
"""Handle a user image frame with function call context.
Marks the associated function call as completed and adds the image
to the conversation context.
Args:
frame: User image frame with request context.
"""
await self._update_function_call_result(
frame.request.function_name, frame.request.tool_call_id, "COMPLETED"
)

68
src/pipecat/services/assemblyai/models.py
View File

@@ -1,10 +1,30 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AssemblyAI WebSocket API message models and connection parameters.
This module defines Pydantic models for handling AssemblyAI's real-time
transcription WebSocket messages and connection configuration.
"""
from typing import List, Literal, Optional
from pydantic import BaseModel, Field
class Word(BaseModel):
"""Represents a single word in a transcription with timing and confidence."""
"""Represents a single word in a transcription with timing and confidence.
Parameters:
start: Start time of the word in milliseconds.
end: End time of the word in milliseconds.
text: The transcribed word text.
confidence: Confidence score for the word (0.0 to 1.0).
word_is_final: Whether this word is finalized and won't change.
"""
start: int
end: int
@@ -14,13 +34,23 @@ class Word(BaseModel):
class BaseMessage(BaseModel):
"""Base class for all AssemblyAI WebSocket messages."""
"""Base class for all AssemblyAI WebSocket messages.
Parameters:
type: The message type identifier.
"""
type: str
class BeginMessage(BaseMessage):
"""Message sent when a new session begins."""
"""Message sent when a new session begins.
Parameters:
type: Always "Begin" for this message type.
id: Unique session identifier.
expires_at: Unix timestamp when the session expires.
"""
type: Literal["Begin"] = "Begin"
id: str
@@ -28,7 +58,17 @@ class BeginMessage(BaseMessage):
class TurnMessage(BaseMessage):
"""Message containing transcription data for a turn of speech."""
"""Message containing transcription data for a turn of speech.
Parameters:
type: Always "Turn" for this message type.
turn_order: Sequential number of this turn in the session.
turn_is_formatted: Whether the transcript has been formatted.
end_of_turn: Whether this marks the end of a speaking turn.
transcript: The transcribed text for this turn.
end_of_turn_confidence: Confidence score for end-of-turn detection.
words: List of individual words with timing and confidence data.
"""
type: Literal["Turn"] = "Turn"
turn_order: int
@@ -40,7 +80,13 @@ class TurnMessage(BaseMessage):
class TerminationMessage(BaseMessage):
"""Message sent when the session is terminated."""
"""Message sent when the session is terminated.
Parameters:
type: Always "Termination" for this message type.
audio_duration_seconds: Total duration of audio processed.
session_duration_seconds: Total duration of the session.
"""
type: Literal["Termination"] = "Termination"
audio_duration_seconds: float
@@ -52,6 +98,18 @@ AnyMessage = BeginMessage | TurnMessage | TerminationMessage
class AssemblyAIConnectionParams(BaseModel):
"""Configuration parameters for AssemblyAI WebSocket connection.
Parameters:
sample_rate: Audio sample rate in Hz. Defaults to 16000.
encoding: Audio encoding format. Defaults to "pcm_s16le".
formatted_finals: Whether to enable transcript formatting. Defaults to True.
word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds.
end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection.
min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn.
max_turn_silence: Maximum silence duration before forcing end-of-turn.
"""
sample_rate: int = 16000
encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le"
formatted_finals: bool = True

65
src/pipecat/services/assemblyai/stt.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AssemblyAI speech-to-text service implementation.
This module provides integration with AssemblyAI's real-time speech-to-text
WebSocket API for streaming audio transcription.
"""
import asyncio
import json
from typing import Any, AsyncGenerator, Dict
@@ -45,6 +51,13 @@ except ModuleNotFoundError as e:
class AssemblyAISTTService(STTService):
"""AssemblyAI real-time speech-to-text service.
Provides real-time speech transcription using AssemblyAI's WebSocket API.
Supports both interim and final transcriptions with configurable parameters
for audio processing and connection management.
"""
def __init__(
self,
*,
@@ -55,6 +68,16 @@ class AssemblyAISTTService(STTService):
vad_force_turn_endpoint: bool = True,
**kwargs,
):
"""Initialize the AssemblyAI STT service.
Args:
api_key: AssemblyAI API key for authentication.
language: Language code for transcription. Defaults to English (Language.EN).
api_endpoint_base_url: WebSocket endpoint URL. Defaults to AssemblyAI's streaming endpoint.
connection_params: Connection configuration parameters. Defaults to AssemblyAIConnectionParams().
vad_force_turn_endpoint: Whether to force turn endpoint on VAD stop. Defaults to True.
**kwargs: Additional arguments passed to parent STTService class.
"""
self._api_key = api_key
self._language = language
self._api_endpoint_base_url = api_endpoint_base_url
@@ -75,22 +98,50 @@ class AssemblyAISTTService(STTService):
self._chunk_size_bytes = 0
def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
Returns:
True if metrics generation is supported.
"""
return True
async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
Args:
frame: Start frame to begin processing.
"""
await super().start(frame)
self._chunk_size_bytes = int(self._chunk_size_ms * self._sample_rate * 2 / 1000)
await self._connect()
async def stop(self, frame: EndFrame):
"""Stop the speech-to-text service.
Args:
frame: End frame to stop processing.
"""
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
"""Cancel the speech-to-text service.
Args:
frame: Cancel frame to abort processing.
"""
await super().cancel(frame)
await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text conversion.
Args:
audio: Raw audio bytes to process.
Yields:
None (processing handled via WebSocket messages).
"""
self._audio_buffer.extend(audio)
while len(self._audio_buffer) >= self._chunk_size_bytes:
@@ -101,6 +152,12 @@ class AssemblyAISTTService(STTService):
yield None
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for VAD and metrics handling.
Args:
frame: Frame to process.
direction: Direction of frame processing.
"""
await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame):
await self.start_ttfb_metrics()
@@ -189,9 +246,11 @@ class AssemblyAISTTService(STTService):
try:
while self._connected:
try:
message = await self._websocket.recv()
message = await asyncio.wait_for(self._websocket.recv(), timeout=1.0)
data = json.loads(message)
await self._handle_message(data)
except asyncio.TimeoutError:
self.reset_watchdog()
except websockets.exceptions.ConnectionClosedOK:
break
except Exception as e:
@@ -252,7 +311,7 @@ class AssemblyAISTTService(STTService):
await self.push_frame(
TranscriptionFrame(
message.transcript,
"", # participant
self._user_id,
time_now_iso8601(),
self._language,
message,
@@ -264,7 +323,7 @@ class AssemblyAISTTService(STTService):
await self.push_frame(
InterimTranscriptionFrame(
message.transcript,
"", # participant
self._user_id,
time_now_iso8601(),
self._language,
message,

300
src/pipecat/services/aws/llm.py
View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AWS Bedrock integration for Large Language Model services.
This module provides AWS Bedrock LLM service implementation with support for
Amazon Nova and Anthropic Claude models, including vision capabilities and
function calling.
"""
import asyncio
import base64
import copy
@@ -61,17 +68,44 @@ except ModuleNotFoundError as e:
@dataclass
class AWSBedrockContextAggregatorPair:
"""Container for AWS Bedrock context aggregators.
Provides convenient access to both user and assistant context aggregators
for AWS Bedrock LLM operations.
Parameters:
_user: The user context aggregator instance.
_assistant: The assistant context aggregator instance.
"""
_user: "AWSBedrockUserContextAggregator"
_assistant: "AWSBedrockAssistantContextAggregator"
def user(self) -> "AWSBedrockUserContextAggregator":
"""Get the user context aggregator.
Returns:
The user context aggregator instance.
"""
return self._user
def assistant(self) -> "AWSBedrockAssistantContextAggregator":
"""Get the assistant context aggregator.
Returns:
The assistant context aggregator instance.
"""
return self._assistant
class AWSBedrockLLMContext(OpenAILLMContext):
"""AWS Bedrock-specific LLM context implementation.
Extends OpenAI LLM context to handle AWS Bedrock's specific message format
and system message handling. Manages conversion between OpenAI and Bedrock
message formats.
"""
def __init__(
self,
messages: Optional[List[dict]] = None,
@@ -80,11 +114,27 @@ class AWSBedrockLLMContext(OpenAILLMContext):
*,
system: Optional[str] = None,
):
"""Initialize AWS Bedrock LLM context.
Args:
messages: List of conversation messages in OpenAI format.
tools: List of available function calling tools.
tool_choice: Tool selection strategy or specific tool choice.
system: System message content for AWS Bedrock.
"""
super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
self.system = system
@staticmethod
def upgrade_to_bedrock(obj: OpenAILLMContext) -> "AWSBedrockLLMContext":
"""Upgrade an OpenAI LLM context to AWS Bedrock format.
Args:
obj: The OpenAI LLM context to upgrade.
Returns:
The upgraded AWS Bedrock LLM context.
"""
logger.debug(f"Upgrading to AWS Bedrock: {obj}")
if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSBedrockLLMContext):
obj.__class__ = AWSBedrockLLMContext
@@ -95,6 +145,14 @@ class AWSBedrockLLMContext(OpenAILLMContext):
@classmethod
def from_openai_context(cls, openai_context: OpenAILLMContext):
"""Create AWS Bedrock context from OpenAI context.
Args:
openai_context: The OpenAI LLM context to convert.
Returns:
New AWS Bedrock LLM context instance.
"""
self = cls(
messages=openai_context.messages,
tools=openai_context.tools,
@@ -106,12 +164,28 @@ class AWSBedrockLLMContext(OpenAILLMContext):
@classmethod
def from_messages(cls, messages: List[dict]) -> "AWSBedrockLLMContext":
"""Create AWS Bedrock context from message list.
Args:
messages: List of messages in OpenAI format.
Returns:
New AWS Bedrock LLM context instance.
"""
self = cls(messages=messages)
self._restructure_from_openai_messages()
return self
@classmethod
def from_image_frame(cls, frame: VisionImageRawFrame) -> "AWSBedrockLLMContext":
"""Create AWS Bedrock context from vision image frame.
Args:
frame: The vision image frame to convert.
Returns:
New AWS Bedrock LLM context instance.
"""
context = cls()
context.add_image_frame_message(
format=frame.format, size=frame.size, image=frame.image, text=frame.text
@@ -119,30 +193,51 @@ class AWSBedrockLLMContext(OpenAILLMContext):
return context
def set_messages(self, messages: List):
"""Set the messages list and restructure for Bedrock format.
Args:
messages: List of messages to set.
"""
self._messages[:] = messages
self._restructure_from_openai_messages()
# convert a message in AWS Bedrock format into one or more messages in OpenAI format
def to_standard_messages(self, obj):
"""Convert AWS Bedrock message format to standard structured format.
Handles text content and function calls for both user and assistant messages.
Args:
obj: Message in AWS Bedrock format:
{
"role": "user/assistant",
"content": [{"text": str} | {"toolUse": {...}} | {"toolResult": {...}}]
}
obj: Message in AWS Bedrock format.
Returns:
List of messages in standard format:
[
List of messages in standard format.
Examples:
AWS Bedrock format input::
{
"role": "user/assistant/tool",
"content": [{"type": "text", "text": str}]
"role": "assistant",
"content": [
{"text": "Hello"},
{"toolUse": {"toolUseId": "123", "name": "search", "input": {"q": "test"}}}
]
}
]
Standard format output::
[
{"role": "assistant", "content": [{"type": "text", "text": "Hello"}]},
{
"role": "assistant",
"tool_calls": [
{
"type": "function",
"id": "123",
"function": {"name": "search", "arguments": '{"q": "test"}'}
}
]
}
]
"""
role = obj.get("role")
content = obj.get("content")
@@ -216,23 +311,38 @@ class AWSBedrockLLMContext(OpenAILLMContext):
Empty text content is converted to "(empty)".
Args:
message: Message in standard format:
{
"role": "user/assistant/tool",
"content": str | [{"type": "text", ...}],
"tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}]
}
message: Message in standard format.
Returns:
Message in AWS Bedrock format:
{
"role": "user/assistant",
"content": [
{"text": str} |
{"toolUse": {"toolUseId": str, "name": str, "input": dict}} |
{"toolResult": {"toolUseId": str, "content": [...], "status": str}}
]
}
Message in AWS Bedrock format.
Examples:
Standard format input::
{
"role": "assistant",
"tool_calls": [
{
"id": "123",
"function": {"name": "search", "arguments": '{"q": "test"}'}
}
]
}
AWS Bedrock format output::
{
"role": "assistant",
"content": [
{
"toolUse": {
"toolUseId": "123",
"name": "search",
"input": {"q": "test"}
}
}
]
}
"""
if message["role"] == "tool":
# Try to parse the content as JSON if it looks like JSON
@@ -295,6 +405,14 @@ class AWSBedrockLLMContext(OpenAILLMContext):
def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
):
"""Add an image message to the context.
Args:
format: The image format (e.g., 'RGB', 'RGBA').
size: The image dimensions as (width, height).
image: The raw image data as bytes.
text: Optional text to accompany the image.
"""
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
@@ -306,6 +424,14 @@ class AWSBedrockLLMContext(OpenAILLMContext):
self.add_message({"role": "user", "content": content})
def add_message(self, message):
"""Add a message to the context, merging with previous message if same role.
AWS Bedrock requires alternating roles, so consecutive messages from the
same role are merged together.
Args:
message: The message to add to the context.
"""
try:
if self.messages:
# AWS Bedrock requires that roles alternate. If this message's
@@ -330,10 +456,10 @@ class AWSBedrockLLMContext(OpenAILLMContext):
logger.error(f"Error adding message: {e}")
def _restructure_from_bedrock_messages(self):
"""Restructure messages in AWS Bedrock format by handling system
messages, merging consecutive messages with the same role, and ensuring
proper content formatting.
"""Restructure messages in AWS Bedrock format.
Handles system messages, merging consecutive messages with the same role,
and ensuring proper content formatting.
"""
# Handle system message if present at the beginning
if self.messages and self.messages[0]["role"] == "system":
@@ -416,12 +542,22 @@ class AWSBedrockLLMContext(OpenAILLMContext):
message["content"] = [{"type": "text", "text": "(empty)"}]
def get_messages_for_persistent_storage(self):
"""Get messages formatted for persistent storage.
Returns:
List of messages including system message if present.
"""
messages = super().get_messages_for_persistent_storage()
if self.system:
messages.insert(0, {"role": "system", "content": self.system})
return messages
def get_messages_for_logging(self) -> str:
"""Get messages formatted for logging with sensitive data redacted.
Returns:
JSON string representation of messages with image data redacted.
"""
msgs = []
for message in self.messages:
msg = copy.deepcopy(message)
@@ -435,11 +571,36 @@ class AWSBedrockLLMContext(OpenAILLMContext):
class AWSBedrockUserContextAggregator(LLMUserContextAggregator):
"""User context aggregator for AWS Bedrock LLM service.
Handles aggregation of user messages and frames for AWS Bedrock format.
Inherits all functionality from the base LLM user context aggregator.
Args:
context: The LLM context to aggregate messages into.
params: Configuration parameters for the aggregator.
"""
pass
class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator):
"""Assistant context aggregator for AWS Bedrock LLM service.
Handles aggregation of assistant responses and function calls for AWS Bedrock
format, including tool use and tool result handling.
Args:
context: The LLM context to aggregate messages into.
params: Configuration parameters for the aggregator.
"""
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
"""Handle function call in progress frame.
Args:
frame: The function call in progress frame to handle.
"""
# Format tool use according to AWS Bedrock API
self._context.add_message(
{
@@ -470,6 +631,11 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator):
)
async def handle_function_call_result(self, frame: FunctionCallResultFrame):
"""Handle function call result frame.
Args:
frame: The function call result frame to handle.
"""
if frame.result:
result = json.dumps(frame.result)
await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
@@ -479,6 +645,11 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator):
)
async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
"""Handle function call cancel frame.
Args:
frame: The function call cancel frame to handle.
"""
await self._update_function_call_result(
frame.function_name, frame.tool_call_id, "CANCELLED"
)
@@ -497,6 +668,11 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator):
content["toolResult"]["content"] = [{"text": result}]
async def handle_user_image_frame(self, frame: UserImageRawFrame):
"""Handle user image frame.
Args:
frame: The user image frame to handle.
"""
await self._update_function_call_result(
frame.request.function_name, frame.request.tool_call_id, "COMPLETED"
)
@@ -509,18 +685,28 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator):
class AWSBedrockLLMService(LLMService):
"""This class implements inference with AWS Bedrock models including Amazon
Nova and Anthropic Claude.
Requires AWS credentials to be configured in the environment or through
boto3 configuration.
"""AWS Bedrock Large Language Model service implementation.
Provides inference capabilities for AWS Bedrock models including Amazon Nova
and Anthropic Claude. Supports streaming responses, function calling, and
vision capabilities.
"""
# Overriding the default adapter to use the Anthropic one.
adapter_class = AWSBedrockLLMAdapter
class InputParams(BaseModel):
"""Input parameters for AWS Bedrock LLM service.
Parameters:
max_tokens: Maximum number of tokens to generate.
temperature: Sampling temperature between 0.0 and 1.0.
top_p: Nucleus sampling parameter between 0.0 and 1.0.
stop_sequences: List of strings that stop generation.
latency: Performance mode - "standard" or "optimized".
additional_model_request_fields: Additional model-specific parameters.
"""
max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1)
temperature: Optional[float] = Field(default_factory=lambda: 0.7, ge=0.0, le=1.0)
top_p: Optional[float] = Field(default_factory=lambda: 0.999, ge=0.0, le=1.0)
@@ -540,6 +726,18 @@ class AWSBedrockLLMService(LLMService):
client_config: Optional[Config] = None,
**kwargs,
):
"""Initialize the AWS Bedrock LLM service.
Args:
model: The AWS Bedrock model identifier to use.
aws_access_key: AWS access key ID. If None, uses default credentials.
aws_secret_key: AWS secret access key. If None, uses default credentials.
aws_session_token: AWS session token for temporary credentials.
aws_region: AWS region for the Bedrock service.
params: Model parameters and configuration.
client_config: Custom boto3 client configuration.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs)
params = params or AWSBedrockLLMService.InputParams()
@@ -573,6 +771,11 @@ class AWSBedrockLLMService(LLMService):
logger.info(f"Using AWS Bedrock model: {model}")
def can_generate_metrics(self) -> bool:
"""Check if the service can generate usage metrics.
Returns:
True if metrics generation is supported.
"""
return True
def create_context_aggregator(
@@ -582,21 +785,21 @@ class AWSBedrockLLMService(LLMService):
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> AWSBedrockContextAggregatorPair:
"""Create an instance of AWSBedrockContextAggregatorPair from an
OpenAILLMContext. Constructor keyword arguments for both the user and
assistant aggregators can be provided.
"""Create AWS Bedrock-specific context aggregators.
Creates a pair of context aggregators optimized for AWS Bedrocks's message
format, including support for function calls, tool usage, and image handling.
Args:
context (OpenAILLMContext): The LLM context.
user_params (LLMUserAggregatorParams, optional): User aggregator
parameters.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
context: The LLM context to create aggregators for.
user_params: Parameters for user message aggregation.
assistant_params: Parameters for assistant message aggregation.
Returns:
AWSBedrockContextAggregatorPair: A pair of context aggregators, one
for the user and one for the assistant, encapsulated in an
AWSBedrockContextAggregatorPair: A pair of context aggregators, one for
the user and one for the assistant, encapsulated in an
AWSBedrockContextAggregatorPair.
"""
context.set_llm_adapter(self.get_llm_adapter())
@@ -711,6 +914,8 @@ class AWSBedrockLLMService(LLMService):
function_calls = []
for event in response["stream"]:
self.reset_watchdog()
# Handle text content
if "contentBlockDelta" in event:
delta = event["contentBlockDelta"]["delta"]
@@ -762,6 +967,7 @@ class AWSBedrockLLMService(LLMService):
completion_tokens += usage.get("outputTokens", 0)
cache_read_input_tokens += usage.get("cacheReadInputTokens", 0)
cache_creation_input_tokens += usage.get("cacheWriteInputTokens", 0)
await self.run_function_calls(function_calls)
except asyncio.CancelledError:
# If we're interrupted, we won't get a complete usage report. So set our flag to use the
@@ -789,6 +995,12 @@ class AWSBedrockLLMService(LLMService):
)
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle LLM-specific frame types.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction)
context = None

80
src/pipecat/services/aws/stt.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AWS Transcribe Speech-to-Text service implementation.
This module provides a WebSocket-based connection to AWS Transcribe for real-time
speech-to-text transcription with support for multiple languages and audio formats.
"""
import asyncio
import json
import os
@@ -37,6 +43,13 @@ except ModuleNotFoundError as e:
class AWSTranscribeSTTService(STTService):
"""AWS Transcribe Speech-to-Text service using WebSocket streaming.
Provides real-time speech transcription using AWS Transcribe's streaming API.
Supports multiple languages, configurable sample rates, and both interim and
final transcription results.
"""
def __init__(
self,
*,
@@ -48,6 +61,17 @@ class AWSTranscribeSTTService(STTService):
language: Language = Language.EN,
**kwargs,
):
"""Initialize the AWS Transcribe STT service.
Args:
api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable.
aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable.
aws_session_token: AWS session token for temporary credentials. If None, uses AWS_SESSION_TOKEN environment variable.
region: AWS region for the service. Defaults to "us-east-1".
sample_rate: Audio sample rate in Hz. Must be 8000 or 16000. Defaults to 16000.
language: Language for transcription. Defaults to English.
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(**kwargs)
self._settings = {
@@ -79,14 +103,28 @@ class AWSTranscribeSTTService(STTService):
self._receive_task = None
def get_service_encoding(self, encoding: str) -> str:
"""Convert internal encoding format to AWS Transcribe format."""
"""Convert internal encoding format to AWS Transcribe format.
Args:
encoding: Internal encoding format string.
Returns:
AWS Transcribe compatible encoding format.
"""
encoding_map = {
"linear16": "pcm", # AWS expects "pcm" for 16-bit linear PCM
}
return encoding_map.get(encoding, encoding)
async def start(self, frame: StartFrame):
"""Initialize the connection when the service starts."""
"""Initialize the connection when the service starts.
Args:
frame: Start frame signaling service initialization.
Raises:
RuntimeError: If WebSocket connection cannot be established after retries.
"""
await super().start(frame)
logger.info("Starting AWS Transcribe service...")
retry_count = 0
@@ -108,15 +146,32 @@ class AWSTranscribeSTTService(STTService):
raise RuntimeError("Failed to establish WebSocket connection after multiple attempts")
async def stop(self, frame: EndFrame):
"""Stop the service and disconnect from AWS Transcribe.
Args:
frame: End frame signaling service shutdown.
"""
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
"""Cancel the service and disconnect from AWS Transcribe.
Args:
frame: Cancel frame signaling service cancellation.
"""
await super().cancel(frame)
await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data and send to AWS Transcribe"""
"""Process audio data and send to AWS Transcribe.
Args:
audio: Raw audio bytes to transcribe.
Yields:
ErrorFrame: If processing fails or connection issues occur.
"""
try:
# Ensure WebSocket is connected
if not self._ws_client or not self._ws_client.open:
@@ -255,7 +310,14 @@ class AWSTranscribeSTTService(STTService):
self._ws_client = None
def language_to_service_language(self, language: Language) -> str | None:
"""Convert internal language enum to AWS Transcribe language code."""
"""Convert internal language enum to AWS Transcribe language code.
Args:
language: Internal language enumeration value.
Returns:
AWS Transcribe compatible language code, or None if unsupported.
"""
language_map = {
Language.EN: "en-US",
Language.ES: "es-US",
@@ -266,6 +328,7 @@ class AWSTranscribeSTTService(STTService):
Language.JA: "ja-JP",
Language.KO: "ko-KR",
Language.ZH: "zh-CN",
Language.PL: "pl-PL",
}
return language_map.get(language)
@@ -283,7 +346,8 @@ class AWSTranscribeSTTService(STTService):
break
try:
response = await self._ws_client.recv()
response = await asyncio.wait_for(self._ws_client.recv(), timeout=1.0)
headers, payload = decode_event(response)
if headers.get(":message-type") == "event":
@@ -302,7 +366,7 @@ class AWSTranscribeSTTService(STTService):
await self.push_frame(
TranscriptionFrame(
transcript,
"",
self._user_id,
time_now_iso8601(),
self._settings["language"],
result=result,
@@ -318,7 +382,7 @@ class AWSTranscribeSTTService(STTService):
await self.push_frame(
InterimTranscriptionFrame(
transcript,
"",
self._user_id,
time_now_iso8601(),
self._settings["language"],
result=result,
@@ -333,6 +397,8 @@ class AWSTranscribeSTTService(STTService):
else:
logger.debug(f"{self} Other message type received: {headers}")
logger.debug(f"{self} Payload: {payload}")
except asyncio.TimeoutError:
self.reset_watchdog()
except websockets.exceptions.ConnectionClosed as e:
logger.error(
f"{self} WebSocket connection closed in receive loop with code {e.code}: {e.reason}"

87
src/pipecat/services/aws/tts.py
View File

@@ -4,14 +4,20 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AWS Polly text-to-speech service implementation.
This module provides integration with Amazon Polly for text-to-speech synthesis,
supporting multiple languages, voices, and SSML features.
"""
import asyncio
import os
from typing import AsyncGenerator, Optional
from typing import AsyncGenerator, List, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import create_default_resampler
from pipecat.audio.utils import create_stream_resampler
from pipecat.frames.frames import (
ErrorFrame,
Frame,
@@ -33,6 +39,14 @@ except ModuleNotFoundError as e:
def language_to_aws_language(language: Language) -> Optional[str]:
"""Convert a Language enum to AWS Polly language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding AWS Polly language code, or None if not supported.
"""
language_map = {
# Arabic
Language.AR: "arb",
@@ -109,12 +123,31 @@ def language_to_aws_language(language: Language) -> Optional[str]:
class AWSPollyTTSService(TTSService):
"""AWS Polly text-to-speech service.
Provides text-to-speech synthesis using Amazon Polly with support for
multiple languages, voices, SSML features, and voice customization
options including prosody controls.
"""
class InputParams(BaseModel):
"""Input parameters for AWS Polly TTS configuration.
Parameters:
engine: TTS engine to use ('standard', 'neural', etc.).
language: Language for synthesis. Defaults to English.
pitch: Voice pitch adjustment (for standard engine only).
rate: Speech rate adjustment.
volume: Voice volume adjustment.
lexicon_names: List of pronunciation lexicons to apply.
"""
engine: Optional[str] = None
language: Optional[Language] = Language.EN
pitch: Optional[str] = None
rate: Optional[str] = None
volume: Optional[str] = None
lexicon_names: Optional[List[str]] = None
def __init__(
self,
@@ -128,6 +161,18 @@ class AWSPollyTTSService(TTSService):
params: Optional[InputParams] = None,
**kwargs,
):
"""Initializes the AWS Polly TTS service.
Args:
api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable.
aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable.
aws_session_token: AWS session token for temporary credentials.
region: AWS region for Polly service. Defaults to 'us-east-1'.
voice_id: Voice ID to use for synthesis. Defaults to 'Joanna'.
sample_rate: Audio sample rate. If None, uses service default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AWSPollyTTSService.InputParams()
@@ -147,9 +192,10 @@ class AWSPollyTTSService(TTSService):
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
"lexicon_names": params.lexicon_names,
}
self._resampler = create_default_resampler()
self._resampler = create_stream_resampler()
self.set_voice(voice_id)
@@ -172,9 +218,22 @@ class AWSPollyTTSService(TTSService):
)
def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as AWS Polly service supports metrics generation.
"""
return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to AWS Polly language format.
Args:
language: The language to convert.
Returns:
The AWS Polly-specific language code, or None if not supported.
"""
return language_to_aws_language(language)
def _construct_ssml(self, text: str) -> str:
@@ -212,6 +271,15 @@ class AWSPollyTTSService(TTSService):
@traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using AWS Polly.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
def read_audio_data(**args):
response = self._polly_client.synthesize_speech(**args)
if "AudioStream" in response:
@@ -235,6 +303,7 @@ class AWSPollyTTSService(TTSService):
"Engine": self._settings["engine"],
# AWS only supports 8000 and 16000 for PCM. We select 16000.
"SampleRate": "16000",
"LexiconNames": self._settings["lexicon_names"],
}
# Filter out None values
@@ -274,7 +343,19 @@ class AWSPollyTTSService(TTSService):
class PollyTTSService(AWSPollyTTSService):
"""Deprecated alias for AWSPollyTTSService.
.. deprecated:: 0.0.67
`PollyTTSService` is deprecated, use `AWSPollyTTSService` instead.
"""
def __init__(self, **kwargs):
"""Initialize the deprecated PollyTTSService.
Args:
**kwargs: All arguments passed to AWSPollyTTSService.
"""
super().__init__(**kwargs)
import warnings

106
src/pipecat/services/aws/utils.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AWS Transcribe utility functions and classes for WebSocket streaming.
This module provides utilities for creating presigned URLs, building event messages,
and handling AWS event stream protocol for real-time transcription services.
"""
import binascii
import datetime
import hashlib
@@ -29,7 +35,29 @@ def get_presigned_url(
show_speaker_label: bool = False,
enable_channel_identification: bool = False,
) -> str:
"""Create a presigned URL for AWS Transcribe streaming."""
"""Create a presigned URL for AWS Transcribe streaming.
Args:
region: AWS region for the service.
credentials: Dictionary containing AWS credentials. Must include
'access_key' and 'secret_key', with optional 'session_token'.
language_code: Language code for transcription (e.g., "en-US").
media_encoding: Audio encoding format. Defaults to "pcm".
sample_rate: Audio sample rate in Hz. Defaults to 16000.
number_of_channels: Number of audio channels. Defaults to 1.
enable_partial_results_stabilization: Whether to enable partial result stabilization.
partial_results_stability: Stability level for partial results.
vocabulary_name: Custom vocabulary name to use.
vocabulary_filter_name: Vocabulary filter name to apply.
show_speaker_label: Whether to include speaker labels.
enable_channel_identification: Whether to enable channel identification.
Returns:
Presigned WebSocket URL for AWS Transcribe streaming.
Raises:
ValueError: If required AWS credentials are missing.
"""
access_key = credentials.get("access_key")
secret_key = credentials.get("secret_key")
session_token = credentials.get("session_token")
@@ -58,9 +86,23 @@ def get_presigned_url(
class AWSTranscribePresignedURL:
"""Generator for AWS Transcribe presigned WebSocket URLs.
Handles AWS Signature Version 4 signing process to create authenticated
WebSocket URLs for streaming transcription requests.
"""
def __init__(
self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1"
):
"""Initialize the presigned URL generator.
Args:
access_key: AWS access key ID.
secret_key: AWS secret access key.
session_token: AWS session token for temporary credentials.
region: AWS region for the service. Defaults to "us-east-1".
"""
self.access_key = access_key
self.secret_key = secret_key
self.session_token = session_token
@@ -96,6 +138,23 @@ class AWSTranscribePresignedURL:
enable_partial_results_stabilization: bool = False,
partial_results_stability: str = "",
) -> str:
"""Generate a presigned WebSocket URL for AWS Transcribe.
Args:
sample_rate: Audio sample rate in Hz.
language_code: Language code for transcription.
media_encoding: Audio encoding format.
vocabulary_name: Custom vocabulary name.
vocabulary_filter_name: Vocabulary filter name.
show_speaker_label: Whether to include speaker labels.
enable_channel_identification: Whether to enable channel identification.
number_of_channels: Number of audio channels.
enable_partial_results_stabilization: Whether to enable partial result stabilization.
partial_results_stability: Stability level for partial results.
Returns:
Presigned WebSocket URL with authentication parameters.
"""
self.endpoint = f"wss://transcribestreaming.{self.region}.amazonaws.com:8443"
self.host = f"transcribestreaming.{self.region}.amazonaws.com:8443"
@@ -172,7 +231,15 @@ class AWSTranscribePresignedURL:
def get_headers(header_name: str, header_value: str) -> bytearray:
"""Build a header following AWS event stream format."""
"""Build a header following AWS event stream format.
Args:
header_name: Name of the header.
header_value: Value of the header.
Returns:
Encoded header as a bytearray following AWS event stream protocol.
"""
name = header_name.encode("utf-8")
name_byte_length = bytes([len(name)])
value_type = bytes([7]) # 7 represents a string
@@ -190,9 +257,21 @@ def get_headers(header_name: str, header_value: str) -> bytearray:
def build_event_message(payload: bytes) -> bytes:
"""
Build an event message for AWS Transcribe streaming.
Matches AWS sample: https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py
"""Build an event message for AWS Transcribe streaming.
Creates a properly formatted AWS event stream message containing audio data
for real-time transcription. Follows the AWS event stream protocol with
prelude, headers, payload, and CRC checksums.
Args:
payload: Raw audio bytes to include in the event message.
Returns:
Complete event message as bytes, ready to send via WebSocket.
Note:
Implementation matches AWS sample:
https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py
"""
# Build headers
content_type_header = get_headers(":content-type", "application/octet-stream")
@@ -235,6 +314,23 @@ def build_event_message(payload: bytes) -> bytes:
def decode_event(message):
"""Decode an AWS event stream message.
Parses an AWS event stream message to extract headers and payload,
verifying CRC checksums for data integrity.
Args:
message: Raw event stream message bytes received from AWS.
Returns:
A tuple of (headers, payload) where:
- headers: Dictionary of parsed headers
- payload: Dictionary of parsed JSON payload
Raises:
AssertionError: If CRC checksum verification fails.
"""
# Extract the prelude, headers, payload and CRC
prelude = message[:8]
total_length, headers_length = struct.unpack(">II", prelude)

139
src/pipecat/services/aws_nova_sonic/aws.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AWS Nova Sonic LLM service implementation for Pipecat AI framework.
This module provides a speech-to-speech LLM service using AWS Nova Sonic, which supports
bidirectional audio streaming, text generation, and function calling capabilities.
"""
import asyncio
import base64
import json
@@ -25,6 +31,7 @@ from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
FunctionCallFromLLM,
InputAudioRawFrame,
InterimTranscriptionFrame,
LLMFullResponseEndFrame,
@@ -55,6 +62,7 @@ from pipecat.services.aws_nova_sonic.context import (
)
from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame
from pipecat.services.llm_service import LLMService
from pipecat.utils.asyncio.watchdog_coroutine import watchdog_coroutine
from pipecat.utils.time import time_now_iso8601
try:
@@ -82,28 +90,55 @@ except ModuleNotFoundError as e:
class AWSNovaSonicUnhandledFunctionException(Exception):
"""Exception raised when the LLM attempts to call an unregistered function."""
pass
class ContentType(Enum):
"""Content types supported by AWS Nova Sonic.
Parameters:
AUDIO: Audio content type.
TEXT: Text content type.
TOOL: Tool content type.
"""
AUDIO = "AUDIO"
TEXT = "TEXT"
TOOL = "TOOL"
class TextStage(Enum):
"""Text generation stages in AWS Nova Sonic responses.
Parameters:
FINAL: Final text that has been fully generated.
SPECULATIVE: Speculative text that is still being generated.
"""
FINAL = "FINAL" # what has been said
SPECULATIVE = "SPECULATIVE" # what's planned to be said
@dataclass
class CurrentContent:
"""Represents content currently being received from AWS Nova Sonic.
Parameters:
type: The type of content (audio, text, or tool).
role: The role generating the content (user, assistant, etc.).
text_stage: The stage of text generation (final or speculative).
text_content: The actual text content if applicable.
"""
type: ContentType
role: Role
text_stage: TextStage # None if not text
text_content: str # starts as None, then fills in if text
def __str__(self):
"""String representation of the current content."""
return (
f"CurrentContent(\n"
f" type={self.type.name},\n"
@@ -114,6 +149,20 @@ class CurrentContent:
class Params(BaseModel):
"""Configuration parameters for AWS Nova Sonic.
Parameters:
input_sample_rate: Audio input sample rate in Hz.
input_sample_size: Audio input sample size in bits.
input_channel_count: Number of input audio channels.
output_sample_rate: Audio output sample rate in Hz.
output_sample_size: Audio output sample size in bits.
output_channel_count: Number of output audio channels.
max_tokens: Maximum number of tokens to generate.
top_p: Nucleus sampling parameter.
temperature: Sampling temperature for text generation.
"""
# Audio input
input_sample_rate: Optional[int] = Field(default=16000)
input_sample_size: Optional[int] = Field(default=16)
@@ -131,6 +180,12 @@ class Params(BaseModel):
class AWSNovaSonicLLMService(LLMService):
"""AWS Nova Sonic speech-to-speech LLM service.
Provides bidirectional audio streaming, real-time transcription, text generation,
and function calling capabilities using AWS Nova Sonic model.
"""
# Override the default adapter to use the AWSNovaSonicLLMAdapter one
adapter_class = AWSNovaSonicLLMAdapter
@@ -139,6 +194,7 @@ class AWSNovaSonicLLMService(LLMService):
*,
secret_access_key: str,
access_key_id: str,
session_token: Optional[str] = None,
region: str,
model: str = "amazon.nova-sonic-v1:0",
voice_id: str = "matthew", # matthew, tiffany, amy
@@ -148,9 +204,25 @@ class AWSNovaSonicLLMService(LLMService):
send_transcription_frames: bool = True,
**kwargs,
):
"""Initializes the AWS Nova Sonic LLM service.
Args:
secret_access_key: AWS secret access key for authentication.
access_key_id: AWS access key ID for authentication.
session_token: AWS session token for authentication.
region: AWS region where the service is hosted.
model: Model identifier. Defaults to "amazon.nova-sonic-v1:0".
voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy.
params: Model parameters for audio configuration and inference.
system_instruction: System-level instruction for the model.
tools: Available tools/functions for the model to use.
send_transcription_frames: Whether to emit transcription frames.
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs)
self._secret_access_key = secret_access_key
self._access_key_id = access_key_id
self._session_token = session_token
self._region = region
self._model = model
self._client: Optional[BedrockRuntimeClient] = None
@@ -187,16 +259,31 @@ class AWSNovaSonicLLMService(LLMService):
#
async def start(self, frame: StartFrame):
"""Start the service and initiate connection to AWS Nova Sonic.
Args:
frame: The start frame triggering service initialization.
"""
await super().start(frame)
self._wants_connection = True
await self._start_connecting()
async def stop(self, frame: EndFrame):
"""Stop the service and close connections.
Args:
frame: The end frame triggering service shutdown.
"""
await super().stop(frame)
self._wants_connection = False
await self._disconnect()
async def cancel(self, frame: CancelFrame):
"""Cancel the service and close connections.
Args:
frame: The cancel frame triggering service cancellation.
"""
await super().cancel(frame)
self._wants_connection = False
await self._disconnect()
@@ -206,6 +293,11 @@ class AWSNovaSonicLLMService(LLMService):
#
async def reset_conversation(self):
"""Reset the conversation state while preserving context.
Handles bot stopped speaking event, disconnects from the service,
and reconnects with the preserved context.
"""
logger.debug("Resetting conversation")
await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False)
@@ -221,6 +313,12 @@ class AWSNovaSonicLLMService(LLMService):
#
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle service-specific logic.
Args:
frame: The frame to process.
direction: The direction the frame is traveling.
"""
await super().process_frame(frame, direction)
if isinstance(frame, OpenAILLMContextFrame):
@@ -428,7 +526,9 @@ class AWSNovaSonicLLMService(LLMService):
region=self._region,
aws_credentials_identity_resolver=StaticCredentialsResolver(
credentials=AWSCredentialsIdentity(
access_key_id=self._access_key_id, secret_access_key=self._secret_access_key
access_key_id=self._access_key_id,
secret_access_key=self._secret_access_key,
session_token=self._session_token,
)
),
http_auth_scheme_resolver=HTTPAuthSchemeResolver(),
@@ -696,7 +796,7 @@ class AWSNovaSonicLLMService(LLMService):
try:
while self._stream and not self._disconnecting:
output = await self._stream.await_output()
result = await output[1].receive()
result = await watchdog_coroutine(output[1].receive(), manager=self.task_manager)
if result.value and result.value.bytes_:
response_data = result.value.bytes_.decode("utf-8")
@@ -725,7 +825,6 @@ class AWSNovaSonicLLMService(LLMService):
elif "completionEnd" in event_json:
# Handle the LLM completion ending
await self._handle_completion_end_event(event_json)
except Exception as e:
logger.error(f"{self} error processing responses: {e}")
if self._wants_connection:
@@ -804,12 +903,16 @@ class AWSNovaSonicLLMService(LLMService):
# Call tool function
if self.has_function(function_name):
if function_name in self._functions.keys() or None in self._functions.keys():
await self.call_function(
context=self._context,
tool_call_id=tool_call_id,
function_name=function_name,
arguments=arguments,
)
function_calls_llm = [
FunctionCallFromLLM(
context=self._context,
tool_call_id=tool_call_id,
function_name=function_name,
arguments=arguments,
)
]
await self.run_function_calls(function_calls_llm)
else:
raise AWSNovaSonicUnhandledFunctionException(
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
@@ -952,6 +1055,16 @@ class AWSNovaSonicLLMService(LLMService):
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> AWSNovaSonicContextAggregatorPair:
"""Create context aggregator pair for managing conversation context.
Args:
context: The OpenAI LLM context to upgrade.
user_params: Parameters for the user context aggregator.
assistant_params: Parameters for the assistant context aggregator.
Returns:
A pair of user and assistant context aggregators.
"""
context.set_llm_adapter(self.get_llm_adapter())
user = AWSNovaSonicUserContextAggregator(context=context, params=user_params)
@@ -970,6 +1083,14 @@ class AWSNovaSonicLLMService(LLMService):
)
async def trigger_assistant_response(self):
"""Trigger an assistant response by sending audio cue.
Sends a pre-recorded "ready" audio trigger to prompt the assistant
to start speaking. This is useful for controlling conversation flow.
Returns:
False if already triggering a response, True otherwise.
"""
if self._triggering_assistant_response:
return False

140
src/pipecat/services/aws_nova_sonic/context.py
View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Context management for AWS Nova Sonic LLM service.
This module provides specialized context aggregators and message handling for AWS Nova Sonic,
including conversation history management and role-specific message processing.
"""
import copy
from dataclasses import dataclass, field
from enum import Enum
@@ -35,6 +41,15 @@ from pipecat.services.openai.llm import (
class Role(Enum):
"""Roles supported in AWS Nova Sonic conversations.
Parameters:
SYSTEM: System-level messages (not used in conversation history).
USER: Messages sent by the user.
ASSISTANT: Messages sent by the assistant.
TOOL: Messages sent by tools (not used in conversation history).
"""
SYSTEM = "SYSTEM"
USER = "USER"
ASSISTANT = "ASSISTANT"
@@ -43,18 +58,45 @@ class Role(Enum):
@dataclass
class AWSNovaSonicConversationHistoryMessage:
"""A single message in AWS Nova Sonic conversation history.
Parameters:
role: The role of the message sender (USER or ASSISTANT only).
text: The text content of the message.
"""
role: Role # only USER and ASSISTANT
text: str
@dataclass
class AWSNovaSonicConversationHistory:
"""Complete conversation history for AWS Nova Sonic initialization.
Parameters:
system_instruction: System-level instruction for the conversation.
messages: List of conversation messages between user and assistant.
"""
system_instruction: str = None
messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list)
class AWSNovaSonicLLMContext(OpenAILLMContext):
"""Specialized LLM context for AWS Nova Sonic service.
Extends OpenAI context with Nova Sonic-specific message handling,
conversation history management, and text buffering capabilities.
"""
def __init__(self, messages=None, tools=None, **kwargs):
"""Initialize AWS Nova Sonic LLM context.
Args:
messages: Initial messages for the context.
tools: Available tools for the context.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(messages=messages, tools=tools, **kwargs)
self.__setup_local()
@@ -67,6 +109,15 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
def upgrade_to_nova_sonic(
obj: OpenAILLMContext, system_instruction: str
) -> "AWSNovaSonicLLMContext":
"""Upgrade an OpenAI context to AWS Nova Sonic context.
Args:
obj: The OpenAI context to upgrade.
system_instruction: System instruction for the context.
Returns:
The upgraded AWS Nova Sonic context.
"""
if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext):
obj.__class__ = AWSNovaSonicLLMContext
obj.__setup_local(system_instruction)
@@ -74,6 +125,14 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
# NOTE: this method has the side-effect of updating _system_instruction from messages
def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory:
"""Get conversation history for initializing AWS Nova Sonic session.
Processes stored messages and extracts system instruction and conversation
history in the format expected by AWS Nova Sonic.
Returns:
Formatted conversation history with system instruction and messages.
"""
history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction)
# Bail if there are no messages
@@ -103,6 +162,11 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
return history
def get_messages_for_persistent_storage(self):
"""Get messages formatted for persistent storage.
Returns:
List of messages including system instruction if present.
"""
messages = super().get_messages_for_persistent_storage()
# If we have a system instruction and messages doesn't already contain it, add it
if self._system_instruction and not (messages and messages[0].get("role") == "system"):
@@ -110,6 +174,14 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
return messages
def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage:
"""Convert standard message format to Nova Sonic format.
Args:
message: Standard message dictionary to convert.
Returns:
Nova Sonic conversation history message, or None if not convertible.
"""
role = message.get("role")
if message.get("role") == "user" or message.get("role") == "assistant":
content = message.get("content")
@@ -131,10 +203,20 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
# Sonic conversation history
def buffer_user_text(self, text):
"""Buffer user text for later flushing to context.
Args:
text: User text to buffer.
"""
self._user_text += f" {text}" if self._user_text else text
# logger.debug(f"User text buffered: {self._user_text}")
def flush_aggregated_user_text(self) -> str:
"""Flush buffered user text to context as a complete message.
Returns:
The flushed user text, or empty string if no text was buffered.
"""
if not self._user_text:
return ""
user_text = self._user_text
@@ -148,10 +230,16 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
return user_text
def buffer_assistant_text(self, text):
"""Buffer assistant text for later flushing to context.
Args:
text: Assistant text to buffer.
"""
self._assistant_text += text
# logger.debug(f"Assistant text buffered: {self._assistant_text}")
def flush_aggregated_assistant_text(self):
"""Flush buffered assistant text to context as a complete message."""
if not self._assistant_text:
return
message = {
@@ -165,13 +253,31 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
@dataclass
class AWSNovaSonicMessagesUpdateFrame(DataFrame):
"""Frame containing updated AWS Nova Sonic context.
Parameters:
context: The updated AWS Nova Sonic LLM context.
"""
context: AWSNovaSonicLLMContext
class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator):
"""Context aggregator for user messages in AWS Nova Sonic conversations.
Extends the OpenAI user context aggregator to emit Nova Sonic-specific
context update frames.
"""
async def process_frame(
self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
):
"""Process frames and emit Nova Sonic-specific context updates.
Args:
frame: The frame to process.
direction: The direction the frame is traveling.
"""
await super().process_frame(frame, direction)
# Parent does not push LLMMessagesUpdateFrame
@@ -180,7 +286,19 @@ class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator):
class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator):
"""Context aggregator for assistant messages in AWS Nova Sonic conversations.
Provides specialized handling for assistant responses and function calls
in AWS Nova Sonic context, with custom frame processing logic.
"""
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with Nova Sonic-specific logic.
Args:
frame: The frame to process.
direction: The direction the frame is traveling.
"""
# HACK: For now, disable the context aggregator by making it just pass through all frames
# that the parent handles (except the function call stuff, which we still need).
# For an explanation of this hack, see
@@ -205,6 +323,11 @@ class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator):
await super().process_frame(frame, direction)
async def handle_function_call_result(self, frame: FunctionCallResultFrame):
"""Handle function call results for AWS Nova Sonic.
Args:
frame: The function call result frame to handle.
"""
await super().handle_function_call_result(frame)
# The standard function callback code path pushes the FunctionCallResultFrame from the LLM
@@ -217,11 +340,28 @@ class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator):
@dataclass
class AWSNovaSonicContextAggregatorPair:
"""Pair of user and assistant context aggregators for AWS Nova Sonic.
Parameters:
_user: The user context aggregator.
_assistant: The assistant context aggregator.
"""
_user: AWSNovaSonicUserContextAggregator
_assistant: AWSNovaSonicAssistantContextAggregator
def user(self) -> AWSNovaSonicUserContextAggregator:
"""Get the user context aggregator.
Returns:
The user context aggregator instance.
"""
return self._user
def assistant(self) -> AWSNovaSonicAssistantContextAggregator:
"""Get the assistant context aggregator.
Returns:
The assistant context aggregator instance.
"""
return self._assistant

Some files were not shown because too many files have changed in this diff Show More