wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/heartbeat-monitor-configurable

Browse Source
This commit is contained in:
OmercohenAviv
2026-03-28 11:57:23 +03:00
parent dccd98ec8a a84c69858e
commit 5fe48da2fb
874 changed files with 100918 additions and 26768 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/pipecat/__init__.py
View File

@@ -1,18 +1,24 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import sys
from importlib.metadata import version
from importlib.metadata import version as lib_version
from loguru import logger
__version__ = version("pipecat-ai")
__version__ = lib_version("pipecat-ai")
logger.info(f"ᓚᘏᗢ Pipecat {__version__} (Python {sys.version}) ᓚᘏᗢ")
def version() -> str:
"""Returns the Pipecat version."""
return __version__
# We replace `asyncio.wait_for()` for `wait_for2.wait_for()` for Python < 3.12.
#
# In Python 3.12, `asyncio.wait_for()` is implemented in terms of

122
src/pipecat/adapters/base_llm_adapter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -11,7 +11,7 @@ adapters that handle tool format conversion and standardization.
"""
from abc import ABC, abstractmethod
from typing import Any, Dict, Generic, List, TypeVar
from typing import Any, Dict, Generic, List, Optional, TypeVar
from loguru import logger
@@ -39,10 +39,16 @@ class BaseLLMAdapter(ABC, Generic[TLLMInvocationParams]):
- Converting standardized tools schema to provider-specific tool formats.
- Extracting messages from the LLM context for the purposes of logging
about the specific provider.
- Resolving conflicts between ``system_instruction`` and initial
system/developer messages in the conversation context.
Subclasses must implement provider-specific conversion logic.
"""
def __init__(self):
"""Initialize the adapter."""
self._warned_system_instruction = False
@property
@abstractmethod
def id_for_llm_specific_messages(self) -> str:
@@ -129,4 +135,114 @@ class BaseLLMAdapter(ABC, Generic[TLLMInvocationParams]):
# Fallback to return the same tools in case they are not in a standard format
return tools
# TODO: we can move the logic to also handle the Messages here
def _extract_initial_system(
self,
messages: list,
*,
system_instruction: Optional[str] = None,
) -> Optional[str]:
"""Extract an initial ``"system"`` message for use as a system instruction.
Only useful for services that expect the system instruction as a
separate parameter, not inline in conversation history (today, all
non-OpenAI services). Does not extract ``"developer"`` messages —
those are converted to ``"user"`` by the adapter's subsequent message
loop, like any other non-system role the provider doesn't support.
Checks ``messages[0]``. If the role is ``"system"``, pops and returns
its content. If extracting would leave the messages list empty
(``len(messages) == 1``), the message is converted to ``"user"``
role instead of being extracted, to prevent sending an empty
conversation history to providers that require at least one
non-system message.
Args:
messages: Message list in standard format (mutated in-place).
system_instruction: The system instruction from service settings
or ``run_inference``. Only used to decide whether to warn
about a conflict in the single-message case.
Returns:
The extracted system message content, or ``None`` if nothing
was extracted.
"""
if not messages:
return None
if messages[0].get("role") != "system":
return None
# Would extracting empty the list? Convert to "user" instead.
if len(messages) == 1:
if system_instruction:
if not self._warned_system_instruction:
self._warned_system_instruction = True
logger.warning(
"Both system_instruction and an initial system message in"
" context are set. Using system_instruction. The context"
" system message is being converted to a user message to"
" avoid sending an empty conversation history."
)
messages[0]["role"] = "user"
return None
# Extract
content = messages[0].get("content", "")
if isinstance(content, list):
# Join text parts for providers that expect a string system instruction
content = " ".join(
part.get("text", "") for part in content if part.get("type") == "text"
)
messages.pop(0)
return content
def _resolve_system_instruction(
self,
system_from_context: Optional[str],
system_instruction: Optional[str],
*,
discard_context_system: bool,
) -> Optional[str]:
"""Resolve conflict between ``system_instruction`` and an extracted context system message.
Args:
system_from_context: Content extracted from an initial ``"system"``
message by :meth:`_extract_initial_system`, or detected
inline (OpenAI adapters).
system_instruction: From service settings or ``run_inference`` param.
discard_context_system: If ``True`` (non-OpenAI adapters), the
context system message is discarded when ``system_instruction``
is also present. If ``False`` (OpenAI adapters), both are kept.
Returns:
The effective system instruction to use, or ``None`` if the system
instruction is already represented in the messages (OpenAI path).
"""
if system_from_context and system_instruction:
if not self._warned_system_instruction:
self._warned_system_instruction = True
if discard_context_system:
logger.warning(
"Both system_instruction and an initial system message"
" in context are set. Using system_instruction."
)
else:
logger.warning(
"Both system_instruction and an initial system message"
" in context are set, which may be unintended. Keeping"
" both, but consider using system_instruction for"
" system-level instructions and developer messages in"
" context for supplementary guidance."
)
if system_instruction:
return system_instruction
if system_from_context:
if discard_context_system:
return system_from_context
else:
# Content is already in messages; nothing to prepend
return None
return None

2
src/pipecat/adapters/schemas/direct_function.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/adapters/schemas/function_schema.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/adapters/schemas/tools_schema.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

111
src/pipecat/adapters/services/anthropic_adapter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -9,7 +9,7 @@
import copy
import json
from dataclasses import dataclass
from typing import Any, Dict, List, TypedDict
from typing import Any, Dict, List, Optional, TypedDict
from anthropic import NOT_GIVEN, NotGiven
from anthropic.types.message_param import MessageParam
@@ -48,24 +48,36 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
return "anthropic"
def get_llm_invocation_params(
self, context: LLMContext, enable_prompt_caching: bool
self,
context: LLMContext,
enable_prompt_caching: bool,
system_instruction: Optional[str] = None,
) -> AnthropicLLMInvocationParams:
"""Get Anthropic-specific LLM invocation parameters from a universal LLM context.
Args:
context: The LLM context containing messages, tools, etc.
enable_prompt_caching: Whether prompt caching should be enabled.
system_instruction: Optional system instruction from service settings
or ``run_inference``.
Returns:
Dictionary of parameters for invoking Anthropic's LLM API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
converted = self._from_universal_context_messages(
self.get_messages(context), system_instruction=system_instruction
)
system = self._resolve_system_instruction(
converted.system if converted.system is not NOT_GIVEN else None,
system_instruction,
discard_context_system=True,
)
return {
"system": messages.system,
"system": system if system is not None else NOT_GIVEN,
"messages": (
self._with_cache_control_markers(messages.messages)
self._with_cache_control_markers(converted.messages)
if enable_prompt_caching
else messages.messages
else converted.messages
),
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools) or [],
@@ -94,6 +106,8 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
for item in msg["content"]:
if item["type"] == "image":
item["source"]["data"] = "..."
if item["type"] == "thinking" and item.get("signature"):
item["signature"] = "..."
messages_for_logging.append(msg)
return messages_for_logging
@@ -105,33 +119,34 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
system: str | NotGiven
def _from_universal_context_messages(
self, universal_context_messages: List[LLMContextMessage]
self,
universal_context_messages: List[LLMContextMessage],
*,
system_instruction: Optional[str] = None,
) -> ConvertedMessages:
system = NOT_GIVEN
messages = []
# First, map messages using self._from_universal_context_message(m)
# Extract initial system message from universal messages BEFORE conversion,
# so the helper works with standard message format (not provider-specific).
remaining = list(universal_context_messages)
if remaining and not isinstance(remaining[0], LLMSpecificMessage):
extracted = self._extract_initial_system(
remaining, system_instruction=system_instruction
)
if extracted is not None:
system = extracted
# Convert remaining messages to Anthropic format
messages = []
try:
messages = [self._from_universal_context_message(m) for m in universal_context_messages]
messages = [self._from_universal_context_message(m) for m in remaining]
except Exception as e:
logger.error(f"Error mapping messages: {e}")
# See if we should pull the system message out of our messages list.
if messages and messages[0]["role"] == "system":
if len(messages) == 1:
# If we have only have a system message in the list, all we can really do
# without introducing too much magic is change the role to "user".
messages[0]["role"] = "user"
else:
# If we have more than one message, we'll pull the system message out of the
# list.
system = messages[0]["content"]
messages.pop(0)
# Convert any subsequent "system"-role messages to "user"-role
# messages, as Anthropic doesn't support system input messages.
# Convert any subsequent "system"/"developer"-role messages to "user"-role
# messages, as Anthropic doesn't support system or developer input messages.
for message in messages:
if message["role"] == "system":
if message["role"] in ("system", "developer"):
message["role"] = "user"
# Merge consecutive messages with the same role.
@@ -165,9 +180,44 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
def _from_universal_context_message(self, message: LLMContextMessage) -> MessageParam:
if isinstance(message, LLMSpecificMessage):
return copy.deepcopy(message.message)
return self._from_anthropic_specific_message(message)
return self._from_standard_message(message)
def _from_anthropic_specific_message(self, message: LLMSpecificMessage) -> MessageParam:
"""Convert LLMSpecificMessage to Anthropic format.
Anthropic-specific messages may either be special thought messages that
need to be handled in a special way, or messages already in Anthropic
format.
Args:
message: Anthropic-specific message.
"""
# Handle special case of thought messages.
# These can be converted to standalone "assistant" messages; later
# these thinking messages will be properly merged into the assistant
# response messages before the context is sent to Anthropic for the
# next turn.
if (
isinstance(message.message, dict)
and message.message.get("type") == "thought"
and (text := message.message.get("text"))
and (signature := message.message.get("signature"))
):
return {
"role": "assistant",
"content": [
{
"type": "thinking",
"thinking": text,
"signature": signature,
}
],
}
# Fall back to assuming that the message is already in Anthropic format
return copy.deepcopy(message.message)
def _from_standard_message(self, message: LLMStandardMessage) -> MessageParam:
"""Convert standard universal context message to Anthropic format.
@@ -246,11 +296,14 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
# handle image_url -> image conversion
if item["type"] == "image_url":
if item["image_url"]["url"].startswith("data:"):
# Extract MIME type from data URL (format: "data:image/jpeg;base64,...")
url = item["image_url"]["url"]
mime_type = url.split(":")[1].split(";")[0]
item["type"] = "image"
item["source"] = {
"type": "base64",
"media_type": "image/jpeg",
"data": item["image_url"]["url"].split(",")[1],
"media_type": mime_type,
"data": url.split(",")[1],
}
del item["image_url"]
elif item["image_url"]["url"].startswith("http"):

26
src/pipecat/adapters/services/aws_nova_sonic_adapter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -72,20 +72,26 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
"""Get the identifier used in LLMSpecificMessage instances for AWS Nova Sonic."""
return "aws-nova-sonic"
def get_llm_invocation_params(self, context: LLMContext) -> AWSNovaSonicLLMInvocationParams:
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> AWSNovaSonicLLMInvocationParams:
"""Get AWS Nova Sonic-specific LLM invocation parameters from a universal LLM context.
This is a placeholder until support for universal LLMContext machinery is added for AWS Nova Sonic.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings.
Returns:
Dictionary of parameters for invoking AWS Nova Sonic's LLM API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
effective_system = self._resolve_system_instruction(
messages.system_instruction,
system_instruction,
discard_context_system=True,
)
return {
"system_instruction": messages.system_instruction,
"system_instruction": effective_system,
"messages": messages.messages,
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools) or [],
@@ -125,7 +131,8 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
universal_context_messages = copy.deepcopy(universal_context_messages)
# If we have a "system" message as our first message, let's pull that out into "instruction"
# If we have a "system" message as our first message,
# pull that out into "instruction"
if universal_context_messages[0].get("role") == "system":
system = universal_context_messages.pop(0)
content = system.get("content")
@@ -136,8 +143,13 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
if system_instruction:
self._system_instruction = system_instruction
# Convert any remaining "system"/"developer" messages to "user",
# as Nova Sonic only supports "user" and "assistant" in history.
for msg in universal_context_messages:
if msg.get("role") in ("system", "developer"):
msg["role"] = "user"
# Process remaining messages to fill out conversation history.
# Nova Sonic supports "user" and "assistant" messages in history.
for universal_context_message in universal_context_messages:
message = self._from_universal_context_message(universal_context_message)
if message:

68
src/pipecat/adapters/services/bedrock_adapter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -10,7 +10,7 @@ import base64
import copy
import json
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, TypedDict
from typing import Any, Dict, List, Optional, TypedDict
from loguru import logger
@@ -47,19 +47,30 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
"""Get the identifier used in LLMSpecificMessage instances for AWS Bedrock."""
return "aws"
def get_llm_invocation_params(self, context: LLMContext) -> AWSBedrockLLMInvocationParams:
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> AWSBedrockLLMInvocationParams:
"""Get AWS Bedrock-specific LLM invocation parameters from a universal LLM context.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings
or ``run_inference``.
Returns:
Dictionary of parameters for invoking AWS Bedrock's LLM API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
converted = self._from_universal_context_messages(
self.get_messages(context), system_instruction=system_instruction
)
effective_system = self._resolve_system_instruction(
converted.system,
system_instruction,
discard_context_system=True,
)
return {
"system": messages.system,
"messages": messages.messages,
"system": [{"text": effective_system}] if effective_system else None,
"messages": converted.messages,
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools) or [],
# To avoid refactoring in AWSBedrockLLMService, we just pass through tool_choice.
@@ -96,32 +107,36 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
@dataclass
class ConvertedMessages:
"""Container for Anthropic-formatted messages converted from universal context."""
"""Container for Bedrock-formatted messages converted from universal context."""
messages: List[dict[str, Any]]
system: Optional[str]
def _from_universal_context_messages(
self, universal_context_messages: List[LLMContextMessage]
self,
universal_context_messages: List[LLMContextMessage],
*,
system_instruction: Optional[str] = None,
) -> ConvertedMessages:
system = None
messages = []
# First, map messages using self._from_universal_context_message(m)
# Extract initial system message from universal messages BEFORE conversion,
# so the helper works with standard message format (not provider-specific).
remaining = list(universal_context_messages)
if remaining and not isinstance(remaining[0], LLMSpecificMessage):
system = self._extract_initial_system(remaining, system_instruction=system_instruction)
# Convert remaining messages to Bedrock format
messages = []
try:
messages = [self._from_universal_context_message(m) for m in universal_context_messages]
messages = [self._from_universal_context_message(m) for m in remaining]
except Exception as e:
logger.error(f"Error mapping messages: {e}")
# See if we should pull the system message out of our messages list
if messages and messages[0]["role"] == "system":
system = messages[0]["content"]
messages.pop(0)
# Convert any subsequent "system"-role messages to "user"-role
# messages, as AWS Bedrock doesn't support system input messages.
# Convert any subsequent "system"/"developer"-role messages to "user"-role
# messages, as AWS Bedrock doesn't support system or developer input messages.
for message in messages:
if message["role"] == "system":
if message["role"] in ("system", "developer"):
message["role"] = "user"
# Merge consecutive messages with the same role.
@@ -209,7 +224,7 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
tool_result_content = [{"json": content_json}]
else:
tool_result_content = [{"text": message["content"]}]
except:
except (json.JSONDecodeError, ValueError, AttributeError):
tool_result_content = [{"text": message["content"]}]
return {
@@ -257,14 +272,15 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
# handle image_url -> image conversion
if item["type"] == "image_url":
if item["image_url"]["url"].startswith("data:"):
# Extract format from data URL (format: "data:image/jpeg;base64,...")
url = item["image_url"]["url"]
mime_type = url.split(":")[1].split(";")[0]
# Bedrock expects format like "jpeg", "png" etc., not "image/jpeg"
image_format = mime_type.split("/")[1]
new_item = {
"image": {
"format": "jpeg",
"source": {
"bytes": base64.b64decode(
item["image_url"]["url"].split(",")[1]
)
},
"format": image_format,
"source": {"bytes": base64.b64decode(url.split(",")[1])},
}
}
new_content.append(new_item)

394
src/pipecat/adapters/services/gemini_adapter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -9,7 +9,7 @@
import base64
import json
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple, TypedDict
from typing import Any, Dict, List, Optional, TypedDict
from loguru import logger
from openai import NotGiven
@@ -53,19 +53,30 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
"""Get the identifier used in LLMSpecificMessage instances for Google."""
return "google"
def get_llm_invocation_params(self, context: LLMContext) -> GeminiLLMInvocationParams:
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> GeminiLLMInvocationParams:
"""Get Gemini-specific LLM invocation parameters from a universal LLM context.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings
or ``run_inference``.
Returns:
Dictionary of parameters for Gemini's API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
converted = self._from_universal_context_messages(
self.get_messages(context), system_instruction=system_instruction
)
effective_system = self._resolve_system_instruction(
converted.system_instruction,
system_instruction,
discard_context_system=True,
)
return {
"system_instruction": messages.system_instruction,
"messages": messages.messages,
"system_instruction": effective_system,
"messages": converted.messages,
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools),
}
@@ -151,6 +162,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
for part in obj["parts"]:
if "inline_data" in part:
part["inline_data"]["data"] = "..."
if "thought_signature" in part:
part["thought_signature"] = "..."
except Exception as e:
logger.debug(f"Error: {e}")
messages_for_logging.append(obj)
@@ -167,69 +180,99 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
class MessageConversionResult:
"""Result of converting a single universal context message to Google format.
Either content (a Google Content object) or a system instruction string
is guaranteed to be set.
Also returns a tool call ID to name mapping for any tool calls
discovered in the message.
Contains a Google Content object and a tool call ID to name mapping
for any tool calls discovered in the message.
"""
content: Optional[Content] = None
system_instruction: Optional[str] = None
tool_call_id_to_name_mapping: Dict[str, str] = field(default_factory=dict)
@dataclass
class MessageConversionParams:
"""Parameters for converting a single universal context message to Google format."""
already_have_system_instruction: bool
tool_call_id_to_name_mapping: Dict[str, str]
def _from_universal_context_messages(
self, universal_context_messages: List[LLMContextMessage]
self,
universal_context_messages: List[LLMContextMessage],
*,
system_instruction: Optional[str] = None,
) -> ConvertedMessages:
"""Restructures messages to ensure proper Google format and message ordering.
This method handles conversion of OpenAI-formatted messages to Google format,
with special handling for function calls, function responses, and system messages.
System messages are added back to the context as user messages when needed.
with special handling for function calls, function responses, and system/developer
messages.
The final message order is preserved as:
Initial system/developer messages are extracted as the system instruction
(only from ``messages[0]``). Subsequent system/developer messages are
converted to user role.
1. Function calls (from model)
2. Function responses (from user)
3. Text messages (converted from system messages)
Note::
System messages are only added back when there are no regular text
messages in the context, ensuring proper conversation continuity
after function calls.
Args:
universal_context_messages: Messages from the LLM context.
system_instruction: Optional system instruction from service settings,
used to decide whether to extract an initial "developer" message.
"""
system_instruction = None
# Extract initial system/developer message from universal messages before conversion.
# We work on a mutable copy so we can pop messages[0] if needed.
remaining_messages = list(universal_context_messages)
extracted_system = None
# Extract initial system message from universal messages BEFORE conversion,
# so the helper works with standard message format.
if remaining_messages and not isinstance(remaining_messages[0], LLMSpecificMessage):
extracted_system = self._extract_initial_system(
remaining_messages, system_instruction=system_instruction
)
messages = []
tool_call_id_to_name_mapping = {}
thought_signature_dicts = []
# Process each message, preserving Google-formatted messages and converting others
for message in universal_context_messages:
result = self._from_universal_context_message(
# Process each message, converting to Google format as needed
for message in remaining_messages:
# We have a Google-specific message; this may either be a
# thought-signature-containing message that we need to handle in a
# special way, or a message already in Google format that we can
# use directly
if isinstance(message, LLMSpecificMessage):
if (
isinstance(message.message, dict)
and message.message.get("type") == "thought_signature"
):
thought_signature_dicts.append(message.message)
continue
# Fall back to assuming that the message is already in Google
# format
messages.append(message.message)
continue
# We have a standard universal context message; convert it to
# Google format
result = self._from_standard_message(
message,
params=self.MessageConversionParams(
already_have_system_instruction=bool(system_instruction),
tool_call_id_to_name_mapping=tool_call_id_to_name_mapping,
),
)
# Each result is either a Content or a system instruction
if result.content:
messages.append(result.content)
elif result.system_instruction:
system_instruction = result.system_instruction
# Merge tool call ID to name mapping
if result.tool_call_id_to_name_mapping:
tool_call_id_to_name_mapping.update(result.tool_call_id_to_name_mapping)
# Apply thought signatures to the corresponding messages
self._apply_thought_signatures_to_messages(thought_signature_dicts, messages)
# When thinking is enabled, merge parallel tool calls into single messages
messages = self._merge_parallel_tool_calls_for_thinking(thought_signature_dicts, messages)
# Check if we only have function-related messages (no regular text)
effective_system = extracted_system or system_instruction
has_regular_messages = any(
len(msg.parts) == 1
and getattr(msg.parts[0], "text", None)
@@ -239,20 +282,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
)
# Add system instruction back as a user message if we only have function messages
if system_instruction and not has_regular_messages:
messages.append(Content(role="user", parts=[Part(text=system_instruction)]))
if effective_system and not has_regular_messages:
messages.append(Content(role="user", parts=[Part(text=effective_system)]))
# Remove any empty messages
messages = [m for m in messages if m.parts]
return self.ConvertedMessages(messages=messages, system_instruction=system_instruction)
def _from_universal_context_message(
self, message: LLMContextMessage, *, params: MessageConversionParams
) -> MessageConversionResult:
if isinstance(message, LLMSpecificMessage):
return self.MessageConversionResult(content=message.message)
return self._from_standard_message(message, params=params)
return self.ConvertedMessages(
messages=messages,
system_instruction=extracted_system,
)
def _from_standard_message(
self, message: LLMStandardMessage, *, params: MessageConversionParams
@@ -260,17 +299,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
"""Convert standard universal context message to Google Content object.
Handles conversion of text, images, and function calls to Google's
format.
System instructions are returned as a plain string.
format. System and developer messages at this stage (i.e. non-initial
ones, since the initial one is already extracted) are converted to
user role.
Args:
message: Message in standard universal context format.
already_have_system_instruction: Whether we already have a system instruction
params: Parameters for conversion.
Returns:
MessageConversionResult containing either a Content object or a
system instruction string.
MessageConversionResult containing a Content object.
Examples:
Standard text message::
@@ -311,20 +349,10 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
role = message["role"]
content = message.get("content", [])
if role == "system":
if params.already_have_system_instruction:
role = "user" # Convert system message to user role if we already have a system instruction
else:
system_instruction: str = None
if isinstance(content, str):
system_instruction = content
elif isinstance(content, list):
# If content is a list, we assume it's a list of text parts, per the standard
system_instruction = " ".join(
part["text"] for part in content if part.get("type") == "text"
)
if system_instruction:
return self.MessageConversionResult(system_instruction=system_instruction)
# Convert non-initial system/developer messages to user role,
# as Gemini doesn't support these as input messages.
if role in ("system", "developer"):
role = "user"
elif role == "assistant":
role = "model"
@@ -380,11 +408,14 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
if c["type"] == "text":
parts.append(Part(text=c["text"]))
elif c["type"] == "image_url" and c["image_url"]["url"].startswith("data:"):
# Extract MIME type from data URL (format: "data:image/jpeg;base64,...")
url = c["image_url"]["url"]
mime_type = url.split(":")[1].split(";")[0]
parts.append(
Part(
inline_data=Blob(
mime_type="image/jpeg",
data=base64.b64decode(c["image_url"]["url"].split(",")[1]),
mime_type=mime_type,
data=base64.b64decode(url.split(",")[1]),
)
)
)
@@ -410,3 +441,236 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
content=Content(role=role, parts=parts),
tool_call_id_to_name_mapping=tool_call_id_to_name_mapping,
)
def _merge_parallel_tool_calls_for_thinking(
self, thought_signature_dicts: List[dict], messages: List[Content]
) -> List[Content]:
"""Merge parallel tool calls into single Content objects when thinking is enabled.
Gemini expects parallel tool calls (multiple function calls made
simultaneously) to be in a single Content with multiple function_call
Parts. This method takes a list of Content messages, where parallel
tool calls may be split across multiple messages, and merges them into
single messages.
This only has an effect when thought_signatures are present (i.e., when
thinking is enabled). When thinking is disabled, merging doesn't matter.
When thinking is enabled, there is a guarantee that the first tool call
(and only the first) in any batch of parallel tool calls will have a
thought_signature. This allows us to distinguish:
- Parallel tool calls: share a single thought_signature (on the first call)
- Sequential tool calls: each have their own thought_signature
Algorithm: A tool call message with a thought_signature starts a new
parallel group. Any tool call messages after it without a
thought_signature get merged into that group, regardless of what
messages appear in between.
Args:
thought_signature_dicts: A list of thought signature dicts, used
to determine if the work of merging is necessary.
messages: List of Content messages to process.
Returns:
List of Content messages with parallel tool calls merged when
thought_signatures are present, otherwise unchanged.
"""
if not messages:
return messages
# Fast-exit if no function-call-related thought signatures
# This is a shortcut for determining both:
# - whether thinking is enabled, and
# - whether there are function calls in the messages
has_function_call_signatures = any(
ts.get("bookmark", {}).get("function_call") for ts in thought_signature_dicts
)
if not has_function_call_signatures:
return messages
def is_tool_call_message(msg: Content) -> bool:
"""Check if message contains only function_call parts."""
return (
msg.role == "model"
and msg.parts
and all(getattr(part, "function_call", None) for part in msg.parts)
)
def message_has_thought_signature(msg: Content) -> bool:
"""Check if any part in the message has a thought_signature."""
return any(getattr(part, "thought_signature", None) for part in msg.parts)
merged_messages = []
i = 0
while i < len(messages):
current = messages[i]
# If this is a tool call message with a thought signature, start merging
if is_tool_call_message(current) and message_has_thought_signature(current):
merged_parts = list(current.parts)
other_messages = []
j = i + 1
# Scan forward, merging tool calls without signatures, collecting others
while j < len(messages):
next_msg = messages[j]
if is_tool_call_message(next_msg):
if message_has_thought_signature(next_msg):
# New parallel group starts, stop here
break
else:
# Merge this call into the current group
merged_parts.extend(next_msg.parts)
j += 1
else:
# Collect non-tool-call message, keep scanning
other_messages.append(next_msg)
j += 1
# Output merged calls, then collected other messages
merged_messages.append(Content(role="model", parts=merged_parts))
merged_messages.extend(other_messages)
i = j
else:
merged_messages.append(current)
i += 1
return merged_messages
def _apply_thought_signatures_to_messages(
self, thought_signature_dicts: List[dict], messages: List[Content]
) -> None:
"""Apply thought signatures to corresponding assistant messages.
See GoogleLLMService for more details about thought signatures.
Args:
thought_signature_dicts: A list of dicts containing:
- "signature": a thought signature
- "bookmark": a bookmark to identify the message part to apply the signature to.
The bookmark may contain one of:
- "function_call" (a function call ID string)
- "text" (a text string)
- "inline_data" (a Blob)
The list of thought signature dicts is in order.
messages: List of messages to apply the thought signatures to.
"""
if not thought_signature_dicts:
return
# For debugging, print out thought signatures and their bookmarks
logger.debug(f"Thought signatures to apply: {len(thought_signature_dicts)}")
for ts in thought_signature_dicts:
bookmark = ts.get("bookmark")
if bookmark.get("function_call"):
logger.trace(f" - To function call: {bookmark['function_call']}")
elif bookmark.get("text"):
text = bookmark["text"]
log_display_text = f"{text[:50]}..." if len(text) > 50 else text
logger.trace(f" - To text: {log_display_text}")
elif bookmark.get("inline_data"):
logger.trace(f" - To inline data")
# Get all assistant messages
assistant_messages = [
message
for message in messages
if isinstance(message, Content) and message.role == "model"
]
# Apply thought signatures to the corresponding assistant messages.
# Thought signatures are already in message order.
thought_signatures_applied = 0
message_start_index = 0 # Track where to start searching for the next matching message.
for thought_signature_dict in thought_signature_dicts:
signature = thought_signature_dict.get("signature")
bookmark = thought_signature_dict.get("bookmark")
if not signature or not bookmark:
continue
# Search through remaining assistant messages for a match
for i in range(message_start_index, len(assistant_messages)):
message = assistant_messages[i]
if not message.parts:
continue
# We're assuming that the thought signature always applies to the last part
last_part = message.parts[-1]
# If the bookmark matches the part...
if self._thought_signature_bookmark_matches_part(bookmark, last_part):
# Apply the thought signature
last_part.thought_signature = signature
thought_signatures_applied += 1
# Update the start index and stop searching for a match
message_start_index = i + 1
break
# For debugging, print out how many thought signatures were applied
logger.debug(f"Applied {thought_signatures_applied} thought signatures.")
def _thought_signature_bookmark_matches_part(self, bookmark: dict, part: Part) -> bool:
if function_call_bookmark := bookmark.get("function_call"):
return self._thought_signature_function_call_bookmark_matches_part(
function_call_bookmark, part
)
elif text_bookmark := bookmark.get("text"):
return self._thought_signature_text_bookmark_matches_part(text_bookmark, part)
elif inline_data := bookmark.get("inline_data"):
return self._thought_signature_inline_data_bookmark_matches_part(inline_data, part)
else:
logger.warning(f"Unknown thought signature bookmark type: {bookmark}")
return False
def _thought_signature_function_call_bookmark_matches_part(
self, bookmark_function_call_id: str, part: Part
) -> bool:
if (
hasattr(part, "function_call")
and part.function_call
and part.function_call.id == bookmark_function_call_id
):
logger.trace(f"Thought signature function call match: {bookmark_function_call_id}")
return True
return False
def _thought_signature_text_bookmark_matches_part(self, bookmark_text: str, part: Part) -> bool:
if hasattr(part, "text") and part.text:
# Normalize whitespace for comparison
bookmark_text = " ".join(bookmark_text.split())
part_text = " ".join(part.text.split())
# Check that either:
# - the part text is the same as the bookmark text
# - a prefix of the bookmark text (in case the part text was truncated due to interruption)
# - the bookmark text is a prefix of the part text (in case the bookmark represents just first chunk of multi-chunk text)
if (
part_text == bookmark_text
or bookmark_text.startswith(part_text)
or part_text.startswith(bookmark_text)
):
log_display_text = f"{part.text[:50]}..." if len(part.text) > 50 else part.text
logger.trace(f"Thought signature text match: {log_display_text}")
return True
return False
def _thought_signature_inline_data_bookmark_matches_part(
self, bookmark_inline_data: Blob, part: Part
) -> bool:
if (
hasattr(part, "inline_data")
and part.inline_data
# Comparing length should be good enough for matching inline data,
# especially since we're already matching thought signatures in
# strict message order. Comparing actual data is expensive.
and len(part.inline_data.data) == len(bookmark_inline_data.data)
):
logger.trace(f"Thought signature inline data match")
return True
return False

266
src/pipecat/adapters/services/grok_realtime_adapter.py Normal file
View File

@@ -0,0 +1,266 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Grok Realtime LLM adapter for Pipecat.
Converts Pipecat's tool schemas and context into the format required by
Grok's Voice Agent API.
"""
import copy
import json
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, TypedDict
from loguru import logger
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage
from pipecat.services.xai.realtime import events
class GrokRealtimeLLMInvocationParams(TypedDict):
"""Context-based parameters for invoking Grok Realtime API.
Attributes:
system_instruction: System prompt/instructions for the session.
messages: List of conversation items formatted for Grok Realtime.
tools: List of tool definitions (function, web_search, x_search, file_search).
"""
system_instruction: Optional[str]
messages: List[events.ConversationItem]
tools: List[Dict[str, Any]]
class GrokRealtimeLLMAdapter(BaseLLMAdapter):
"""LLM adapter for Grok Voice Agent API.
Converts Pipecat's universal context and tool schemas into the specific
format required by Grok's Voice Agent Realtime API.
"""
@property
def id_for_llm_specific_messages(self) -> str:
"""Get the identifier used in LLMSpecificMessage instances for Grok Realtime."""
return "grok-realtime"
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> GrokRealtimeLLMInvocationParams:
"""Get Grok Realtime-specific LLM invocation parameters from a universal LLM context.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings.
Returns:
Dictionary of parameters for invoking Grok's Voice Agent API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
effective_system = self._resolve_system_instruction(
messages.system_instruction,
system_instruction,
discard_context_system=True,
)
return {
"system_instruction": effective_system,
"messages": messages.messages,
"tools": self.from_standard_tools(context.tools) or [],
}
def get_messages_for_logging(self, context) -> List[Dict[str, Any]]:
"""Get messages from context in a format safe for logging.
Removes or truncates sensitive data like audio content.
Args:
context: The LLM context containing messages.
Returns:
List of messages with sensitive data redacted.
"""
msgs = []
for message in self.get_messages(context):
msg = copy.deepcopy(message)
if "content" in msg:
if isinstance(msg["content"], list):
for item in msg["content"]:
if item.get("type") == "input_audio":
item["audio"] = "..."
if item.get("type") == "audio":
item["audio"] = "..."
msgs.append(msg)
return msgs
@dataclass
class ConvertedMessages:
"""Container for Grok-formatted messages converted from universal context."""
messages: List[events.ConversationItem]
system_instruction: Optional[str] = None
def _from_universal_context_messages(
self, universal_context_messages: List[LLMContextMessage]
) -> ConvertedMessages:
"""Convert universal context messages to Grok Realtime format.
Similar to OpenAI Realtime, we pack conversation history into a single
user message since the realtime API doesn't support loading long histories.
Args:
universal_context_messages: List of messages in universal format.
Returns:
ConvertedMessages with Grok-formatted messages and system instruction.
"""
if not universal_context_messages:
return self.ConvertedMessages(messages=[])
messages = copy.deepcopy(universal_context_messages)
system_instruction = None
# Extract system message as session instructions
if messages[0].get("role") == "system":
system = messages.pop(0)
content = system.get("content")
if isinstance(content, str):
system_instruction = content
elif isinstance(content, list):
system_instruction = content[0].get("text")
if not messages:
return self.ConvertedMessages(messages=[], system_instruction=system_instruction)
# Convert any remaining "system"/"developer" messages to "user"
for msg in messages:
if msg.get("role") in ("system", "developer"):
msg["role"] = "user"
# Single user message can be sent normally
if len(messages) == 1 and messages[0].get("role") == "user":
return self.ConvertedMessages(
messages=[self._from_universal_context_message(messages[0])],
system_instruction=system_instruction,
)
# Pack multiple messages into a single user message
intro_text = """
This is a previously saved conversation. Please treat this conversation history as a
starting point for the current conversation."""
trailing_text = """
This is the end of the previously saved conversation. Please continue the conversation
from here. If the last message is a user instruction or question, act on that instruction
or answer the question. If the last message is an assistant response, simply say that you
are ready to continue the conversation."""
return self.ConvertedMessages(
messages=[
events.ConversationItem(
role="user",
type="message",
content=[
events.ItemContent(
type="input_text",
text="\n\n".join(
[
intro_text,
json.dumps(messages, indent=2),
trailing_text,
]
),
)
],
)
],
system_instruction=system_instruction,
)
def _from_universal_context_message(
self, message: LLMContextMessage
) -> events.ConversationItem:
"""Convert a single universal context message to Grok format.
Args:
message: Message in universal format.
Returns:
ConversationItem formatted for Grok Realtime API.
"""
if message.get("role") == "user":
content = message.get("content")
if isinstance(content, list):
text_content = ""
for c in content:
if c.get("type") == "text":
text_content += " " + c.get("text")
else:
logger.error(
f"Unhandled content type in context message: {c.get('type')} - {message}"
)
content = text_content.strip()
return events.ConversationItem(
role="user",
type="message",
content=[events.ItemContent(type="input_text", text=content)],
)
if message.get("role") == "assistant" and message.get("tool_calls"):
tc = message.get("tool_calls")[0]
return events.ConversationItem(
type="function_call",
call_id=tc["id"],
name=tc["function"]["name"],
arguments=tc["function"]["arguments"],
)
logger.error(f"Unhandled message type in _from_universal_context_message: {message}")
@staticmethod
def _to_grok_function_format(function: FunctionSchema) -> Dict[str, Any]:
"""Convert a function schema to Grok Realtime function format.
Args:
function: The function schema to convert.
Returns:
Dictionary in Grok Realtime function format.
"""
return {
"type": "function",
"name": function.name,
"description": function.description,
"parameters": {
"type": "object",
"properties": function.properties,
"required": function.required,
},
}
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
"""Convert tool schemas to Grok Realtime format.
Supports both standard function tools and Grok-specific tools
(web_search, x_search, file_search).
Args:
tools_schema: The tools schema containing functions to convert.
Returns:
List of tool definitions in Grok Realtime format.
"""
# Convert standard function tools
functions_schema = tools_schema.standard_tools
standard_tools = [self._to_grok_function_format(func) for func in functions_schema]
# Support shimmed custom tools for backward compatibility
shimmed_tools = []
if tools_schema.custom_tools:
shimmed_tools = tools_schema.custom_tools.get(AdapterType.SHIM, [])
return standard_tools + shimmed_tools

50
src/pipecat/adapters/services/open_ai_adapter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -7,10 +7,8 @@
"""OpenAI LLM adapter for Pipecat."""
import copy
import json
from typing import Any, Dict, List, TypedDict
from typing import Any, Dict, List, Optional, TypedDict
from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN
from openai._types import NotGiven as OpenAINotGiven
from openai.types.chat import (
ChatCompletionMessageParam,
@@ -53,17 +51,46 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
"""Get the identifier used in LLMSpecificMessage instances for OpenAI."""
return "openai"
def get_llm_invocation_params(self, context: LLMContext) -> OpenAILLMInvocationParams:
def get_llm_invocation_params(
self,
context: LLMContext,
*,
system_instruction: Optional[str] = None,
convert_developer_to_user: bool,
) -> OpenAILLMInvocationParams:
"""Get OpenAI-specific LLM invocation parameters from a universal LLM context.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings
or ``run_inference``. If provided, prepended as a system message.
convert_developer_to_user: If True, convert "developer"-role messages
to "user"-role messages. Used by OpenAI-compatible services that
don't support the "developer" role.
Returns:
Dictionary of parameters for OpenAI's ChatCompletion API.
"""
messages = self._from_universal_context_messages(
self.get_messages(context), convert_developer_to_user=convert_developer_to_user
)
if system_instruction:
# Detect initial system message for warning purposes (don't extract)
initial_content = (
messages[0].get("content", "")
if messages and messages[0].get("role") == "system"
else None
)
self._resolve_system_instruction(
initial_content,
system_instruction,
discard_context_system=False,
)
messages = [{"role": "system", "content": system_instruction}] + messages
return {
"messages": self._from_universal_context_messages(self.get_messages(context)),
"messages": messages,
# NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools),
"tool_choice": context.tool_choice,
@@ -113,7 +140,10 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
return msgs
def _from_universal_context_messages(
self, messages: List[LLMContextMessage]
self,
messages: List[LLMContextMessage],
*,
convert_developer_to_user: bool,
) -> List[ChatCompletionMessageParam]:
result = []
for message in messages:
@@ -123,6 +153,12 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
else:
# Standard message, pass through unchanged
result.append(message)
if convert_developer_to_user:
for msg in result:
if msg.get("role") == "developer":
msg["role"] = "user"
return result
def _from_standard_tool_choice(

25
src/pipecat/adapters/services/open_ai_realtime_adapter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -43,20 +43,26 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
"""Get the identifier used in LLMSpecificMessage instances for OpenAI Realtime."""
return "openai-realtime"
def get_llm_invocation_params(self, context: LLMContext) -> OpenAIRealtimeLLMInvocationParams:
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> OpenAIRealtimeLLMInvocationParams:
"""Get OpenAI Realtime-specific LLM invocation parameters from a universal LLM context.
This is a placeholder until support for universal LLMContext machinery is added for OpenAI Realtime.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings.
Returns:
Dictionary of parameters for invoking OpenAI Realtime's API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
effective_system = self._resolve_system_instruction(
messages.system_instruction,
system_instruction,
discard_context_system=True,
)
return {
"system_instruction": messages.system_instruction,
"system_instruction": effective_system,
"messages": messages.messages,
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools) or [],
@@ -116,8 +122,8 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
messages = copy.deepcopy(universal_context_messages)
system_instruction = None
# If we have a "system" message as our first message, let's pull that out into session
# "instructions"
# If we have a "system" message as our first message,
# pull that out into session "instructions"
if messages[0].get("role") == "system":
system = messages.pop(0)
content = system.get("content")
@@ -128,6 +134,11 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
if not messages:
return self.ConvertedMessages(messages=[], system_instruction=system_instruction)
# Convert any remaining "system"/"developer" messages to "user"
for msg in messages:
if msg.get("role") in ("system", "developer"):
msg["role"] = "user"
# If we have just a single "user" item, we can just send it normally
if len(messages) == 1 and messages[0].get("role") == "user":
return self.ConvertedMessages(

248
src/pipecat/adapters/services/open_ai_responses_adapter.py Normal file
View File

@@ -0,0 +1,248 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""OpenAI Responses API adapter for Pipecat."""
import copy
from typing import Any, Dict, List, Optional, TypedDict
from openai._types import NotGiven as OpenAINotGiven
from openai.types.responses import FunctionToolParam, ResponseInputItemParam
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.processors.aggregators.llm_context import (
LLMContext,
LLMContextMessage,
LLMSpecificMessage,
NotGiven,
)
class OpenAIResponsesLLMInvocationParams(TypedDict, total=False):
"""Context-based parameters for invoking OpenAI Responses API."""
input: List[ResponseInputItemParam]
tools: List[FunctionToolParam] | OpenAINotGiven
instructions: str
class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParams]):
"""OpenAI Responses API adapter for Pipecat.
Handles:
- Converting LLMContext messages to Responses API input items
- Converting Pipecat's standardized tools schema to Responses API function tool format
- Extracting and sanitizing messages from the LLM context for logging
"""
@property
def id_for_llm_specific_messages(self) -> str:
"""Get the identifier used in LLMSpecificMessage instances."""
return "openai_responses"
def get_llm_invocation_params(
self,
context: LLMContext,
*,
system_instruction: Optional[str] = None,
) -> OpenAIResponsesLLMInvocationParams:
"""Get Responses API invocation parameters from a universal LLM context.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings.
Returns:
Dictionary of parameters for the Responses API.
"""
messages = self.get_messages(context)
# Check for conflict: system_instruction + initial system message
if system_instruction and messages:
first_msg = messages[0] if not isinstance(messages[0], LLMSpecificMessage) else None
if first_msg and first_msg.get("role") == "system":
self._resolve_system_instruction(
first_msg.get("content", ""),
system_instruction,
discard_context_system=False,
)
input_items = self._convert_messages_to_input(messages)
params: OpenAIResponsesLLMInvocationParams = {
"input": input_items,
"tools": self.from_standard_tools(context.tools),
}
if system_instruction:
# Compatibility: The Responses API requires at least one input
# message when instructions are provided. Contexts that worked with
# OpenAILLMService (system_instruction + empty messages) need the
# instructions converted to an initial developer message.
#
# NOTE: if/when we support `previous_response_id` and/or
# `conversation_id`, we'll need to revisit this logic, as it'll
# be legit to provide instructions without input items. Worth
# noting that OpenAI's docs suggest these parameters are primarily
# for development convenience rather than performance (the model
# still processes the full context), and come with the tradeoff
# of requiring OpenAI-side 30-day conversation storage, which may
# not be desirable for many users. But it could give folks an easy
# way to store/switch between conversations without needing to
# manage that storage themselves.
if not input_items:
params["input"] = [{"role": "developer", "content": system_instruction}]
else:
params["instructions"] = system_instruction
return params
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[FunctionToolParam]:
"""Convert function schemas to Responses API function tool format.
Args:
tools_schema: The Pipecat tools schema to convert.
Returns:
List of Responses API function tool definitions.
"""
functions_schema = tools_schema.standard_tools
result = []
for func in functions_schema:
d = func.to_default_dict()
tool: FunctionToolParam = {
"type": "function",
"name": d["name"],
"parameters": d.get("parameters", {}),
"strict": d.get("strict", None),
}
if "description" in d:
tool["description"] = d["description"]
result.append(tool)
return result
def get_messages_for_logging(self, context: LLMContext) -> List[Dict[str, Any]]:
"""Get messages from context in a format ready for logging.
Removes or truncates sensitive data like image content for safe logging.
Args:
context: The LLM context containing messages.
Returns:
List of messages in a format ready for logging.
"""
msgs = []
for message in self.get_messages(context):
msg = copy.deepcopy(message)
if "content" in msg:
if isinstance(msg["content"], list):
for item in msg["content"]:
if item.get("type") == "image_url":
if item["image_url"]["url"].startswith("data:image/"):
item["image_url"]["url"] = "data:image/..."
if item.get("type") == "input_audio":
item["input_audio"]["data"] = "..."
msgs.append(msg)
return msgs
def _convert_messages_to_input(
self, messages: List[LLMContextMessage]
) -> List[ResponseInputItemParam]:
"""Convert LLMContext messages to Responses API input items.
Args:
messages: Messages from the LLMContext.
Returns:
List of Responses API input items.
"""
result: List[ResponseInputItemParam] = []
for message in messages:
if isinstance(message, LLMSpecificMessage):
result.append(message.message)
continue
role = message.get("role")
if role in ("system", "developer"):
content = message.get("content", "")
if isinstance(content, list):
content = self._convert_multimodal_content(content)
result.append({"role": "developer", "content": content})
elif role == "user":
content = message.get("content", "")
if isinstance(content, list):
content = self._convert_multimodal_content(content)
result.append({"role": "user", "content": content})
elif role == "assistant":
tool_calls = message.get("tool_calls")
if tool_calls:
for tc in tool_calls:
func = tc.get("function", {})
result.append(
{
"type": "function_call",
"call_id": tc.get("id", ""),
"name": func.get("name", ""),
"arguments": func.get("arguments", ""),
}
)
else:
content = message.get("content", "")
if isinstance(content, list):
content = self._convert_multimodal_content(content)
result.append({"role": "assistant", "content": content})
elif role == "tool":
content = message.get("content", "")
if not isinstance(content, str):
content = str(content)
result.append(
{
"type": "function_call_output",
"call_id": message.get("tool_call_id", ""),
"output": content,
}
)
return result
def _convert_multimodal_content(self, content: list) -> list:
"""Convert multimodal content parts to Responses API format.
Args:
content: List of content parts from the LLMContext message.
Returns:
List of content parts in Responses API format.
"""
result = []
for part in content:
part_type = part.get("type")
if part_type == "text":
result.append({"type": "input_text", "text": part.get("text", "")})
elif part_type == "image_url":
image_url_obj = part.get("image_url", {})
result.append(
{
"type": "input_image",
"image_url": image_url_obj.get("url", ""),
"detail": image_url_obj.get("detail", "auto"),
}
)
else:
# Pass through other types as-is. Note: "input_audio" is not
# yet supported by the Responses API (coming soon per OpenAI
# docs) but the LLMContext format already matches the expected
# shape, so it should work once support is enabled.
result.append(part)
return result

169
src/pipecat/adapters/services/perplexity_adapter.py Normal file
View File

@@ -0,0 +1,169 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Perplexity LLM adapter for Pipecat.
Perplexity's API uses an OpenAI-compatible interface but enforces stricter
constraints on conversation history structure:
1. **Strict role alternation** — Messages must alternate between "user"/"tool"
and "assistant" roles. Consecutive messages with the same role (e.g. two
"user" messages in a row) are rejected with:
``"messages must be an alternating sequence of user/tool and assistant messages"``
2. **No non-initial system messages** — "system" messages are only allowed at
the start of the conversation. A system message after a non-system message
causes:
``"only the initial message can have the system role"``
3. **Last message must be user/tool** — The final message in the conversation
must have role "user" or "tool". A trailing "assistant" message causes:
``"the last message must have the user or tool role"``
This adapter transforms the message list to satisfy all three constraints before
the messages are sent to Perplexity's API.
"""
import copy
from typing import List, Optional
from openai.types.chat import ChatCompletionMessageParam
from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter, OpenAILLMInvocationParams
from pipecat.processors.aggregators.llm_context import LLMContext
class PerplexityLLMAdapter(OpenAILLMAdapter):
"""Adapter that transforms messages to satisfy Perplexity's API constraints.
Perplexity's API is stricter than OpenAI about message structure. This
adapter extends ``OpenAILLMAdapter`` and applies message transformations
to ensure compliance with Perplexity's constraints (role alternation,
no non-initial system messages, last message must be user/tool).
The transformations are applied in ``get_llm_invocation_params`` after the
parent adapter extracts messages from the LLM context (including any
``system_instruction`` prepend).
"""
def get_llm_invocation_params(
self,
context: LLMContext,
*,
system_instruction: Optional[str] = None,
convert_developer_to_user: bool,
) -> OpenAILLMInvocationParams:
"""Get OpenAI-compatible invocation parameters with Perplexity message fixes applied.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings
or ``run_inference``. Forwarded to the parent adapter.
convert_developer_to_user: If True, convert "developer"-role messages
to "user"-role messages. Forwarded to the parent adapter.
Returns:
Dictionary of parameters for Perplexity's ChatCompletion API, with
messages transformed to satisfy Perplexity's constraints.
"""
params = super().get_llm_invocation_params(
context,
system_instruction=system_instruction,
convert_developer_to_user=convert_developer_to_user,
)
params["messages"] = self._transform_messages(list(params["messages"]))
return params
def _transform_messages(
self, messages: List[ChatCompletionMessageParam]
) -> List[ChatCompletionMessageParam]:
"""Transform messages to satisfy Perplexity's API constraints.
Applies three transformation steps in order:
1. **Convert non-initial system messages to user** — Any system message
after the initial system message block is converted to role "user",
since Perplexity rejects system messages after a non-system message.
2. **Merge consecutive same-role messages** — After the above
conversions, adjacent messages with the same role are merged using
list-of-dicts content format. This ensures strict role alternation
(e.g. a converted system→user message adjacent to an existing user
message gets merged).
3. **Remove trailing assistant messages** — If the last message is
"assistant", remove it. OpenAI appears to silently ignore trailing
assistant messages server-side, so removing them preserves equivalent
behavior while satisfying Perplexity's "last message must be
user/tool" constraint.
Note: we intentionally do *not* convert a trailing system message to
"user". That would make the transformation unstable across calls —
Perplexity appears to have statefulness/caching within a conversation,
so a message that was sent as "user" in one call but becomes "system"
in the next (once more messages are appended) causes errors. If the
context consists entirely of system messages, the Perplexity API call
will fail, but that mistake will be caught right away.
Args:
messages: List of message dicts with "role" and "content" keys.
Returns:
Transformed list of message dicts satisfying Perplexity's constraints.
"""
if not messages:
return messages
messages = copy.deepcopy(messages)
# Note: "developer" → "user" conversion is handled by the parent adapter
# via the convert_developer_to_user parameter.
# Step 1: Convert non-initial system messages to "user".
# Perplexity allows system messages at the start, but rejects them
# after any non-system message.
in_initial_system_block = True
for i in range(len(messages)):
if messages[i].get("role") == "system":
if not in_initial_system_block:
messages[i]["role"] = "user"
else:
in_initial_system_block = False
# Step 2: Merge consecutive same-role messages.
# After system→user conversions above, we may have adjacent same-role
# messages that violate Perplexity's strict alternation requirement.
# Skip consecutive system messages at the start — Perplexity allows those.
i = 0
while i < len(messages) - 1:
current = messages[i]
next_msg = messages[i + 1]
if current["role"] == next_msg["role"] == "system":
# Perplexity allows multiple initial system messages, don't merge
i += 1
elif current["role"] == next_msg["role"]:
# Convert string content to list-of-dicts format for merging
if isinstance(current.get("content"), str):
current["content"] = [{"type": "text", "text": current["content"]}]
if isinstance(next_msg.get("content"), str):
next_msg["content"] = [{"type": "text", "text": next_msg["content"]}]
# Merge content from next message into current
if isinstance(current.get("content"), list) and isinstance(
next_msg.get("content"), list
):
current["content"].extend(next_msg["content"])
messages.pop(i + 1)
else:
i += 1
# Step 3: Remove trailing assistant messages.
# Perplexity requires the last message to be "user" or "tool".
# OpenAI appears to silently ignore trailing assistant messages
# server-side, so removing them preserves equivalent behavior.
while messages and messages[-1].get("role") == "assistant":
messages.pop()
return messages

2
src/pipecat/audio/dtmf/types.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/dtmf/utils.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

504
src/pipecat/audio/filters/aic_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -9,117 +9,351 @@
This module provides an audio filter implementation using ai-coustics' AIC SDK to
enhance audio streams in real time. It mirrors the structure of other filters like
the Koala filter and integrates with Pipecat's input transport pipeline.
Classes:
AICFilter: For aic-sdk (uses 'aic_sdk' module)
AICModelManager: Singleton manager for read-only AIC Model instances.
"""
from typing import List, Optional
import asyncio
from pathlib import Path
from threading import Lock
from typing import List, Optional, Tuple
import numpy as np
from aic_sdk import (
Model,
ParameterOutOfRangeError,
ProcessorAsync,
ProcessorConfig,
ProcessorParameter,
set_sdk_id,
)
from loguru import logger
from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.audio.vad.aic_vad import AICVADAnalyzer
from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
try:
# AIC SDK (https://ai-coustics.github.io/aic-sdk-py/api/)
from aic import AICModelType, AICParameter, Model
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
raise Exception(f"Missing module: {e}")
class AICModelManager:
"""Singleton manager for read-only AIC Model instances with reference counting.
Caches Model instances by path or (model_id + download_dir). Multiple
AICFilter instances using the same model share one Model; the manager
acquires on first use and releases when the last reference is dropped.
"""
_cache: dict[str, Tuple[Model, int]] = {} # key -> (model, ref_count)
_lock = Lock()
_loading: dict[
str, asyncio.Task[Model]
] = {} # key -> load task (deduplicates concurrent loads)
@classmethod
def _increment_reference(cls, cache_key: str, entry: Tuple[Model, int]) -> Tuple[Model, str]:
"""Increment reference count for cached entry. Caller must hold _lock."""
cached_model, ref_count = entry
cls._cache[cache_key] = (cached_model, ref_count + 1)
logger.debug(f"AIC model cache key={cache_key!r} ref_count={ref_count + 1}")
return cached_model, cache_key
@classmethod
def _store_new_reference(cls, cache_key: str, model: Model) -> Tuple[Model, str]:
"""Store new model in cache with ref count 1. Caller must hold _lock."""
cls._cache[cache_key] = (model, 1)
logger.debug(f"AIC model cached key={cache_key!r} ref_count=1")
return model, cache_key
@classmethod
async def _load_model_from_file(
cls,
cache_key: str,
*,
model_path: Optional[Path] = None,
model_id: Optional[str] = None,
model_download_dir: Optional[Path] = None,
) -> Model:
"""Run the actual load (file or download). Separate to allow create_task and deduplication."""
if model_path is not None:
logger.debug(f"Loading AIC model from file: {model_path}")
model_path_str = str(model_path)
elif model_id is not None and model_download_dir is not None:
logger.debug(f"Downloading AIC model: {model_id}")
model_download_dir.mkdir(parents=True, exist_ok=True)
model_path_str = await Model.download_async(model_id, str(model_download_dir))
logger.debug(f"Model downloaded to: {model_path_str}")
else:
raise ValueError("Unexpected model_path or (model_id and model_download_dir) state.")
loop = asyncio.get_running_loop()
return await loop.run_in_executor(None, lambda: Model.from_file(model_path_str))
@staticmethod
def _get_cache_key(
*,
model_path: Optional[Path] = None,
model_id: Optional[str] = None,
model_download_dir: Optional[Path] = None,
) -> str:
"""Build a stable cache key for the model.
Args:
model_path: Path to a local .aicmodel file.
model_id: Model identifier (See https://artifacts.ai-coustics.io/ for available models).
model_download_dir: Directory used for downloading models.
Returns:
A string key unique per (path) or (model_id + download_dir).
"""
if model_path is not None:
return f"path:{model_path.resolve()}"
if model_id is not None and model_download_dir is not None:
return f"id:{model_id}:{model_download_dir.resolve()}"
raise ValueError("Either model_path or (model_id and model_download_dir) must be set.")
@classmethod
async def acquire(
cls,
*,
model_path: Optional[Path] = None,
model_id: Optional[str] = None,
model_download_dir: Optional[Path] = None,
) -> Tuple[Model, str]:
"""Get or load a Model and increment its reference count.
Call this when starting a filter. Store the returned key and pass it
to release() when stopping the filter.
Args:
model_path: Path to a local .aicmodel file. If set, model_id is ignored.
model_id: Model identifier to download from CDN.
model_download_dir: Directory for downloading models. Required if
model_id is used.
Returns:
Tuple of (shared Model instance, cache key for release).
Raises:
ValueError: If neither model_path nor (model_id + model_download_dir)
is provided, or if model_id is set without model_download_dir.
"""
cache_key = cls._get_cache_key(
model_path=model_path,
model_id=model_id,
model_download_dir=model_download_dir,
)
with cls._lock:
entry = cls._cache.get(cache_key)
if entry is not None:
return cls._increment_reference(cache_key, entry)
# Deduplicate concurrent loads for the same key
load_task = cls._loading.get(cache_key)
if load_task is None:
load_task = asyncio.create_task(
cls._load_model_from_file(
cache_key,
model_path=model_path,
model_id=model_id,
model_download_dir=model_download_dir,
)
)
cls._loading[cache_key] = load_task
try:
model = await load_task
finally:
with cls._lock:
cls._loading.pop(cache_key, None)
with cls._lock:
entry = cls._cache.get(cache_key)
if entry is not None:
return cls._increment_reference(cache_key, entry)
return cls._store_new_reference(cache_key, model)
@classmethod
def release(cls, key: str) -> None:
"""Release a reference to a cached model.
Call this when stopping a filter, with the key returned from
get_model(). When the last reference is released, the model
is removed from the cache.
Args:
key: Cache key returned by get_model().
"""
with cls._lock:
entry = cls._cache.get(key)
if entry is None:
logger.warning(f"AIC model release unknown key={key!r}")
return
model, ref_count = entry
ref_count -= 1
if ref_count <= 0:
del cls._cache[key]
logger.debug(f"AIC model evicted key={key!r}")
else:
cls._cache[key] = (model, ref_count)
logger.debug(f"AIC model key={key!r} ref_count={ref_count}")
class AICFilter(BaseAudioFilter):
"""Audio filter using ai-coustics' AIC SDK for real-time enhancement.
Buffers incoming audio to the model's preferred block size and processes
planar frames in-place using float32 samples in the linear -1..+1 range.
frames using float32 samples normalized to the range -1 to +1.
"""
def __init__(
self,
*,
license_key: str = "",
model_type: AICModelType = AICModelType.QUAIL_L,
enhancement_level: Optional[float] = 1.0,
voice_gain: Optional[float] = 1.0,
noise_gate_enable: Optional[bool] = True,
license_key: str,
model_id: Optional[str] = None,
model_path: Optional[Path] = None,
model_download_dir: Optional[Path] = None,
enhancement_level: Optional[float] = None,
) -> None:
"""Initialize the AIC filter.
Args:
license_key: ai-coustics license key for authentication.
model_type: Model variant to load.
model_id: Model identifier to download from CDN. Required if model_path
is not provided. See https://artifacts.ai-coustics.io/ for available models.
model_path: Optional path to a local .aicmodel file. If provided,
model_id is ignored and no download occurs.
model_download_dir: Directory for downloading models as a Path object.
Defaults to a cache directory in user's home folder.
enhancement_level: Optional overall enhancement strength (0.0..1.0).
voice_gain: Optional linear gain applied to detected speech (0.0..4.0).
noise_gate_enable: Optional enable/disable noise gate (default: True).
If None, the model default is used.
Raises:
ValueError: If neither model_id nor model_path is provided, or if
enhancement_level is out of range.
"""
# Set SDK ID for telemetry identification (6 = pipecat)
set_sdk_id(6)
if model_id is None and model_path is None:
raise ValueError(
"Either 'model_id' or 'model_path' must be provided. "
"See https://artifacts.ai-coustics.io/ for available models."
)
if enhancement_level is not None and not 0.0 <= enhancement_level <= 1.0:
raise ValueError("'enhancement_level' must be between 0.0 and 1.0.")
self._license_key = license_key
self._model_type = model_type
self._model_id = model_id
self._model_path = model_path
self._model_download_dir = model_download_dir or (
Path.home() / ".cache" / "pipecat" / "aic-models"
)
self._enhancement_level = enhancement_level
self._voice_gain = voice_gain
self._noise_gate_enable = noise_gate_enable
self._bypass = False
self._enabled = True
self._sample_rate = 0
self._aic_ready = False
self._frames_per_block = 0
self._audio_buffer = bytearray()
# Model will be created in start() since the API now requires sample_rate
self._aic = None
def get_vad_factory(self):
"""Return a zero-arg factory that will create the VAD once the model exists.
# Audio format constants
self._bytes_per_sample = 2 # int16 = 2 bytes
self._dtype = np.int16
self._scale = (
32768.0 # 2^15, for normalizing int16 (-32768 to 32767) to float32 (-1.0 to 1.0)
)
# AIC SDK objects; model is shared via AICModelManager
self._model_cache_key: Optional[str] = None
self._model = None
self._processor = None
self._processor_ctx = None
self._vad_ctx = None
# Pre-allocated buffers (resized in start() once frames_per_block is known)
self._in_f32 = None
self._out_i16 = None
def get_vad_context(self):
"""Return the VAD context once the processor exists.
Returns:
A zero-argument callable that, when invoked, returns an initialized
VoiceActivityDetector bound to the underlying AIC model. Raises a
RuntimeError if the model has not been initialized (i.e. start()
has not been called successfully).
The VadContext instance bound to the underlying processor.
Raises RuntimeError if the processor has not been initialized.
"""
def _factory():
if self._aic is None:
raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.")
return self._aic.create_vad()
return _factory
if self._vad_ctx is None:
raise RuntimeError("AIC processor not initialized yet. Call start(sample_rate) first.")
return self._vad_ctx
def create_vad_analyzer(
self,
*,
lookback_buffer_size: Optional[float] = None,
speech_hold_duration: Optional[float] = None,
minimum_speech_duration: Optional[float] = None,
sensitivity: Optional[float] = None,
):
"""Return an analyzer that will lazily instantiate the AIC VAD when ready.
AIC VAD parameters:
- lookback_buffer_size:
Number of window-length audio buffers used as a lookback buffer.
Higher values increase prediction stability but add latency.
Range: 1.0 .. 20.0, Default (SDK): 6.0
- speech_hold_duration:
How long VAD continues detecting after speech ends (in seconds).
Range: 0.0 to 100x model window length, Default (SDK): 0.05s
- minimum_speech_duration:
Minimum duration of speech required before VAD reports speech detected
(in seconds). Range: 0.0 to 1.0, Default (SDK): 0.0s
- sensitivity:
Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity).
Range: 1.0 .. 15.0, Default (SDK): 6.0
Range: 1.0 to 15.0, Default (SDK): 6.0
Args:
lookback_buffer_size: Optional lookback buffer size to configure on the VAD.
Range: 1.0 .. 20.0. If None, SDK default is used.
speech_hold_duration: Optional speech hold duration to configure on the VAD.
If None, SDK default (0.05s) is used.
minimum_speech_duration: Optional minimum speech duration before VAD reports
speech detected. If None, SDK default (0.0s) is used.
sensitivity: Optional sensitivity (energy threshold) to configure on the VAD.
Range: 1.0 .. 15.0. If None, SDK default is used.
Range: 1.0 to 15.0. If None, SDK default (6.0) is used.
Returns:
A lazily-initialized AICVADAnalyzer that will bind to the VAD backend
once the filter's model has been created (after start(sample_rate)).
A lazily-initialized AICVADAnalyzer that will bind to the VAD context
once the filter's processor has been created (after start(sample_rate)).
"""
from pipecat.audio.vad.aic_vad import AICVADAnalyzer
return AICVADAnalyzer(
vad_factory=self.get_vad_factory(),
lookback_buffer_size=lookback_buffer_size,
vad_context_factory=lambda: self.get_vad_context(),
speech_hold_duration=speech_hold_duration,
minimum_speech_duration=minimum_speech_duration,
sensitivity=sensitivity,
)
def _apply_enhancement_level(self):
"""Apply enhancement_level if configured and supported by the active model."""
if self._processor_ctx is None or self._enhancement_level is None:
return
try:
self._processor_ctx.set_parameter(
ProcessorParameter.EnhancementLevel, self._enhancement_level
)
except ParameterOutOfRangeError as e:
logger.warning(f"AIC EnhancementLevel set_parameter out-of-range: {e}")
self._enhancement_level = None
def _apply_bypass(self):
"""Apply bypass parameter to the active processor."""
if self._processor_ctx is None:
return
self._processor_ctx.set_parameter(ProcessorParameter.Bypass, 1.0 if self._bypass else 0.0)
async def start(self, sample_rate: int):
"""Initialize the filter with the transport's sample rate.
@@ -131,59 +365,87 @@ class AICFilter(BaseAudioFilter):
"""
self._sample_rate = sample_rate
# Acquire shared read-only model from singleton manager
self._model, self._model_cache_key = await AICModelManager.acquire(
model_path=self._model_path,
model_id=self._model_id,
model_download_dir=self._model_download_dir,
)
# Get optimal frames for this sample rate
self._frames_per_block = self._model.get_optimal_num_frames(self._sample_rate)
# Allocate processing buffers now that we know the block size
self._in_f32 = np.zeros((1, self._frames_per_block), dtype=np.float32)
self._out_i16 = np.zeros(self._frames_per_block, dtype=np.int16)
# Create configuration
config = ProcessorConfig.optimal(
self._model,
sample_rate=self._sample_rate,
)
# Create async processor
try:
# Create model with required runtime parameters
self._aic = Model(
model_type=self._model_type,
license_key=self._license_key or None,
sample_rate=self._sample_rate,
channels=1,
)
self._frames_per_block = self._aic.optimal_num_frames()
# Optional parameter configuration
if self._enhancement_level is not None:
self._aic.set_parameter(
AICParameter.ENHANCEMENT_LEVEL,
float(self._enhancement_level if self._enabled else 0.0),
)
if self._voice_gain is not None:
self._aic.set_parameter(AICParameter.VOICE_GAIN, float(self._voice_gain))
if self._noise_gate_enable is not None:
self._aic.set_parameter(
AICParameter.NOISE_GATE_ENABLE, 1.0 if bool(self._noise_gate_enable) else 0.0
)
self._aic_ready = True
# Log processor information
logger.debug(f"ai-coustics filter started:")
logger.debug(f" Sample rate: {self._sample_rate} Hz")
logger.debug(f" Frames per chunk: {self._frames_per_block}")
logger.debug(f" Enhancement strength: {int(self._enhancement_level * 100)}%")
logger.debug(f" Optimal input buffer size: {self._aic.optimal_num_frames()} samples")
logger.debug(f" Optimal sample rate: {self._aic.optimal_sample_rate()} Hz")
logger.debug(
f" Current algorithmic latency: {self._aic.processing_latency() / self._sample_rate * 1000:.2f}ms"
)
self._processor = ProcessorAsync(self._model, self._license_key, config)
except Exception as e: # noqa: BLE001 - surfacing SDK initialization errors
logger.error(f"AIC model initialization failed: {e}")
self._aic_ready = False
self._processor = None
self._aic_ready = self._processor is not None
if not self._aic_ready:
logger.debug(f"ai-coustics filter is not ready.")
return
# Get contexts for parameter control and VAD
self._processor_ctx = self._processor.get_processor_context()
self._vad_ctx = self._processor.get_vad_context()
# Apply initial control parameters
self._apply_bypass()
self._apply_enhancement_level()
# Log processor information
logger.debug(f"ai-coustics filter started:")
logger.debug(f" Model ID: {self._model.get_id()}")
logger.debug(f" Sample rate: {self._sample_rate} Hz")
logger.debug(f" Frames per chunk: {self._frames_per_block}")
if self._enhancement_level is not None:
logger.debug(f" Enhancement level: {self._enhancement_level}")
else:
logger.debug(" Enhancement level not configured; using the model's default behavior.")
logger.debug(f" Optimal sample rate: {self._model.get_optimal_sample_rate()} Hz")
logger.debug(
f" Optimal number of frames for {self._sample_rate} Hz: "
f"{self._model.get_optimal_num_frames(self._sample_rate)}"
)
logger.debug(
f" Output delay: {self._processor_ctx.get_output_delay()} samples "
f"({self._processor_ctx.get_output_delay() / self._sample_rate * 1000:.2f}ms)"
)
async def stop(self):
"""Clean up the AIC model when stopping.
"""Clean up the AIC processor when stopping.
Returns:
None
"""
try:
if self._aic is not None:
self._aic.close()
if self._processor_ctx is not None:
self._processor_ctx.reset()
finally:
self._aic = None
self._processor = None
self._processor_ctx = None
self._vad_ctx = None
self._model = None
self._aic_ready = False
self._audio_buffer.clear()
if self._model_cache_key is not None:
AICModelManager.release(self._model_cache_key)
self._model_cache_key = None
async def process_frame(self, frame: FilterControlFrame):
"""Process control frames to enable/disable filtering.
@@ -194,11 +456,11 @@ class AICFilter(BaseAudioFilter):
None
"""
if isinstance(frame, FilterEnableFrame):
self._enabled = frame.enable
if self._aic is not None:
self._bypass = not frame.enable
if self._processor_ctx is not None:
try:
level = float(self._enhancement_level if self._enabled else 0.0)
self._aic.set_parameter(AICParameter.ENHANCEMENT_LEVEL, level)
self._apply_bypass()
self._apply_enhancement_level()
except Exception as e: # noqa: BLE001
logger.error(f"AIC set_parameter failed: {e}")
@@ -209,43 +471,43 @@ class AICFilter(BaseAudioFilter):
model's required block length. Returns enhanced audio data.
Args:
audio: Raw audio data as bytes to be filtered (int16 PCM, planar).
audio: Raw audio data as bytes (int16 PCM).
Returns:
Enhanced audio data as bytes (int16 PCM, planar).
Enhanced audio data as bytes (int16 PCM).
"""
if not self._aic_ready or self._aic is None:
if not self._aic_ready or self._processor is None:
return audio
self._audio_buffer.extend(audio)
available_frames = len(self._audio_buffer) // self._bytes_per_sample
num_blocks = available_frames // self._frames_per_block
if num_blocks == 0:
return b""
block_size = self._frames_per_block * self._bytes_per_sample
total_size = num_blocks * block_size
blocks_data = bytes(self._audio_buffer[:total_size])
self._audio_buffer = self._audio_buffer[total_size:]
filtered_chunks: List[bytes] = []
# Number of int16 samples currently buffered
available_frames = len(self._audio_buffer) // 2
for i in range(num_blocks):
start = i * block_size
block_i16 = np.frombuffer(blocks_data[start : start + block_size], dtype=self._dtype)
while available_frames >= self._frames_per_block:
# Consume exactly one block worth of frames
samples_to_consume = self._frames_per_block * 1
bytes_to_consume = samples_to_consume * 2
block_bytes = bytes(self._audio_buffer[:bytes_to_consume])
# Reuse input buffer, in-place divide
np.copyto(self._in_f32[0], block_i16)
self._in_f32 /= self._scale
# Convert to float32 in -1..+1 range and reshape to planar (channels, frames)
block_i16 = np.frombuffer(block_bytes, dtype=np.int16)
block_f32 = (block_i16.astype(np.float32) / 32768.0).reshape(
(1, self._frames_per_block)
)
out_f32 = await self._processor.process_async(self._in_f32)
# Process planar in-place; returns ndarray (same shape)
out_f32 = await self._aic.process_async(block_f32)
# Convert float32 output back to int16
np.multiply(out_f32, self._scale, out=self._in_f32) # reuse in_f32 as temp
np.clip(self._in_f32, -self._scale, self._scale - 1, out=self._in_f32)
np.copyto(self._out_i16, self._in_f32[0].astype(self._dtype))
# Convert back to int16 bytes, planar layout
out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16)
filtered_chunks.append(out_i16.reshape(-1).tobytes())
filtered_chunks.append(self._out_i16.tobytes())
# Slide buffer
self._audio_buffer = self._audio_buffer[bytes_to_consume:]
available_frames = len(self._audio_buffer) // 2
# Do not flush incomplete frames; keep them buffered for the next call
return b"".join(filtered_chunks)

2
src/pipecat/audio/filters/base_audio_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/filters/koala_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/filters/krisp_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

221
src/pipecat/audio/filters/krisp_viva_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -15,105 +15,122 @@ import numpy as np
from loguru import logger
from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.audio.krisp_instance import (
KrispVivaSDKManager,
int_to_krisp_frame_duration,
int_to_krisp_sample_rate,
)
from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
try:
import krisp_audio
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the Krisp filter, you need to install krisp_audio.")
logger.error("In order to use KrispVivaFilter, you need to install krisp_audio.")
raise Exception(f"Missing module: {e}")
def _log_callback(log_message, log_level):
logger.info(f"[{log_level}] {log_message}")
class KrispVivaFilter(BaseAudioFilter):
"""Audio filter using the Krisp VIVA SDK.
Provides real-time noise reduction for audio streams using Krisp's
proprietary noise suppression algorithms. This filter requires a
valid Krisp model file to operate.
Supported sample rates:
- 8000 Hz
- 16000 Hz
- 24000 Hz
- 32000 Hz
- 44100 Hz
- 48000 Hz
"""
# Initialize Krisp Audio SDK globally
krisp_audio.globalInit("", _log_callback, krisp_audio.LogLevel.Off)
SDK_VERSION = krisp_audio.getVersion()
logger.debug(
f"Krisp Audio Python SDK Version: {SDK_VERSION.major}."
f"{SDK_VERSION.minor}.{SDK_VERSION.patch}"
)
SAMPLE_RATES = {
8000: krisp_audio.SamplingRate.Sr8000Hz,
16000: krisp_audio.SamplingRate.Sr16000Hz,
24000: krisp_audio.SamplingRate.Sr24000Hz,
32000: krisp_audio.SamplingRate.Sr32000Hz,
44100: krisp_audio.SamplingRate.Sr44100Hz,
48000: krisp_audio.SamplingRate.Sr48000Hz,
}
FRAME_SIZE_MS = 10 # Krisp requires audio frames of 10ms duration for processing.
def __init__(self, model_path: str = None, noise_suppression_level: int = 100) -> None:
def __init__(
self,
model_path: str = None,
frame_duration: int = 10,
noise_suppression_level: int = 100,
api_key: str = "",
) -> None:
"""Initialize the Krisp noise reduction filter.
Args:
model_path: Path to the Krisp model file (.kef extension).
If None, uses KRISP_VIVA_MODEL_PATH environment variable.
If None, uses KRISP_VIVA_FILTER_MODEL_PATH environment variable.
frame_duration: Frame duration in milliseconds.
noise_suppression_level: Noise suppression level.
api_key: Krisp SDK API key. If empty, falls back to
the KRISP_VIVA_API_KEY environment variable.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_MODEL_PATH is not set.
ValueError: If model_path is not provided and KRISP_VIVA_FILTER_MODEL_PATH is not set.
Exception: If model file doesn't have .kef extension.
FileNotFoundError: If model file doesn't exist.
RuntimeError: If Krisp SDK initialization fails.
"""
super().__init__()
# Set model path, checking environment if not specified
self._model_path = model_path or os.getenv("KRISP_VIVA_MODEL_PATH")
if not self._model_path:
logger.error("Model path is not provided and KRISP_VIVA_MODEL_PATH is not set.")
raise ValueError("Model path for KrispAudioProcessor must be provided.")
self._api_key = api_key
if not self._model_path.endswith(".kef"):
raise Exception("Model is expected with .kef extension")
try:
# Set model path, checking environment if not specified
if model_path:
self._model_path = model_path
else:
# Check new environment variable first
self._model_path = os.getenv("KRISP_VIVA_FILTER_MODEL_PATH")
# Fall back to old environment variable for backward compatibility
if not self._model_path:
self._model_path = os.getenv("KRISP_VIVA_MODEL_PATH")
if self._model_path:
logger.warning(
"KRISP_VIVA_MODEL_PATH is deprecated. "
"Please use KRISP_VIVA_FILTER_MODEL_PATH instead."
)
if not self._model_path:
logger.error(
"Model path is not provided and KRISP_VIVA_FILTER_MODEL_PATH is not set."
)
raise ValueError("Model path for KrispAudioProcessor must be provided.")
if not os.path.isfile(self._model_path):
raise FileNotFoundError(f"Model file not found: {self._model_path}")
if not self._model_path.endswith(".kef"):
raise Exception("Model is expected with .kef extension")
self._filtering = True
self._session = None
self._samples_per_frame = None
self._noise_suppression_level = noise_suppression_level
if not os.path.isfile(self._model_path):
raise FileNotFoundError(f"Model file not found: {self._model_path}")
# Audio buffer to accumulate samples for complete frames
self._audio_buffer = bytearray()
self._session = None
self._samples_per_frame = None
self._noise_suppression_level = noise_suppression_level
self._frame_duration_ms = frame_duration
self._audio_buffer = bytearray()
self._filtering = True
def _int_to_sample_rate(self, sample_rate):
"""Convert integer sample rate to krisp_audio SamplingRate enum.
except Exception:
# If initialization fails, release the SDK reference
KrispVivaSDKManager.release()
raise
def _create_session(self, sample_rate: int, frame_duration: int):
"""Create a Krisp session with a specific sample rate.
Args:
sample_rate: Sample rate as integer
Returns:
krisp_audio.SamplingRate enum value
sample_rate: Sample rate for the session
frame_duration: Frame duration in milliseconds
Raises:
ValueError: If sample rate is not supported
Exception: If session creation fails
"""
if sample_rate not in self.SAMPLE_RATES:
raise ValueError("Unsupported sample rate")
return self.SAMPLE_RATES[sample_rate]
try:
model_info = krisp_audio.ModelInfo()
model_info.path = self._model_path
nc_cfg = krisp_audio.NcSessionConfig()
nc_cfg.inputSampleRate = int_to_krisp_sample_rate(sample_rate)
nc_cfg.inputFrameDuration = int_to_krisp_frame_duration(frame_duration)
nc_cfg.outputSampleRate = nc_cfg.inputSampleRate
nc_cfg.modelInfo = model_info
self._samples_per_frame = int((sample_rate * frame_duration) / 1000)
self._current_sample_rate = sample_rate
session = krisp_audio.NcInt16.create(nc_cfg)
return session
except Exception as e:
logger.error(f"Failed to create Krisp session: {e}", exc_info=True)
raise RuntimeError(f"Failed to create Krisp processing session: {e}") from e
async def start(self, sample_rate: int):
"""Initialize the Krisp processor with the transport's sample rate.
@@ -121,21 +138,24 @@ class KrispVivaFilter(BaseAudioFilter):
Args:
sample_rate: The sample rate of the input transport in Hz.
"""
model_info = krisp_audio.ModelInfo()
model_info.path = self._model_path
nc_cfg = krisp_audio.NcSessionConfig()
nc_cfg.inputSampleRate = self._int_to_sample_rate(sample_rate)
nc_cfg.inputFrameDuration = krisp_audio.FrameDuration.Fd10ms
nc_cfg.outputSampleRate = nc_cfg.inputSampleRate
nc_cfg.modelInfo = model_info
self._samples_per_frame = int((sample_rate * self.FRAME_SIZE_MS) / 1000)
self._session = krisp_audio.NcInt16.create(nc_cfg)
try:
# Acquire SDK reference (will initialize on first call)
KrispVivaSDKManager.acquire(api_key=self._api_key)
self._session = self._create_session(sample_rate, self._frame_duration_ms)
except Exception as e:
logger.error(f"Failed to start Krisp session: {e}", exc_info=True)
self._session = None
raise RuntimeError(f"Failed to create Krisp processing session: {e}") from e
async def stop(self):
"""Clean up the Krisp processor when stopping."""
self._session = None
try:
self._session = None
self._audio_buffer.clear()
KrispVivaSDKManager.release()
except Exception as e:
logger.error(f"Error in stop: {e}", exc_info=True)
raise RuntimeError(f"Failed to stop Krisp processor: {e}") from e
async def process_frame(self, frame: FilterControlFrame):
"""Process control frames to enable/disable filtering.
@@ -158,36 +178,41 @@ class KrispVivaFilter(BaseAudioFilter):
if not self._filtering:
return audio
# Add incoming audio to our buffer
self._audio_buffer.extend(audio)
try:
# Add incoming audio to our buffer
self._audio_buffer.extend(audio)
# Calculate how many complete frames we can process
total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample
num_complete_frames = total_samples // self._samples_per_frame
# Calculate how many complete frames we can process
total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample
num_complete_frames = total_samples // self._samples_per_frame
if num_complete_frames == 0:
# Not enough samples for a complete frame yet, return empty
return b""
if num_complete_frames == 0:
# Not enough samples for a complete frame yet, return empty
return b""
# Calculate how many bytes we need for complete frames
complete_samples_count = num_complete_frames * self._samples_per_frame
bytes_to_process = complete_samples_count * 2 # 2 bytes per sample
# Calculate how many bytes we need for complete frames
complete_samples_count = num_complete_frames * self._samples_per_frame
bytes_to_process = complete_samples_count * 2 # 2 bytes per sample
# Extract the bytes we can process
audio_to_process = bytes(self._audio_buffer[:bytes_to_process])
# Extract the bytes we can process
audio_to_process = bytes(self._audio_buffer[:bytes_to_process])
# Remove processed bytes from buffer, keep the remainder
self._audio_buffer = self._audio_buffer[bytes_to_process:]
# Remove processed bytes from buffer, keep the remainder
self._audio_buffer = self._audio_buffer[bytes_to_process:]
# Process the complete frames
samples = np.frombuffer(audio_to_process, dtype=np.int16)
frames = samples.reshape(-1, self._samples_per_frame)
processed_samples = np.empty_like(samples)
# Process the complete frames
samples = np.frombuffer(audio_to_process, dtype=np.int16)
frames = samples.reshape(-1, self._samples_per_frame)
processed_samples = np.empty_like(samples)
for i, frame in enumerate(frames):
cleaned_frame = self._session.process(frame, self._noise_suppression_level)
processed_samples[i * self._samples_per_frame : (i + 1) * self._samples_per_frame] = (
cleaned_frame
)
for i, frame in enumerate(frames):
cleaned_frame = self._session.process(frame, self._noise_suppression_level)
processed_samples[
i * self._samples_per_frame : (i + 1) * self._samples_per_frame
] = cleaned_frame
return processed_samples.tobytes()
return processed_samples.tobytes()
except Exception as e:
logger.error(f"Error during Krisp filtering: {e}", exc_info=True)
return audio

2
src/pipecat/audio/filters/noisereduce_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

154
src/pipecat/audio/filters/rnnoise_filter.py Normal file
View File

@@ -0,0 +1,154 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""RNNoise noise suppression audio filter for Pipecat.
This module provides an audio filter implementation using RNNoise, a recurrent
neural network for audio noise reduction, via the pyrnnoise library.
"""
import numpy as np
from loguru import logger
from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
try:
from pyrnnoise import RNNoise
except ModuleNotFoundError as e:
RNNoise = None
logger.error(f"Exception: {e}")
logger.error(
"In order to use the RNNoise filter, you need to `pip install pipecat-ai[rnnoise]`."
)
class RNNoiseFilter(BaseAudioFilter):
"""Audio filter using RNNoise for noise suppression.
Provides real-time noise suppression for audio streams using RNNoise, a
recurrent neural network for audio noise reduction. The filter buffers audio
data to match RNNoise's required frame length (480 samples at 48kHz) and
processes it in chunks.
"""
def __init__(self, resampler_quality: str = "QQ") -> None:
"""Initialize the RNNoise noise suppression filter.
Args:
resampler_quality: Quality of the resampler if resampling is needed.
One of "VHQ", "HQ", "MQ", "LQ", "QQ". Defaults to "QQ"
(Quick) for lowest latency.
"""
self._filtering = True
self._sample_rate = 0
self._rnnoise = None
self._rnnoise_ready = False
self._resampler_in = None
self._resampler_out = None
self._resampler_quality = resampler_quality
async def start(self, sample_rate: int):
"""Initialize the filter with the transport's sample rate.
Args:
sample_rate: The sample rate of the input transport in Hz.
"""
self._sample_rate = sample_rate
try:
# RNNoise always requires 48kHz
self._rnnoise = RNNoise(sample_rate=48000)
self._rnnoise_ready = True
except Exception as e:
logger.error(f"Failed to initialize RNNoise: {e}")
self._rnnoise_ready = False
return
if self._sample_rate != 48000:
logger.info(f"RNNoise filter enabling resampling: {self._sample_rate} <-> 48000")
try:
from pipecat.audio.resamplers.soxr_stream_resampler import SOXRStreamAudioResampler
self._resampler_in = SOXRStreamAudioResampler(quality=self._resampler_quality)
self._resampler_out = SOXRStreamAudioResampler(quality=self._resampler_quality)
except ImportError as e:
logger.error(f"Could not import SOXRStreamAudioResampler for resampling: {e}")
self._rnnoise_ready = False
async def stop(self):
"""Clean up the RNNoise engine when stopping."""
self._rnnoise = None
self._rnnoise_ready = False
self._resampler_in = None
self._resampler_out = None
async def process_frame(self, frame: FilterControlFrame):
"""Process control frames to enable/disable filtering.
Args:
frame: The control frame containing filter commands.
"""
if isinstance(frame, FilterEnableFrame):
self._filtering = frame.enable
async def filter(self, audio: bytes) -> bytes:
"""Apply RNNoise noise suppression to audio data.
Buffers incoming audio and processes it in chunks that match RNNoise's
required frame length (480 samples at 48kHz). Returns filtered audio data.
Args:
audio: Raw audio data as bytes to be filtered.
Returns:
Noise-suppressed audio data as bytes.
"""
if not self._rnnoise_ready or not self._filtering:
return audio
# Resample input if needed
in_audio = audio
if self._sample_rate != 48000 and self._resampler_in:
in_audio = await self._resampler_in.resample(audio, self._sample_rate, 48000)
# If audio is empty, return empty bytes (no point in noise cancellation)
if len(in_audio) == 0:
return b""
# Convert bytes to numpy array (int16)
audio_samples = np.frombuffer(in_audio, dtype=np.int16)
# Process chunk through RNNoise
# denoise_chunk handles buffering internally and yields (speech_prob, denoised_frame)
# denoised_frame is in float32 format normalized to [-1.0, 1.0]
filtered_frames = []
for speech_prob, denoised_frame in self._rnnoise.denoise_chunk(audio_samples):
# Check if output is float (needs scaling) or int16 (ready)
if np.issubdtype(denoised_frame.dtype, np.floating):
denoised_int16 = (denoised_frame * 32767).astype(np.int16)
else:
denoised_int16 = denoised_frame.astype(np.int16)
# Handle shape (pyrnnoise returns (channels, samples), e.g. (1, 480))
# We want flat array for mono
if denoised_int16.ndim > 1:
denoised_int16 = denoised_int16.squeeze()
filtered_frames.append(denoised_int16)
# Combine all processed frames
if filtered_frames:
filtered_audio = np.concatenate(filtered_frames).tobytes()
# Resample output if needed
if self._sample_rate != 48000 and self._resampler_out:
return await self._resampler_out.resample(filtered_audio, 48000, self._sample_rate)
return filtered_audio
# No frames processed yet (buffering)
return b""

2
src/pipecat/audio/interruptions/base_interruption_strategy.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

20
src/pipecat/audio/interruptions/min_words_interruption_strategy.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -17,6 +17,13 @@ class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
This is an interruption strategy based on a minimum number of words said
by the user. That is, the strategy will be true if the user has said at
least that amount of words.
.. deprecated:: 0.0.99
This class is deprecated, use
`pipecat.turns.user_start.MinWordsUserTurnStartStrategy` with `PipelineTask`'s
new `user_turn_strategies` parameter instead.
"""
def __init__(self, *, min_words: int):
@@ -29,6 +36,17 @@ class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
self._min_words = min_words
self._text = ""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'pipecat.audio.interruptions' is deprecated. "
"Use `pipecat.turns.user_start.MinWordsUserTurnStartStrategy` with `PipelineTask`'s "
"new `user_turn_strategies` parameter instead.",
DeprecationWarning,
)
async def append_text(self, text: str):
"""Append text for word count analysis.

205
src/pipecat/audio/krisp_instance.py Normal file
View File

@@ -0,0 +1,205 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Krisp Instance manager for pipecat audio."""
import atexit
import os
from threading import Lock
from loguru import logger
try:
import krisp_audio
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the Krisp instance, you need to install krisp_audio.")
raise Exception(f"Missing module: {e}")
# Mapping of sample rates (Hz) to Krisp SDK SamplingRate enums
KRISP_SAMPLE_RATES = {
8000: krisp_audio.SamplingRate.Sr8000Hz,
16000: krisp_audio.SamplingRate.Sr16000Hz,
24000: krisp_audio.SamplingRate.Sr24000Hz,
32000: krisp_audio.SamplingRate.Sr32000Hz,
44100: krisp_audio.SamplingRate.Sr44100Hz,
48000: krisp_audio.SamplingRate.Sr48000Hz,
}
KRISP_FRAME_DURATIONS = {
10: krisp_audio.FrameDuration.Fd10ms,
15: krisp_audio.FrameDuration.Fd15ms,
20: krisp_audio.FrameDuration.Fd20ms,
30: krisp_audio.FrameDuration.Fd30ms,
32: krisp_audio.FrameDuration.Fd32ms,
}
def int_to_krisp_sample_rate(sample_rate: int):
"""Convert integer sample rate to Krisp SDK enum value.
Args:
sample_rate: Sample rate in Hz (e.g., 16000, 24000, 48000).
Returns:
Corresponding Krisp SDK SampleRate enum value.
Raises:
ValueError: If the sample rate is not supported by Krisp SDK.
"""
if sample_rate not in KRISP_SAMPLE_RATES:
supported_rates = ", ".join(str(rate) for rate in sorted(KRISP_SAMPLE_RATES.keys()))
raise ValueError(
f"Unsupported sample rate: {sample_rate} Hz. Supported rates: {supported_rates} Hz"
)
return KRISP_SAMPLE_RATES[sample_rate]
def int_to_krisp_frame_duration(frame_duration_ms: int):
"""Convert integer frame duration to Krisp SDK enum value.
Args:
frame_duration_ms: Frame duration in milliseconds (e.g., 10, 20, 30).
Returns:
Corresponding Krisp SDK FrameDuration enum value.
Raises:
ValueError: If the frame duration is not supported by Krisp SDK.
"""
if frame_duration_ms not in KRISP_FRAME_DURATIONS:
supported_durations = ", ".join(
str(duration) for duration in sorted(KRISP_FRAME_DURATIONS.keys())
)
raise ValueError(
f"Unsupported frame duration: {frame_duration_ms} ms. "
f"Supported durations: {supported_durations} ms"
)
return KRISP_FRAME_DURATIONS[frame_duration_ms]
class KrispVivaSDKManager:
"""Singleton manager for Krisp VIVA SDK with reference counting."""
_initialized = False
_lock = Lock()
_reference_count = 0
@staticmethod
def _license_callback(error, error_message):
"""Callback for Krisp SDK licensing errors."""
logger.error(f"Krisp licensing error: {error} - {error_message}")
@staticmethod
def _log_callback(log_message, log_level):
"""Thread-safe callback for Krisp SDK logging."""
logger.info(f"[{log_level}] {log_message}")
@classmethod
def acquire(cls, api_key: str = ""):
"""Acquire a reference to the SDK (initializes if needed).
Call this when creating a filter instance.
Args:
api_key: Krisp SDK API key. If empty, falls back to the
KRISP_VIVA_API_KEY environment variable.
Raises:
Exception: If SDK initialization fails (propagated from krisp_audio)
"""
with cls._lock:
# Initialize SDK on first acquire
if cls._reference_count == 0:
try:
key = api_key or os.environ.get("KRISP_VIVA_API_KEY", "")
try:
# New SDK signature (requires license key)
krisp_audio.globalInit(
"",
key,
cls._license_callback,
cls._log_callback,
krisp_audio.LogLevel.Off,
)
except TypeError:
# Old SDK signature (no license key)
krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
cls._initialized = True
SDK_VERSION = krisp_audio.getVersion()
logger.debug(
f"Krisp Audio Python SDK initialized - Version: "
f"{SDK_VERSION.major}.{SDK_VERSION.minor}.{SDK_VERSION.patch}"
)
# Register cleanup on program exit (failsafe)
atexit.register(cls._force_cleanup)
except Exception as e:
cls._initialized = False
logger.error(f"Krisp SDK initialization failed: {e}")
raise
cls._reference_count += 1
logger.debug(f"Krisp SDK reference count: {cls._reference_count}")
@classmethod
def release(cls):
"""Release a reference to the SDK (destroys if last reference).
Call this when destroying a filter instance.
"""
with cls._lock:
if cls._reference_count > 0:
cls._reference_count -= 1
logger.debug(f"Krisp SDK reference count: {cls._reference_count}")
# Destroy SDK when last reference is released
if cls._reference_count == 0 and cls._initialized:
try:
krisp_audio.globalDestroy()
cls._initialized = False
logger.debug("Krisp Audio SDK destroyed (all references released)")
except Exception as e:
logger.error(f"Error during Krisp SDK cleanup: {e}")
cls._initialized = False
@classmethod
def get_reference_count(cls) -> int:
"""Get the current reference count.
Returns:
Current number of active references to the SDK.
"""
with cls._lock:
return cls._reference_count
@classmethod
def is_initialized(cls) -> bool:
"""Check if the SDK is currently initialized.
Returns:
True if SDK is initialized, False otherwise.
"""
with cls._lock:
return cls._initialized
@classmethod
def _force_cleanup(cls):
"""Force cleanup on program exit (failsafe)."""
with cls._lock:
if cls._initialized:
try:
logger.warning(
f"Force cleaning up Krisp SDK at exit (ref count: {cls._reference_count})"
)
krisp_audio.globalDestroy()
cls._initialized = False
except Exception as e:
logger.error(f"Error during forced Krisp SDK cleanup: {e}")

2
src/pipecat/audio/mixers/base_audio_mixer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/mixers/soundfile_mixer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/resamplers/base_audio_resampler.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/resamplers/resampy_resampler.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/resamplers/soxr_resampler.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/audio/resamplers/soxr_stream_resampler.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

18
src/pipecat/audio/turn/base_turn_analyzer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -116,7 +116,23 @@ class BaseTurnAnalyzer(ABC):
"""
pass
def update_vad_start_secs(self, vad_start_secs: float):
"""Update the VAD start trigger time.
The turn analyzer may choose to change its buffer size depending
on this value.
Args:
vad_start_secs (float): The number of seconds of voice activity
before triggering the user speaking event.
"""
pass
@abstractmethod
def clear(self):
"""Reset the turn analyzer to its initial state."""
pass
async def cleanup(self):
"""Cleanup the turn analyzer."""
pass

369
src/pipecat/audio/turn/krisp_viva_turn.py Normal file
View File

@@ -0,0 +1,369 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Krisp turn analyzer for end-of-turn detection using Krisp VIVA SDK.
This module provides a turn analyzer implementation using Krisp's turn detection
(Tt) API to determine when a user has finished speaking in a conversation.
Note: This analyzer uses a different model than KrispVivaFilter. The model path
can be specified via the KRISP_VIVA_TURN_MODEL_PATH environment variable or
passed directly to the constructor.
"""
import os
import time
from typing import Optional, Tuple
import numpy as np
from loguru import logger
from pipecat.audio.krisp_instance import (
KrispVivaSDKManager,
int_to_krisp_frame_duration,
int_to_krisp_sample_rate,
)
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData, TurnMetricsData
try:
import krisp_audio
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use KrispVivaTurn, you need to install krisp_audio.")
raise Exception(f"Missing module: {e}")
class KrispTurnParams(BaseTurnParams):
"""Configuration parameters for Krisp turn analysis.
Parameters:
threshold: Probability threshold for turn completion (0.0 to 1.0).
Higher values require more confidence before marking turn as complete.
frame_duration_ms: Frame duration in milliseconds for turn detection.
Supported values: 10, 15, 20, 30, 32.
"""
threshold: float = 0.5
frame_duration_ms: int = 20
class KrispVivaTurn(BaseTurnAnalyzer):
"""Turn analyzer using Krisp VIVA SDK for end-of-turn detection.
Uses Krisp's turn detection (Tt) API to determine when a user has finished
speaking. This analyzer requires a valid Krisp model file to operate.
"""
def __init__(
self,
*,
model_path: Optional[str] = None,
sample_rate: Optional[int] = None,
params: Optional[KrispTurnParams] = None,
api_key: str = "",
) -> None:
"""Initialize the Krisp turn analyzer.
Args:
model_path: Path to the Krisp turn detection model file (.kef extension).
If None, uses KRISP_VIVA_TURN_MODEL_PATH environment variable.
sample_rate: Optional initial sample rate for audio processing.
If provided, this will be used as the fixed sample rate.
params: Configuration parameters for turn analysis behavior.
api_key: Krisp SDK API key. If empty, falls back to
the KRISP_VIVA_API_KEY environment variable.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_TURN_MODEL_PATH is not set.
Exception: If model file doesn't have .kef extension.
FileNotFoundError: If model file doesn't exist.
RuntimeError: If Krisp SDK initialization fails.
"""
super().__init__(sample_rate=sample_rate)
# Acquire SDK reference (will initialize on first call)
try:
KrispVivaSDKManager.acquire(api_key=api_key)
self._sdk_acquired = True
except Exception as e:
self._sdk_acquired = False
raise RuntimeError(f"Failed to initialize Krisp SDK: {e}")
try:
# Set model path, checking environment if not specified
self._model_path = model_path or os.getenv("KRISP_VIVA_TURN_MODEL_PATH")
if not self._model_path:
logger.error(
"Model path is not provided and KRISP_VIVA_TURN_MODEL_PATH is not set."
)
raise ValueError("Model path for KrispVivaTurn must be provided.")
if not self._model_path.endswith(".kef"):
raise Exception("Model is expected with .kef extension")
if not os.path.isfile(self._model_path):
raise FileNotFoundError(f"Model file not found: {self._model_path}")
self._params = params or KrispTurnParams()
self._tt_session = None
self._preload_tt_session = None
self._samples_per_frame = None
self._audio_buffer = bytearray()
# State tracking
self._speech_triggered = False
self._last_probability = None
self._frame_probabilities = []
self._last_state = EndOfTurnState.INCOMPLETE
self._speech_stopped_time: Optional[float] = None
self._e2e_processing_time_ms: Optional[float] = None
self._last_metrics: Optional[TurnMetricsData] = None
# Create session with provided sample rate or default to 16000 Hz
# This preloads the model to improve latency when set_sample_rate is called later
preload_sample_rate = sample_rate if sample_rate else 16000
try:
self._preload_tt_session = self._create_tt_session(preload_sample_rate)
except Exception as e:
logger.error(f"Failed to create turn detection session: {e}", exc_info=True)
self._preload_tt_session = None
raise RuntimeError(f"Failed to create turn detection session: {e}") from e
except Exception:
# If initialization fails, release the SDK reference
if self._sdk_acquired:
KrispVivaSDKManager.release()
self._sdk_acquired = False
raise
async def cleanup(self):
"""Release SDK reference when analyzer is destroyed."""
if self._sdk_acquired:
try:
# Clean up session first
if hasattr(self, "_tt_session") and self._tt_session is not None:
self._tt_session = None
if hasattr(self, "_preload_tt_session") and self._preload_tt_session is not None:
self._preload_tt_session = None
KrispVivaSDKManager.release()
self._sdk_acquired = False
except Exception as e:
logger.error(f"Error in __del__: {e}", exc_info=True)
def _create_tt_session(self, sample_rate: int):
"""Create a turn detection session with the specified sample rate.
Args:
sample_rate: Sample rate for the session
Returns:
krisp_audio.TtFloat instance
Raises:
ValueError: If sample rate or frame duration is not supported
RuntimeError: If session creation fails
"""
try:
model_info = krisp_audio.ModelInfo()
model_info.path = self._model_path
tt_cfg = krisp_audio.TtSessionConfig()
tt_cfg.inputSampleRate = int_to_krisp_sample_rate(sample_rate)
tt_cfg.inputFrameDuration = int_to_krisp_frame_duration(self._params.frame_duration_ms)
tt_cfg.modelInfo = model_info
# Calculate samples per frame for this sample rate
self._samples_per_frame = int((sample_rate * self._params.frame_duration_ms) / 1000)
tt_instance = krisp_audio.TtFloat.create(tt_cfg)
return tt_instance
except Exception as e:
logger.error(f"Failed to create Krisp turn detection session: {e}", exc_info=True)
raise RuntimeError(f"Failed to create Krisp turn detection session: {e}") from e
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate and create/update the turn detection session.
Args:
sample_rate: The sample rate to set.
"""
if self._sample_rate == sample_rate:
return
super().set_sample_rate(sample_rate)
# Create session when sample rate is set
try:
self._tt_session = self._create_tt_session(self._sample_rate)
self.clear()
except Exception as e:
logger.error(f"Failed to create turn detection session: {e}", exc_info=True)
self._tt_session = None
@property
def frame_probabilities(self) -> list:
"""Get all probabilities from the last append_audio call.
Returns:
List of probability values for each frame processed in the last append_audio call.
"""
return self._frame_probabilities
@property
def last_probability(self) -> Optional[float]:
"""Get the last turn probability value computed.
Returns:
Last probability value, or None if no frames have been processed yet.
"""
return self._last_probability
@property
def speech_triggered(self) -> bool:
"""Check if speech has been detected and triggered analysis.
Returns:
True if speech has been detected and turn analysis is active.
"""
return self._speech_triggered
@property
def params(self) -> KrispTurnParams:
"""Get the current turn analyzer parameters.
Returns:
Current turn analyzer configuration parameters.
"""
return self._params
def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
"""Append audio data for turn analysis.
Args:
buffer: Raw audio data bytes to append for analysis.
is_speech: Whether the audio buffer contains detected speech.
Returns:
Current end-of-turn state after processing the audio.
"""
if self._tt_session is None:
logger.warning("Turn detection session not initialized, returning INCOMPLETE")
self._last_state = EndOfTurnState.INCOMPLETE
return EndOfTurnState.INCOMPLETE
if self._samples_per_frame is None:
logger.warning("Samples per frame not initialized, returning INCOMPLETE")
self._last_state = EndOfTurnState.INCOMPLETE
return EndOfTurnState.INCOMPLETE
try:
# Add incoming audio to our buffer
self._audio_buffer.extend(buffer)
# Clear frame probabilities from previous call
self._frame_probabilities = []
total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample
num_complete_frames = total_samples // self._samples_per_frame
if num_complete_frames == 0:
# Not enough samples for a complete frame yet, return current state
self._last_state = EndOfTurnState.INCOMPLETE
return EndOfTurnState.INCOMPLETE
complete_samples_count = num_complete_frames * self._samples_per_frame
bytes_to_process = complete_samples_count * 2 # 2 bytes per sample
audio_to_process = bytes(self._audio_buffer[:bytes_to_process])
self._audio_buffer = self._audio_buffer[bytes_to_process:]
audio_int16 = np.frombuffer(audio_to_process, dtype=np.int16)
audio_float32 = audio_int16.astype(np.float32) / 32768.0
frames = audio_float32.reshape(-1, self._samples_per_frame)
state = EndOfTurnState.INCOMPLETE
# Process each complete frame
for frame in frames:
if is_speech:
# Track speech start time
if not self._speech_triggered:
logger.trace("Speech detected, turn analysis started")
self._e2e_processing_time_ms = None
self._speech_triggered = True
# Reset speech stopped time when speech resumes
self._speech_stopped_time = None
else:
# Record the moment speech transitions to non-speech
if self._speech_triggered and self._speech_stopped_time is None:
self._speech_stopped_time = time.perf_counter()
# Note: We don't immediately mark as complete on silence detection.
# Instead, we wait for the model's probability check below to confirm
# end-of-turn based on the threshold.
prob = self._tt_session.process(frame.tolist())
# Negative values indicate the model is not ready yet (working with 100ms data)
# Skip processing until we get positive probabilities
if prob < 0:
continue
# Store the probability for external access
self._last_probability = prob
self._frame_probabilities.append(prob)
# Check if turn is complete based on probability threshold
# Only mark as complete if we've detected speech and the model
# confirms with sufficient confidence
if self._speech_triggered and prob >= self._params.threshold:
# Calculate e2e processing time: time from speech stop to threshold crossing
if self._speech_stopped_time is not None:
self._e2e_processing_time_ms = (
time.perf_counter() - self._speech_stopped_time
) * 1000
self._last_metrics = TurnMetricsData(
processor="KrispVivaTurn",
is_complete=True,
probability=prob,
e2e_processing_time_ms=self._e2e_processing_time_ms,
)
logger.debug(f"Krisp turn complete")
state = EndOfTurnState.COMPLETE
self.clear()
break
# Store the last state for analyze_end_of_turn()
self._last_state = state
return state
except Exception as e:
logger.error(f"Error during Krisp turn detection: {e}", exc_info=True)
error_state = EndOfTurnState.INCOMPLETE
self._last_state = error_state
return error_state
async def analyze_end_of_turn(self) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
"""Analyze the current audio state to determine if turn has ended.
Returns:
Tuple containing the end-of-turn state and optional metrics data.
Returns the last state determined by append_audio().
"""
# For real-time processing, the state is determined in append_audio.
# Consume metrics so they aren't pushed twice.
metrics = self._last_metrics
self._last_metrics = None
return self._last_state, metrics
def clear(self):
"""Reset the turn analyzer to its initial state."""
self._speech_triggered = False
self._audio_buffer.clear()
self._last_state = EndOfTurnState.INCOMPLETE
self._speech_stopped_time = None

31
src/pipecat/audio/turn/smart_turn/base_smart_turn.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -19,16 +19,14 @@ from typing import Any, Dict, Optional, Tuple
import numpy as np
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData
from pipecat.metrics.metrics import MetricsData, TurnMetricsData
# Default timing parameters
STOP_SECS = 3
PRE_SPEECH_MS = 0
PRE_SPEECH_MS = 500
MAX_DURATION_SECONDS = 8 # Max allowed segment duration
USE_ONLY_LAST_VAD_SEGMENT = True
class SmartTurnParams(BaseTurnParams):
@@ -43,8 +41,6 @@ class SmartTurnParams(BaseTurnParams):
stop_secs: float = STOP_SECS
pre_speech_ms: float = PRE_SPEECH_MS
max_duration_secs: float = MAX_DURATION_SECONDS
# not exposing this for now yet until the model can handle it.
# use_only_last_vad_segment: bool = USE_ONLY_LAST_VAD_SEGMENT
class SmartTurnTimeoutException(Exception):
@@ -82,6 +78,7 @@ class BaseSmartTurn(BaseTurnAnalyzer):
# Thread executor that will run the model. We only need one thread per
# analyzer because one analyzer just handles one audio stream.
self._executor = ThreadPoolExecutor(max_workers=1)
self._vad_start_secs: float = 0.0
@property
def speech_triggered(self) -> bool:
@@ -160,11 +157,15 @@ class BaseSmartTurn(BaseTurnAnalyzer):
state, result = await loop.run_in_executor(
self._executor, self._process_speech_segment, self._audio_buffer
)
if state == EndOfTurnState.COMPLETE or USE_ONLY_LAST_VAD_SEGMENT:
if state == EndOfTurnState.COMPLETE:
self._clear(state)
logger.debug(f"End of Turn result: {state}")
return state, result
def update_vad_start_secs(self, vad_start_secs: float):
"""Store the new vad_start_secs value."""
self._vad_start_secs = vad_start_secs
def clear(self):
"""Reset the turn analyzer to its initial state."""
self._clear(EndOfTurnState.COMPLETE)
@@ -185,7 +186,8 @@ class BaseSmartTurn(BaseTurnAnalyzer):
return state, None
# Extract recent audio segment for prediction
start_time = self._speech_start_time - (self._params.pre_speech_ms / 1000)
effective_pre_speech_ms = self._params.pre_speech_ms + (self._vad_start_secs * 1000)
start_time = self._speech_start_time - (effective_pre_speech_ms / 1000)
start_index = 0
for i, (t, _) in enumerate(audio_buffer):
if t >= start_time:
@@ -220,18 +222,11 @@ class BaseSmartTurn(BaseTurnAnalyzer):
# Calculate processing time
e2e_processing_time_ms = (end_time - start_time) * 1000
# Extract metrics from the nested structure
metrics = result.get("metrics", {})
inference_time = metrics.get("inference_time", 0)
total_time = metrics.get("total_time", 0)
# Prepare the result data
result_data = SmartTurnMetricsData(
result_data = TurnMetricsData(
processor="BaseSmartTurn",
is_complete=result["prediction"] == 1,
probability=result["probability"],
inference_time_ms=inference_time * 1000,
server_total_time_ms=total_time * 1000,
e2e_processing_time_ms=e2e_processing_time_ms,
)
@@ -239,8 +234,6 @@ class BaseSmartTurn(BaseTurnAnalyzer):
f"Prediction: {'Complete' if result_data.is_complete else 'Incomplete'}"
)
logger.trace(f"Probability of complete: {result_data.probability:.4f}")
logger.trace(f"Inference time: {result_data.inference_time_ms:.2f}ms")
logger.trace(f"Server total time: {result_data.server_total_time_ms:.2f}ms")
logger.trace(f"E2E processing time: {result_data.e2e_processing_time_ms:.2f}ms")
except SmartTurnTimeoutException:
logger.debug(

BIN
src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx → src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.2-cpu.onnx
View File

Binary file not shown.

16
src/pipecat/audio/turn/smart_turn/fal_smart_turn.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -14,6 +14,7 @@ Note: To learn more about the smart-turn model, visit:
- https://github.com/pipecat-ai/smart-turn
"""
import warnings
from typing import Optional
import aiohttp
@@ -26,6 +27,10 @@ class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
Extends HttpSmartTurnAnalyzer to provide integration with Fal.ai's
smart turn detection API endpoint with proper authentication.
.. deprecated:: 0.98.0
FalSmartTurnAnalyzer is deprecated and will be removed in a future version.
Use LocalSmartTurnAnalyzerV3 instead.
"""
def __init__(
@@ -48,3 +53,12 @@ class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
if api_key:
headers = {"Authorization": f"Key {api_key}"}
super().__init__(url=url, aiohttp_session=aiohttp_session, headers=headers, **kwargs)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"FalSmartTurnAnalyzer is deprecated and will be removed in a future version. "
"Use LocalSmartTurnAnalyzerV3 instead.",
DeprecationWarning,
stacklevel=2,
)

2
src/pipecat/audio/turn/smart_turn/http_smart_turn.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

16
src/pipecat/audio/turn/smart_turn/local_coreml_smart_turn.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -10,6 +10,7 @@ This module provides a smart turn analyzer that uses CoreML models for
local end-of-turn detection without requiring network connectivity.
"""
import warnings
from typing import Any, Dict
import numpy as np
@@ -35,6 +36,10 @@ class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
Provides end-of-turn detection using locally-stored CoreML models,
enabling offline operation without network dependencies. Optimized
for Apple Silicon and other CoreML-compatible hardware.
.. deprecated:: 0.0.106
LocalCoreMLSmartTurnAnalyzer is deprecated and will be removed in a future version.
Use LocalSmartTurnAnalyzerV3 instead.
"""
def __init__(self, *, smart_turn_model_path: str, **kwargs):
@@ -50,6 +55,15 @@ class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
"""
super().__init__(**kwargs)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"LocalCoreMLSmartTurnAnalyzer is deprecated and will be removed in a future "
"version. Use LocalSmartTurnAnalyzerV3 instead.",
DeprecationWarning,
stacklevel=2,
)
if not smart_turn_model_path:
logger.error("smart_turn_model_path is not set.")
raise Exception("smart_turn_model_path must be provided.")

16
src/pipecat/audio/turn/smart_turn/local_smart_turn.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -10,6 +10,7 @@ This module provides a smart turn analyzer that uses PyTorch models for
local end-of-turn detection without requiring network connectivity.
"""
import warnings
from typing import Any, Dict
import numpy as np
@@ -34,6 +35,10 @@ class LocalSmartTurnAnalyzer(BaseSmartTurn):
Provides end-of-turn detection using locally-stored PyTorch models,
enabling offline operation without network dependencies. Uses
Wav2Vec2-BERT architecture for audio sequence classification.
.. deprecated:: 0.0.98
LocalSmartTurnAnalyzer is deprecated and will be removed in a future version.
Use LocalSmartTurnAnalyzerV3 instead.
"""
def __init__(self, *, smart_turn_model_path: str, **kwargs):
@@ -46,6 +51,15 @@ class LocalSmartTurnAnalyzer(BaseSmartTurn):
"""
super().__init__(**kwargs)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"LocalSmartTurnAnalyzer is deprecated and will be removed in a future version. "
"Use LocalSmartTurnAnalyzerV3 instead.",
DeprecationWarning,
stacklevel=2,
)
if not smart_turn_model_path:
# Define the path to the pretrained model on Hugging Face
smart_turn_model_path = "pipecat-ai/smart-turn"

16
src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -10,6 +10,7 @@ This module provides a smart turn analyzer that uses PyTorch models for
local end-of-turn detection without requiring network connectivity.
"""
import warnings
from typing import Any, Dict
import numpy as np
@@ -41,6 +42,10 @@ class LocalSmartTurnAnalyzerV2(BaseSmartTurn):
Provides end-of-turn detection using locally-stored PyTorch models,
enabling offline operation without network dependencies. Uses
Wav2Vec2 architecture for audio sequence classification.
.. deprecated:: 0.0.106
LocalSmartTurnAnalyzerV2 is deprecated and will be removed in a future version.
Use LocalSmartTurnAnalyzerV3 instead.
"""
def __init__(self, *, smart_turn_model_path: str, **kwargs):
@@ -53,6 +58,15 @@ class LocalSmartTurnAnalyzerV2(BaseSmartTurn):
"""
super().__init__(**kwargs)
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"LocalSmartTurnAnalyzerV2 is deprecated and will be removed in a future version. "
"Use LocalSmartTurnAnalyzerV3 instead.",
DeprecationWarning,
stacklevel=2,
)
if not smart_turn_model_path:
# Define the path to the pretrained model on Hugging Face
smart_turn_model_path = "pipecat-ai/smart-turn-v2"

103
src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -13,19 +13,16 @@ local end-of-turn detection without requiring network connectivity.
from typing import Any, Dict, Optional
import numpy as np
import onnxruntime as ort
import soxr
from loguru import logger
from transformers import WhisperFeatureExtractor
from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn
from pipecat.utils.env import env_truthy
try:
import onnxruntime as ort
from transformers import WhisperFeatureExtractor
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use LocalSmartTurnAnalyzerV3, you need to `pip install pipecat-ai[local-smart-turn-v3]`."
)
raise Exception(f"Missing module: {e}")
# The Whisper-based ONNX model expects 16 kHz audio input.
_MODEL_SAMPLE_RATE = 16000
class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
@@ -42,17 +39,17 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
Args:
smart_turn_model_path: Path to the ONNX model file. If this is not
set, the bundled smart-turn-v3.0 model will be used.
set, the bundled smart-turn-v3.2-cpu model will be used.
cpu_count: The number of CPUs to use for inference. Defaults to 1.
**kwargs: Additional arguments passed to BaseSmartTurn.
"""
super().__init__(**kwargs)
logger.debug("Loading Local Smart Turn v3 model...")
self._log_data = env_truthy("PIPECAT_SMART_TURN_LOG_DATA", default=False)
if not smart_turn_model_path:
# Load bundled model
model_name = "smart-turn-v3.0.onnx"
model_name = "smart-turn-v3.2-cpu.onnx"
package_path = "pipecat.audio.turn.smart_turn.data"
try:
@@ -70,6 +67,8 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
impresources.files(package_path).joinpath(model_name)
)
logger.debug(f"Loading Local Smart Turn v3.x model from {smart_turn_model_path}...")
so = ort.SessionOptions()
so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
so.inter_op_num_threads = 1
@@ -79,12 +78,72 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
self._feature_extractor = WhisperFeatureExtractor(chunk_length=8)
self._session = ort.InferenceSession(smart_turn_model_path, sess_options=so)
logger.debug("Loaded Local Smart Turn v3")
logger.debug("Loaded Local Smart Turn v3.x")
def _write_audio_to_wav(
self, audio_array: np.ndarray, sample_rate: int = _MODEL_SAMPLE_RATE, suffix: str = ""
) -> None:
"""Write audio data to a WAV file in a background thread.
Args:
audio_array: The audio data as a numpy array (float32, normalized to [-1, 1]).
sample_rate: The sample rate of the audio data.
suffix: Optional suffix to append to the filename (e.g., "_raw", "_padded").
"""
import os
import threading
import wave
from datetime import datetime
# Generate filename with current timestamp (millisecond precision)
timestamp = datetime.now().strftime("%Y-%m-%d__%H:%M:%S.%f")[:-3]
log_dir = "./smart_turn_audio_log"
os.makedirs(log_dir, exist_ok=True)
filename = os.path.join(log_dir, f"{timestamp}{suffix}.wav")
# Make a copy of the audio data to avoid issues with the array being modified
audio_copy = audio_array.copy()
def write_wav():
try:
# Convert float32 audio to int16 for WAV file
audio_int16 = (audio_copy * 32767).astype(np.int16)
with wave.open(filename, "wb") as wav_file:
wav_file.setnchannels(1) # Mono
wav_file.setsampwidth(2) # 2 bytes for int16
wav_file.setframerate(sample_rate)
wav_file.writeframes(audio_int16.tobytes())
logger.debug(f"Wrote audio to {filename}")
except Exception as e:
logger.error(f"Failed to write audio to {filename}: {e}")
# Start background thread to write the WAV file
thread = threading.Thread(target=write_wav, daemon=True)
thread.start()
def _resample_to_model_rate(self, audio_array: np.ndarray) -> np.ndarray:
"""Resample audio to the model's expected sample rate (16 kHz).
Args:
audio_array: Audio data as a float32 numpy array.
Returns:
Resampled audio array at 16 kHz.
"""
actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
if actual_rate == _MODEL_SAMPLE_RATE:
return audio_array
return soxr.resample(audio_array, actual_rate, _MODEL_SAMPLE_RATE, quality="VHQ")
def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
"""Predict end-of-turn using local ONNX model."""
def truncate_audio_to_last_n_seconds(audio_array, n_seconds=8, sample_rate=16000):
def truncate_audio_to_last_n_seconds(
audio_array, n_seconds=8, sample_rate=_MODEL_SAMPLE_RATE
):
"""Truncate audio to last n seconds or pad with zeros to meet n seconds."""
max_samples = n_seconds * sample_rate
if len(audio_array) > max_samples:
@@ -95,16 +154,22 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
return np.pad(audio_array, (padding, 0), mode="constant", constant_values=0)
return audio_array
audio_for_logging = audio_array
actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
# Resample to 16 kHz if the pipeline uses a different sample rate
audio_array = self._resample_to_model_rate(audio_array)
# Truncate to 8 seconds (keeping the end) or pad to 8 seconds
audio_array = truncate_audio_to_last_n_seconds(audio_array, n_seconds=8)
# Process audio using Whisper's feature extractor
inputs = self._feature_extractor(
audio_array,
sampling_rate=16000,
sampling_rate=_MODEL_SAMPLE_RATE,
return_tensors="np",
padding="max_length",
max_length=8 * 16000,
max_length=8 * _MODEL_SAMPLE_RATE,
truncation=True,
do_normalize=True,
)
@@ -122,6 +187,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
# Make prediction (1 for Complete, 0 for Incomplete)
prediction = 1 if probability > 0.5 else 0
if self._log_data:
suffix = "_complete" if prediction == 1 else "_incomplete"
self._write_audio_to_wav(audio_for_logging, sample_rate=actual_rate, suffix=suffix)
return {
"prediction": prediction,
"probability": probability,

2
src/pipecat/audio/utils.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

153
src/pipecat/audio/vad/aic_vad.py
View File

@@ -1,44 +1,44 @@
"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend.
This analyzer queries the backend's is_speech_detected() and maps it to a float
confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies
optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available.
This module provides VAD analyzer implementations that query the AIC SDK's
is_speech_detected() and map it to a float confidence (1.0/0.0).
Classes:
AICVADAnalyzer: For aic-sdk (uses 'aic_sdk' module)
"""
from typing import Any, Callable, Optional
from aic_sdk import VadParameter
from loguru import logger
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
try:
from aic import AICVadParameter
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
raise Exception(f"Missing module: {e}")
class AICVADAnalyzer(VADAnalyzer):
"""VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory.
"""VAD analyzer that lazily binds to the AIC VadContext via a factory.
The analyzer can be constructed before the AIC Model exists. Once the filter has
started and the Model is available, the provided factory will succeed and the
backend VAD will be created. We then switch to single-sample updates where
num_frames_required() returns 1 and confidence is derived from the backend's
boolean is_speech_detected() state.
The analyzer can be constructed before the AIC Processor exists. Once the filter has
started and the Processor is available, the provided factory will succeed and the
VadContext will be obtained. The context's is_speech_detected() boolean state is
then mapped to 1.0 (speech) or 0.0 (no speech) to satisfy the VADAnalyzer interface.
AIC VAD runtime parameters:
- lookback_buffer_size:
Controls the lookback buffer size used by the VAD, i.e. the number of
window-length audio buffers used as a lookback buffer. Larger values improve
stability but increase latency.
Range: 1.0 .. 20.0
Default (SDK): 6.0
- speech_hold_duration:
Controls for how long the VAD continues to detect speech after the audio signal
no longer contains speech (in seconds).
Range: 0.0 to 100x model window length
Default (SDK): 0.05s (50ms)
- minimum_speech_duration:
Controls for how long speech needs to be present in the audio signal before the
VAD considers it speech (in seconds).
Range: 0.0 to 1.0
Default (SDK): 0.0s
- sensitivity:
Controls the energy threshold sensitivity. Higher values make the detector
less sensitive (require more energy to count as speech).
Range: 1.0 .. 15.0
Controls the sensitivity (energy threshold) of the VAD. This value is used by
the VAD as the threshold a speech audio signal's energy has to exceed in order
to be considered speech.
Range: 1.0 to 15.0
Formula: Energy threshold = 10 ** (-sensitivity)
Default (SDK): 6.0
"""
@@ -46,69 +46,80 @@ class AICVADAnalyzer(VADAnalyzer):
def __init__(
self,
*,
vad_factory: Optional[Callable[[], Any]] = None,
lookback_buffer_size: Optional[float] = None,
vad_context_factory: Optional[Callable[[], Any]] = None,
speech_hold_duration: Optional[float] = None,
minimum_speech_duration: Optional[float] = None,
sensitivity: Optional[float] = None,
):
"""Create an AIC VAD analyzer.
Args:
vad_factory:
Zero-arg callable that returns an initialized AIC VoiceActivityDetector.
This may raise until the filter's Model has been created; the analyzer
vad_context_factory:
Zero-arg callable that returns the AIC VadContext.
This may raise until the filter's Processor has been created; the analyzer
will retry on set_sample_rate/first use.
lookback_buffer_size:
Optional override for AIC VAD lookback buffer size.
Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency.
If None, the SDK default (6.0) is used.
speech_hold_duration:
Optional override for AIC VAD speech hold duration (in seconds).
Range: 0.0 to 100x model window length.
If None, the SDK default (0.05s) is used.
minimum_speech_duration:
Optional override for minimum speech duration before VAD reports
speech detected (in seconds).
Range: 0.0 to 1.0.
If None, the SDK default (0.0s) is used.
sensitivity:
Optional override for AIC VAD sensitivity (energy threshold).
Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity).
Range: 1.0 to 15.0. Energy threshold = 10 ** (-sensitivity).
If None, the SDK default (6.0) is used.
"""
# Use fixed VAD parameters for AIC: no user override
fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0)
super().__init__(sample_rate=None, params=fixed_params)
self._vad_factory = vad_factory
self._backend_vad: Optional[Any] = None
self._pending_lookback: Optional[float] = lookback_buffer_size
self._vad_context_factory = vad_context_factory
self._vad_ctx: Optional[Any] = None
self._pending_speech_hold_duration: Optional[float] = speech_hold_duration
self._pending_minimum_speech_duration: Optional[float] = minimum_speech_duration
self._pending_sensitivity: Optional[float] = sensitivity
def bind_vad_factory(self, vad_factory: Callable[[], Any]):
def bind_vad_context_factory(self, vad_context_factory: Callable[[], Any]):
"""Attach or replace the factory post-construction."""
self._vad_factory = vad_factory
self._ensure_backend_initialized()
self._vad_context_factory = vad_context_factory
self._ensure_vad_context_initialized()
def _apply_backend_params(self):
def _apply_vad_params(self):
"""Apply optional AIC VAD parameters if available."""
if self._backend_vad is None or AICVadParameter is None:
if self._vad_ctx is None or VadParameter is None:
return
try:
if self._pending_lookback is not None:
self._backend_vad.set_parameter(
AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback)
if self._pending_speech_hold_duration is not None:
self._vad_ctx.set_parameter(
VadParameter.SpeechHoldDuration, self._pending_speech_hold_duration
)
if self._pending_minimum_speech_duration is not None:
self._vad_ctx.set_parameter(
VadParameter.MinimumSpeechDuration, self._pending_minimum_speech_duration
)
if self._pending_sensitivity is not None:
self._backend_vad.set_parameter(
AICVadParameter.SENSITIVITY, float(self._pending_sensitivity)
)
self._vad_ctx.set_parameter(VadParameter.Sensitivity, self._pending_sensitivity)
except Exception as e: # noqa: BLE001
logger.debug(f"AIC VAD parameter application deferred/failed: {e}")
def _ensure_backend_initialized(self):
if self._backend_vad is not None:
def _ensure_vad_context_initialized(self):
if self._vad_ctx is not None:
return
if not self._vad_factory:
if not self._vad_context_factory:
return
try:
self._backend_vad = self._vad_factory()
self._apply_backend_params()
# With backend ready, recompute internal frame sizing
self._vad_ctx = self._vad_context_factory()
self._apply_vad_params()
# With VAD context ready, recompute internal frame sizing
super().set_params(self._params)
logger.debug("AIC VAD backend initialized in analyzer.")
logger.debug("AIC VAD context initialized in analyzer.")
except Exception as e: # noqa: BLE001
# Filter may not be started yet; try again later
logger.debug(f"Deferring AIC VAD backend initialization: {e}")
logger.debug(f"Deferring AIC VAD context initialization: {e}")
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate for audio processing.
@@ -116,10 +127,10 @@ class AICVADAnalyzer(VADAnalyzer):
Args:
sample_rate: Audio sample rate in Hz.
"""
# Set rate and attempt backend initialization once we know SR
# Set rate and attempt VAD context initialization once we know SR
self._sample_rate = self._init_sample_rate or sample_rate
self._ensure_backend_initialized()
# Ensure params are initialized even if backend not ready yet
self._ensure_vad_context_initialized()
# Ensure params are initialized even if VAD context not ready yet
try:
super().set_params(self._params)
except Exception:
@@ -135,23 +146,29 @@ class AICVADAnalyzer(VADAnalyzer):
return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160
def voice_confidence(self, buffer: bytes) -> float:
"""Calculate voice activity confidence for the given audio buffer.
"""Return voice activity detection result for the given audio buffer.
Note:
The AIC SDK provides binary speech detection (not a probability score).
This method returns 1.0 when speech is detected and 0.0 otherwise,
rather than a true confidence value.
Args:
buffer: Audio buffer to analyze.
buffer: Audio buffer (unused - AIC VAD state is updated internally
by the enhancement pipeline).
Returns:
Voice confidence score is 0.0 or 1.0.
1.0 if speech is detected, 0.0 otherwise.
"""
# Ensure backend exists (filter might have started since last call)
self._ensure_backend_initialized()
if self._backend_vad is None:
# Ensure VAD context exists (filter might have started since last call)
self._ensure_vad_context_initialized()
if self._vad_ctx is None:
return 0.0
# We do not need to analyze 'buffer' here since the model's VAD is updated
# We do not need to analyze 'buffer' here since the processor's VAD is updated
# as part of the enhancement pipeline. Simply query the boolean and map it.
try:
is_speech = self._backend_vad.is_speech_detected()
is_speech = self._vad_ctx.is_speech_detected()
return 1.0 if is_speech else 0.0
except Exception as e: # noqa: BLE001
logger.error(f"AIC VAD inference error: {e}")

217
src/pipecat/audio/vad/krisp_viva_vad.py Normal file
View File

@@ -0,0 +1,217 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Krisp Voice Activity Detection (VAD) implementation for Pipecat.
This module provides a VAD analyzer based on the Krisp VIVA SDK,
which can detect voice activity in audio streams with high accuracy.
Supports 8kHz, 16kHz, 32kHz, 44.1kHz and 48kHz sample rates.
"""
import os
from typing import Optional
import numpy as np
from loguru import logger
from pipecat.audio.krisp_instance import (
KrispVivaSDKManager,
int_to_krisp_frame_duration,
int_to_krisp_sample_rate,
)
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
try:
import krisp_audio
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use KrispVivaVADAnalyzer, you need to install krisp_audio.")
raise Exception(f"Missing module: {e}")
class KrispVivaVadAnalyzer(VADAnalyzer):
"""Voice Activity Detection analyzer using the Krisp VIVA SDK."""
def __init__(
self,
*,
model_path: Optional[str] = None,
frame_duration: int = 10,
sample_rate: Optional[int] = None,
params: Optional[VADParams] = None,
):
"""Initialize the Krisp VIVA VAD analyzer.
Args:
model_path: Path to the Krisp model file (.kef extension).
If None, uses KRISP_VIVA_VAD_MODEL_PATH environment variable.
frame_duration: Frame duration in milliseconds (default: 10ms).
sample_rate: Audio sample rate (must be 8000, 16000, 32000, 44100 or 48000 Hz).
If None, will be set later.
params: VAD parameters for detection configuration.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_VAD_MODEL_PATH is not set.
Exception: If model file doesn't have .kef extension.
FileNotFoundError: If model file doesn't exist.
"""
super().__init__(sample_rate=sample_rate, params=params)
logger.debug("Loading Krisp VIVA VAD model...")
try:
# Set model path, checking environment if not specified
if model_path:
self._model_path = model_path
else:
self._model_path = os.getenv("KRISP_VIVA_VAD_MODEL_PATH")
if not self._model_path:
logger.error(
"Model path is not provided and KRISP_VIVA_VAD_MODEL_PATH is not set."
)
raise ValueError("Model path for KrispVivaVADAnalyzer must be provided.")
if not self._model_path.endswith(".kef"):
raise Exception("Model is expected with .kef extension")
if not os.path.isfile(self._model_path):
raise FileNotFoundError(f"Model file not found: {self._model_path}")
self._session = None
self._frame_duration_ms = frame_duration
self._samples_per_frame = None
# Calculate samples per frame if sample_rate is provided
if sample_rate is not None:
self._samples_per_frame = int((sample_rate * frame_duration) / 1000)
# Acquire SDK reference (will initialize on first call)
KrispVivaSDKManager.acquire()
logger.debug("Loaded Krisp VIVA VAD")
except Exception:
# If initialization fails, release the SDK reference
KrispVivaSDKManager.release()
raise
def _create_session(self, sample_rate: int, frame_duration: int):
"""Create a Krisp VAD session with a specific sample rate.
Args:
sample_rate: Sample rate for the session
frame_duration: Frame duration in milliseconds
Returns:
Krisp VAD session instance
Raises:
RuntimeError: If session creation fails
"""
try:
model_info = krisp_audio.ModelInfo()
model_info.path = self._model_path
vad_cfg = krisp_audio.VadSessionConfig()
vad_cfg.inputSampleRate = int_to_krisp_sample_rate(sample_rate)
vad_cfg.inputFrameDuration = int_to_krisp_frame_duration(frame_duration)
vad_cfg.modelInfo = model_info
self._samples_per_frame = int((sample_rate * frame_duration) / 1000)
session = krisp_audio.VadFloat.create(vad_cfg)
return session
except Exception as e:
logger.error(f"Failed to create Krisp VAD session: {e}", exc_info=True)
raise RuntimeError(f"Failed to create Krisp VAD session: {e}") from e
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate for audio processing.
Args:
sample_rate: Audio sample rate (must be 8000, 16000, 32000 or 48000 Hz).
Raises:
ValueError: If sample rate is not 8000, 16000, 32000 or 48000 Hz.
RuntimeError: If VAD session creation fails.
"""
if (
sample_rate != 48000
and sample_rate != 44100
and sample_rate != 32000
and sample_rate != 16000
and sample_rate != 8000
):
raise ValueError(
f"Krisp VIVA VAD sample rate needs to be 8000, 16000, 32000, 44100 or 48000 (sample rate: {sample_rate})"
)
# Create or recreate session with new sample rate
try:
self._session = self._create_session(sample_rate, self._frame_duration_ms)
except Exception as e:
logger.error(f"Failed to set sample rate: {e}", exc_info=True)
raise RuntimeError(f"Failed to create Krisp VAD session: {e}") from e
super().set_sample_rate(sample_rate)
def num_frames_required(self) -> int:
"""Get the number of audio frames required for analysis.
Returns:
Number of frames (samples) needed for VAD processing based on
current sample rate and frame duration.
"""
# If already calculated from session creation, return it
if self._samples_per_frame is not None:
return self._samples_per_frame
# Calculate from current sample rate if available
if self.sample_rate > 0:
return int((self.sample_rate * self._frame_duration_ms) / 1000)
# Fallback: calculate from initial sample rate if provided
if self._init_sample_rate is not None:
return int((self._init_sample_rate * self._frame_duration_ms) / 1000)
# Default fallback: assume 16kHz @ 10ms = 160 samples
return int((16000 * self._frame_duration_ms) / 1000)
def voice_confidence(self, buffer) -> float:
"""Calculate voice activity confidence for the given audio buffer.
Args:
buffer: Audio buffer to analyze (bytes, int16 format).
Returns:
Voice confidence score between 0.0 and 1.0.
"""
if self._session is None:
logger.warning("VAD session not initialized. Cannot process audio.")
return 0.0
try:
# Convert bytes buffer to float32 numpy array
# Buffer is int16 (2 bytes per sample), need to convert to float32
audio_int16 = np.frombuffer(buffer, dtype=np.int16)
# Normalize to [-1.0, 1.0] range
audio_float32 = audio_int16.astype(np.float32) / 32768.0
# Process through VAD session
voice_probability = self._session.process(audio_float32)
return voice_probability
except Exception as e:
logger.error(f"Error analyzing audio with Krisp VIVA VAD: {e}", exc_info=True)
return 0.0
async def cleanup(self):
"""Cleanup analyzer resources."""
try:
self._session = None
KrispVivaSDKManager.release()
except Exception:
# Ignore errors during cleanup
pass

4
src/pipecat/audio/vad/silero.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -27,7 +27,7 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Silero VAD, you need to `pip install pipecat-ai[silero]`.")
logger.error("In order to use Silero VAD, you need to `pip install pipecat-ai`.")
raise Exception(f"Missing module(s): {e}")

15
src/pipecat/audio/vad/vad_analyzer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -24,7 +24,7 @@ from pipecat.audio.utils import calculate_audio_volume, exp_smoothing
VAD_CONFIDENCE = 0.7
VAD_START_SECS = 0.2
VAD_STOP_SECS = 0.8
VAD_STOP_SECS = 0.2
VAD_MIN_VOLUME = 0.6
@@ -127,7 +127,7 @@ class VADAnalyzer(ABC):
pass
@abstractmethod
def voice_confidence(self, buffer) -> float:
def voice_confidence(self, buffer: bytes) -> float:
"""Calculate voice activity confidence for the given audio buffer.
Args:
@@ -242,3 +242,12 @@ class VADAnalyzer(ABC):
self._vad_stopping_count = 0
return self._vad_state
async def cleanup(self):
"""Clean up resources.
This method should be called when the object is no longer needed.
It waits for all currently executing event handler tasks to finish
before returning.
"""
pass

181
src/pipecat/audio/vad/vad_controller.py Normal file
View File

@@ -0,0 +1,181 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Voice Activity Detection controller for managing speech state transitions.
This module provides a controller that wraps a VADAnalyzer to track speech state
and emit events when speech starts, stops, or is actively detected.
"""
import time
from typing import Type
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
from pipecat.frames.frames import (
Frame,
InputAudioRawFrame,
SpeechControlParamsFrame,
StartFrame,
VADParamsUpdateFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.utils.base_object import BaseObject
class VADController(BaseObject):
"""Manages voice activity detection state and emits speech events.
Wraps a `VADAnalyzer` to process audio and trigger events based on speech
state transitions. Tracks whether the user is speaking, quiet, or
transitioning between states.
Event handlers available:
- on_speech_started: Called when speech begins.
- on_speech_stopped: Called when speech ends.
- on_speech_activity: Called periodically while speech is detected.
- on_push_frame: Called when the controller wants to push a frame.
- on_broadcast_frame: Called when the controller wants to broadcast a frame.
Example::
@vad_controller.event_handler("on_speech_started")
async def on_speech_started(controller):
...
@vad_controller.event_handler("on_speech_stopped")
async def on_speech_stopped(controller):
...
@vad_controller.event_handler("on_speech_activity")
async def on_speech_activity(controller):
...
@vad_controller.event_handler("on_push_frame")
async def on_push_frame(controller, frame: Frame, direction: FrameDirection):
...
@vad_controller.event_handler("on_broadcast_frame")
async def on_broadcast_frame(controller, frame_cls: Type[Frame], **kwargs):
...
"""
def __init__(self, vad_analyzer: VADAnalyzer, *, speech_activity_period: float = 0.2):
"""Initialize the VAD controller.
Args:
vad_analyzer: The `VADAnalyzer` instance for processing audio.
speech_activity_period: Minimum interval in seconds between
`on_speech_activity` events. Defaults to 0.2.
"""
super().__init__()
self._vad_analyzer = vad_analyzer
self._vad_state: VADState = VADState.QUIET
# Last time a on_speech_activity was triggered.
self._speech_activity_time = 0
# How often a on_speech_activity event should be triggered (value should
# be greater than the audio chunks to have any effect).
self._speech_activity_period = speech_activity_period
self._register_event_handler("on_speech_started", sync=True)
self._register_event_handler("on_speech_stopped", sync=True)
self._register_event_handler("on_speech_activity", sync=True)
self._register_event_handler("on_push_frame", sync=True)
self._register_event_handler("on_broadcast_frame", sync=True)
async def process_frame(self, frame: Frame):
"""Process a frame and handle VAD-related events.
Handles `StartFrame` to initialize the sample rate and `InputAudioRawFrame`
to analyze audio for voice activity.
Args:
frame: The frame to process.
"""
if isinstance(frame, StartFrame):
await self._start(frame)
elif isinstance(frame, InputAudioRawFrame):
await self._handle_audio(frame)
elif isinstance(frame, VADParamsUpdateFrame):
self._vad_analyzer.set_params(frame.params)
await self.broadcast_frame(SpeechControlParamsFrame, vad_params=frame.params)
async def _start(self, frame: StartFrame):
self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate)
# Broadcast initial VAD params so other services (e.g. STT) can use them
await self.broadcast_frame(SpeechControlParamsFrame, vad_params=self._vad_analyzer.params)
async def cleanup(self):
"""Clean up resources.
This method should be called when the object is no longer needed.
It waits for all currently executing event handler tasks to finish
before returning.
"""
if self._vad_analyzer:
await self._vad_analyzer.cleanup()
async def _handle_audio(self, frame: InputAudioRawFrame):
"""Process an audio chunk and emit speech events as needed.
Analyzes the audio for voice activity and triggers `on_speech_started`,
`on_speech_stopped`, or `on_speech_activity` events based on state changes.
Args:
frame: Audio frame to process.
"""
self._vad_state = await self._handle_vad(frame.audio, self._vad_state)
if self._vad_state == VADState.SPEAKING:
await self._call_event_handler("on_speech_activity")
async def _handle_vad(self, audio: bytes, vad_state: VADState) -> VADState:
"""Handle Voice Activity Detection results and trigger appropriate events."""
new_vad_state = await self._vad_analyzer.analyze_audio(audio)
if (
new_vad_state != vad_state
and new_vad_state != VADState.STARTING
and new_vad_state != VADState.STOPPING
):
if new_vad_state == VADState.SPEAKING:
await self._call_event_handler("on_speech_started")
elif new_vad_state == VADState.QUIET:
await self._call_event_handler("on_speech_stopped")
vad_state = new_vad_state
return vad_state
async def _maybe_speech_activity(self):
"""Handle user speaking frame."""
diff_time = time.time() - self._speech_activity_time
if diff_time >= self._speech_activity_period:
self._speech_activity_time = time.time()
await self._call_event_handler("on_speech_activity")
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Request a frame to be pushed through the pipeline.
This emits an on_push_frame event that must be handled by a processor
to actually push the frame into the pipeline.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
await self._call_event_handler("on_push_frame", frame, direction)
async def broadcast_frame(self, frame_cls: Type[Frame], **kwargs):
"""Request a frame to be broadcast upstream and downstream.
This emits an on_broadcast_frame event that must be handled by a processor
to actually broadcast the frame in the pipeline.
Args:
frame_cls: The class of the frame to broadcast.
**kwargs: Arguments to pass to the frame constructor.
"""
await self._call_event_handler("on_broadcast_frame", frame_cls, **kwargs)

2
src/pipecat/clocks/base_clock.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/clocks/system_clock.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

42
src/pipecat/extensions/ivr/ivr_navigator.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -18,8 +18,11 @@ from loguru import logger
from pipecat.audio.dtmf.types import KeypadEntry
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.frames.frames import (
AggregatedTextFrame,
EndFrame,
Frame,
LLMContextFrame,
LLMFullResponseEndFrame,
LLMMessagesUpdateFrame,
LLMTextFrame,
OutputDTMFUrgentFrame,
@@ -31,7 +34,11 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.llm_service import LLMService
from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator
from pipecat.utils.text.pattern_pair_aggregator import (
MatchAction,
PatternMatch,
PatternPairAggregator,
)
class IVRStatus(Enum):
@@ -114,15 +121,15 @@ class IVRProcessor(FrameProcessor):
def _setup_xml_patterns(self):
"""Set up XML pattern detection and handlers."""
# Register DTMF pattern
self._aggregator.add_pattern_pair("dtmf", "<dtmf>", "</dtmf>", remove_match=True)
self._aggregator.add_pattern("dtmf", "<dtmf>", "</dtmf>", action=MatchAction.REMOVE)
self._aggregator.on_pattern_match("dtmf", self._handle_dtmf_action)
# Register mode pattern
self._aggregator.add_pattern_pair("mode", "<mode>", "</mode>", remove_match=True)
self._aggregator.add_pattern("mode", "<mode>", "</mode>", action=MatchAction.REMOVE)
self._aggregator.on_pattern_match("mode", self._handle_mode_action)
# Register IVR pattern
self._aggregator.add_pattern_pair("ivr", "<ivr>", "</ivr>", remove_match=True)
self._aggregator.add_pattern("ivr", "<ivr>", "</ivr>", action=MatchAction.REMOVE)
self._aggregator.on_pattern_match("ivr", self._handle_ivr_action)
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -145,10 +152,23 @@ class IVRProcessor(FrameProcessor):
elif isinstance(frame, LLMTextFrame):
# Process text through the pattern aggregator
result = await self._aggregator.aggregate(frame.text)
if result:
async for result in self._aggregator.aggregate(frame.text):
# Push aggregated text that doesn't contain XML patterns
await self.push_frame(LLMTextFrame(result), direction)
await self.push_frame(
AggregatedTextFrame(text=result.text, aggregated_by=result.type),
direction,
)
elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
# Flush any remaining text from the aggregator
remaining = await self._aggregator.flush()
if remaining:
await self.push_frame(
AggregatedTextFrame(text=remaining.text, aggregated_by=remaining.type),
direction,
)
# Push the end frame
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
@@ -159,7 +179,7 @@ class IVRProcessor(FrameProcessor):
Args:
match: The pattern match containing DTMF content.
"""
value = match.content
value = match.text
logger.debug(f"DTMF detected: {value}")
try:
@@ -180,7 +200,7 @@ class IVRProcessor(FrameProcessor):
Args:
match: The pattern match containing IVR status content.
"""
status = match.content
status = match.text
logger.trace(f"IVR status detected: {status}")
# Convert string to enum, with validation
@@ -211,7 +231,7 @@ class IVRProcessor(FrameProcessor):
Args:
match: The pattern match containing mode content.
"""
mode = match.content
mode = match.text
logger.debug(f"Mode detected: {mode}")
if mode == "conversation":
await self._handle_conversation()

30
src/pipecat/extensions/voicemail/voicemail_detector.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -37,11 +37,15 @@ from pipecat.frames.frames import (
)
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
from pipecat.processors.aggregators.llm_response_universal import (
LLMContextAggregatorPair,
LLMUserAggregatorParams,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
from pipecat.services.llm_service import LLMService
from pipecat.sync.base_notifier import BaseNotifier
from pipecat.sync.event_notifier import EventNotifier
from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies
from pipecat.utils.sync.base_notifier import BaseNotifier
from pipecat.utils.sync.event_notifier import EventNotifier
class NotifierGate(FrameProcessor):
@@ -252,7 +256,8 @@ class ClassificationProcessor(FrameProcessor):
self._voicemail_notifier = voicemail_notifier
self._voicemail_response_delay = voicemail_response_delay
# Register the voicemail detected event
# Register the conversation and voicemail detected events
self._register_event_handler("on_conversation_detected")
self._register_event_handler("on_voicemail_detected")
# Aggregation state for collecting complete LLM responses
@@ -317,11 +322,13 @@ class ClassificationProcessor(FrameProcessor):
# User started speaking - set the voicemail event
if self._voicemail_detected:
self._voicemail_event.set()
await self.push_frame(frame, direction)
elif isinstance(frame, UserStoppedSpeakingFrame):
# User stopped speaking - clear the voicemail event
if self._voicemail_detected:
self._voicemail_event.clear()
await self.push_frame(frame, direction)
else:
# Pass all non-LLM frames through
@@ -350,6 +357,7 @@ class ClassificationProcessor(FrameProcessor):
logger.info(f"{self}: CONVERSATION detected")
await self._gate_notifier.notify() # Close the classifier gate
await self._conversation_notifier.notify() # Release buffered TTS frames
await self._call_event_handler("on_conversation_detected")
elif "VOICEMAIL" in response:
# Voicemail detected - trigger voicemail handling
@@ -360,7 +368,7 @@ class ClassificationProcessor(FrameProcessor):
await self._voicemail_notifier.notify() # Clear buffered TTS frames
# Interrupt the current pipeline to stop any ongoing processing
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
# Set the voicemail event to trigger the voicemail handler
self._voicemail_event.clear()
@@ -539,6 +547,9 @@ class VoicemailDetector(ParallelPipeline):
custom_prompt = "Your custom classification logic here. " + VoicemailDetector.CLASSIFIER_RESPONSE_INSTRUCTION
Events:
on_conversation_detected: Triggered when a human conversation is detected. The
event handler receives one argument: the ClassificationProcessor instance
which can be used to push frames.
on_voicemail_detected: Triggered when voicemail is detected after the configured
delay. The event handler receives one argument: the ClassificationProcessor
instance which can be used to push frames.
@@ -616,7 +627,10 @@ VOICEMAIL SYSTEM (respond "VOICEMAIL"):
# Create the LLM context and aggregators for conversation management
self._context = LLMContext(self._messages)
self._context_aggregator = LLMContextAggregatorPair(self._context)
self._context_aggregator = LLMContextAggregatorPair(
self._context,
user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()),
)
# Create notification system for coordinating between components
self._gate_notifier = EventNotifier() # Signals classification completion
@@ -701,7 +715,7 @@ VOICEMAIL SYSTEM (respond "VOICEMAIL"):
event_name: The name of the event to handle.
handler: The function to call when the event occurs.
"""
if event_name == "on_voicemail_detected":
if event_name in ("on_conversation_detected", "on_voicemail_detected"):
self._classification_processor.add_event_handler(event_name, handler)
else:
super().add_event_handler(event_name, handler)

2
src/pipecat/frames/frames.proto
View File

@@ -1,5 +1,5 @@
//
// Copyright (c) 2024–2025, Daily
// Copyright (c) 2024–2026, Daily
//
// SPDX-License-Identifier: BSD 2-Clause License
//

841
src/pipecat/frames/frames.py
View File

File diff suppressed because it is too large Load Diff

8
src/pipecat/frames/protobufs/frames_pb2.py
View File

@@ -2,7 +2,7 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: frames.proto
# Protobuf Python Version: 5.27.2
# Protobuf Python Version: 6.31.1
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
@@ -11,9 +11,9 @@ from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
5,
27,
2,
6,
31,
1,
'',
'frames.proto'
)

41
src/pipecat/metrics/metrics.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -87,19 +87,44 @@ class TTSUsageMetricsData(MetricsData):
value: int
class SmartTurnMetricsData(MetricsData):
"""Metrics data for smart turn predictions.
class TextAggregationMetricsData(MetricsData):
"""Text aggregation time metrics data.
Measures the time from the first LLM token to the first complete sentence,
representing the latency cost of sentence aggregation in the TTS pipeline.
Parameters:
value: Aggregation time in seconds.
"""
value: float
class TurnMetricsData(MetricsData):
"""Metrics data for turn detection predictions.
Parameters:
is_complete: Whether the turn is predicted to be complete.
probability: Confidence probability of the turn completion prediction.
inference_time_ms: Time taken for inference in milliseconds.
server_total_time_ms: Total server processing time in milliseconds.
e2e_processing_time_ms: End-to-end processing time in milliseconds.
e2e_processing_time_ms: End-to-end processing time in milliseconds,
measured from VAD speech-to-silence transition to turn completion.
"""
is_complete: bool
probability: float
inference_time_ms: float
server_total_time_ms: float
e2e_processing_time_ms: float
class SmartTurnMetricsData(TurnMetricsData):
"""Metrics data for smart turn predictions.
.. deprecated:: 0.0.104
Use :class:`TurnMetricsData` instead. This class will be removed in a future version.
Parameters:
inference_time_ms: Time taken for inference in milliseconds.
server_total_time_ms: Total server processing time in milliseconds.
"""
inference_time_ms: float = 0.0
server_total_time_ms: float = 0.0

10
src/pipecat/observers/base_observer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -100,3 +100,11 @@ class BaseObserver(BaseObject):
data: The event data containing details about the frame transfer.
"""
pass
async def on_pipeline_started(self):
"""Called when the pipeline has fully started.
Fired after the ``StartFrame`` has been processed by all processors
in the pipeline, including nested ``ParallelPipeline`` branches.
"""
pass

2
src/pipecat/observers/loggers/debug_log_observer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/observers/loggers/llm_log_observer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

35
src/pipecat/observers/loggers/metrics_log_observer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -24,6 +24,7 @@ from pipecat.metrics.metrics import (
SmartTurnMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
TurnMetricsData,
)
from pipecat.observers.base_observer import BaseObserver, FramePushed
@@ -37,7 +38,7 @@ class MetricsLogObserver(BaseObserver):
- ProcessingMetricsData (General processing time)
- LLMUsageMetricsData (Token usage statistics)
- TTSUsageMetricsData (Text-to-Speech character counts)
- SmartTurnMetricsData (Turn prediction metrics)
- TurnMetricsData (Turn prediction metrics)
This allows developers to track performance metrics, token usage,
and other statistics throughout the pipeline.
@@ -70,6 +71,17 @@ class MetricsLogObserver(BaseObserver):
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
# Normalize deprecated types in include_metrics
if include_metrics and SmartTurnMetricsData in include_metrics:
import warnings
warnings.warn(
"SmartTurnMetricsData is deprecated in include_metrics, "
"use TurnMetricsData instead.",
DeprecationWarning,
stacklevel=2,
)
include_metrics = (include_metrics - {SmartTurnMetricsData}) | {TurnMetricsData}
self._include_metrics = include_metrics
self._frames_seen = set()
@@ -144,8 +156,8 @@ class MetricsLogObserver(BaseObserver):
logger.debug(
f"📊 {processor_info} TTS USAGE{model_info}: {metrics_data.value} characters at {time_sec:.3f}s"
)
elif isinstance(metrics_data, SmartTurnMetricsData):
self._log_smart_turn(metrics_data, processor_info, model_info, time_sec)
elif isinstance(metrics_data, TurnMetricsData):
self._log_turn(metrics_data, processor_info, model_info, time_sec)
else:
# Generic fallback for unknown metrics types
logger.debug(
@@ -191,28 +203,27 @@ class MetricsLogObserver(BaseObserver):
f"📊 {processor_info} LLM TOKEN USAGE{model_info}: {usage_str} at {time_sec:.2f}s"
)
def _log_smart_turn(
def _log_turn(
self,
metrics_data: SmartTurnMetricsData,
metrics_data: TurnMetricsData,
processor_info: str,
model_info: str,
time_sec: float,
):
"""Log smart turn prediction metrics.
"""Log turn prediction metrics.
Args:
metrics_data: The smart turn metrics data.
metrics_data: The turn metrics data.
processor_info: Formatted processor name string.
model_info: Formatted model name string.
time_sec: Timestamp in seconds.
"""
complete_str = "COMPLETE" if metrics_data.is_complete else "INCOMPLETE"
e2e_str = f"{metrics_data.e2e_processing_time_ms:.1f}ms"
logger.debug(
f"📊 {processor_info} SMART TURN{model_info}: {complete_str} "
f"📊 {processor_info} TURN{model_info}: {complete_str} "
f"(probability: {metrics_data.probability:.2%}, "
f"inference: {metrics_data.inference_time_ms:.1f}ms, "
f"server: {metrics_data.server_total_time_ms:.1f}ms, "
f"e2e: {metrics_data.e2e_processing_time_ms:.1f}ms) "
f"e2e: {e2e_str}) "
f"at {time_sec:.2f}s"
)

2
src/pipecat/observers/loggers/transcription_log_observer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

40
src/pipecat/observers/loggers/user_bot_latency_log_observer.py
View File

@@ -1,12 +1,18 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Observer for measuring user-to-bot response latency."""
"""Observer for measuring user-to-bot response latency.
.. deprecated:: 0.0.102
This module is deprecated. Use :class:`UserBotLatencyObserver` directly
with its ``on_latency_measured`` event handler instead.
"""
import time
import warnings
from statistics import mean
from loguru import logger
@@ -15,8 +21,8 @@ from pipecat.frames.frames import (
BotStartedSpeakingFrame,
CancelFrame,
EndFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.frame_processor import FrameDirection
@@ -27,6 +33,10 @@ class UserBotLatencyLogObserver(BaseObserver):
This helps measure how quickly the AI services respond by tracking
conversation turn timing and logging latency metrics.
.. deprecated:: 0.0.102
This class is deprecated. Use :class:`UserBotLatencyObserver` directly
with its ``on_latency_measured`` event handler for custom logging.
"""
def __init__(self):
@@ -34,9 +44,19 @@ class UserBotLatencyLogObserver(BaseObserver):
Sets up tracking for processed frames and user speech timing
to calculate response latencies.
.. deprecated:: 0.0.102
This class is deprecated. Use :class:`UserBotLatencyObserver`
directly with its ``on_latency_measured`` event handler.
"""
warnings.warn(
"UserBotLatencyLogObserver is deprecated and will be removed in a future version. "
"Use UserBotLatencyObserver directly with its on_latency_measured event handler instead.",
DeprecationWarning,
stacklevel=2,
)
super().__init__()
self._processed_frames = set()
self._user_bot_latency_processed_frames = set()
self._user_stopped_time = 0
self._latencies = []
@@ -51,15 +71,15 @@ class UserBotLatencyLogObserver(BaseObserver):
return
# Skip already processed frames
if data.frame.id in self._processed_frames:
if data.frame.id in self._user_bot_latency_processed_frames:
return
self._processed_frames.add(data.frame.id)
self._user_bot_latency_processed_frames.add(data.frame.id)
if isinstance(data.frame, UserStartedSpeakingFrame):
if isinstance(data.frame, VADUserStartedSpeakingFrame):
self._user_stopped_time = 0
elif isinstance(data.frame, UserStoppedSpeakingFrame):
self._user_stopped_time = time.time()
elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
elif isinstance(data.frame, (EndFrame, CancelFrame)):
self._log_summary()
elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time:

328
src/pipecat/observers/startup_timing_observer.py Normal file
View File

@@ -0,0 +1,328 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Observer for tracking pipeline startup timing.
This module provides an observer that measures how long each processor's
``start()`` method takes during pipeline startup. It works by tracking
when a ``StartFrame`` arrives at a processor (``on_process_frame``) versus
when it leaves (``on_push_frame``), giving the exact ``start()`` duration
for each processor in the pipeline.
It also measures transport timing — the time from ``StartFrame`` to the
first ``BotConnectedFrame`` (SFU transports only) and ``ClientConnectedFrame``
— via a separate ``on_transport_timing_report`` event.
Example::
observer = StartupTimingObserver()
@observer.event_handler("on_startup_timing_report")
async def on_report(observer, report):
for t in report.processor_timings:
print(f"{t.processor_name}: {t.duration_secs:.3f}s")
@observer.event_handler("on_transport_timing_report")
async def on_transport(observer, report):
if report.bot_connected_secs is not None:
print(f"Bot connected in {report.bot_connected_secs:.3f}s")
print(f"Client connected in {report.client_connected_secs:.3f}s")
task = PipelineTask(pipeline, observers=[observer])
"""
import time
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Type
from pydantic import BaseModel, Field
from pipecat.frames.frames import BotConnectedFrame, ClientConnectedFrame, StartFrame
from pipecat.observers.base_observer import BaseObserver, FrameProcessed, FramePushed
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.pipeline import PipelineSource
from pipecat.processors.frame_processor import FrameProcessor
# Internal pipeline types excluded from tracking by default.
_INTERNAL_TYPES = (PipelineSource, BasePipeline)
@dataclass
class _ArrivalInfo:
"""Internal record of when a StartFrame arrived at a processor."""
processor: FrameProcessor
arrival_ts_ns: int
class ProcessorStartupTiming(BaseModel):
"""Startup timing for a single processor.
Parameters:
processor_name: The name of the processor.
start_offset_secs: Offset in seconds from the StartFrame to when this
processor's start() began.
duration_secs: How long the processor's start() took, in seconds.
"""
processor_name: str
start_offset_secs: float
duration_secs: float
class StartupTimingReport(BaseModel):
"""Report of startup timings for all measured processors.
Parameters:
start_time: Unix timestamp when the first processor began starting.
total_duration_secs: Total wall-clock time from first to last processor start.
processor_timings: Per-processor timing data, in pipeline order.
"""
start_time: float
total_duration_secs: float
processor_timings: List[ProcessorStartupTiming] = Field(default_factory=list)
class TransportTimingReport(BaseModel):
"""Time from pipeline start to transport connection milestones.
Parameters:
start_time: Unix timestamp of the StartFrame (pipeline start).
bot_connected_secs: Seconds from StartFrame to first BotConnectedFrame
(only set for SFU transports).
client_connected_secs: Seconds from StartFrame to first ClientConnectedFrame.
"""
start_time: float
bot_connected_secs: Optional[float] = None
client_connected_secs: Optional[float] = None
class StartupTimingObserver(BaseObserver):
"""Observer that measures processor startup times during pipeline initialization.
Tracks how long each processor's ``start()`` method takes by measuring the
time between when a ``StartFrame`` arrives at a processor and when it is
pushed downstream. This captures WebSocket connections, API authentication,
model loading, and other initialization work.
Also measures transport timing, the time from ``StartFrame`` to connection
milestones:
- ``bot_connected_secs``: When the bot joins the transport room
(SFU transports only, triggered by ``BotConnectedFrame``).
- ``client_connected_secs``: When a remote participant connects
(triggered by ``ClientConnectedFrame``).
By default, internal pipeline processors (``PipelineSource``, ``Pipeline``)
are excluded from the report. Pass ``processor_types`` to measure only
specific types.
Event handlers available:
- on_startup_timing_report: Called once after startup completes with the full
timing report.
- on_transport_timing_report: Called once when the first client connects with a
TransportTimingReport containing client_connected_secs and bot_connected_secs
(if available).
Example::
observer = StartupTimingObserver(
processor_types=(STTService, TTSService)
)
@observer.event_handler("on_startup_timing_report")
async def on_report(observer, report):
for t in report.processor_timings:
logger.info(f"{t.processor_name}: {t.duration_secs:.3f}s")
@observer.event_handler("on_transport_timing_report")
async def on_transport(observer, report):
if report.bot_connected_secs is not None:
logger.info(f"Bot connected in {report.bot_connected_secs:.3f}s")
logger.info(f"Client connected in {report.client_connected_secs:.3f}s")
task = PipelineTask(pipeline, observers=[observer])
Args:
processor_types: Optional tuple of processor types to measure. If None,
all non-internal processors are measured.
"""
def __init__(
self,
*,
processor_types: Optional[Tuple[Type[FrameProcessor], ...]] = None,
**kwargs,
):
"""Initialize the startup timing observer.
Args:
processor_types: Optional tuple of processor types to measure.
If None, all non-internal processors are measured.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._processor_types = processor_types
# Map processor ID -> arrival info.
self._arrivals: Dict[int, _ArrivalInfo] = {}
# Collected timings in pipeline order.
self._timings: List[ProcessorStartupTiming] = []
# Lock onto the first StartFrame we see (by frame ID).
self._start_frame_id: Optional[str] = None
# Whether we've already emitted the startup timing report.
self._startup_timing_reported = False
# Whether we've already measured transport timing.
self._transport_timing_reported = False
# Timestamp (ns) when we first see a StartFrame arrive at a processor.
self._start_frame_arrival_ns: Optional[int] = None
# Bot connected timing (stored for inclusion in the transport report).
self._bot_connected_secs: Optional[float] = None
# Wall clock time when the StartFrame was first seen.
self._start_wall_clock: Optional[float] = None
self._register_event_handler("on_startup_timing_report")
self._register_event_handler("on_transport_timing_report")
def _should_track(self, processor: FrameProcessor) -> bool:
"""Check if a processor should be tracked for timing.
Args:
processor: The processor to check.
Returns:
True if the processor matches the filter or no filter is set.
"""
if self._processor_types is not None:
return isinstance(processor, self._processor_types)
# Default: exclude internal pipeline plumbing.
return not isinstance(processor, _INTERNAL_TYPES)
async def on_pipeline_started(self):
"""Emit the startup timing report when the pipeline has fully started.
Called by the ``PipelineTask`` after the ``StartFrame`` has been
processed by all processors, including nested ``ParallelPipeline``
branches.
"""
if self._timings:
await self._emit_report()
async def on_process_frame(self, data: FrameProcessed):
"""Record when a StartFrame arrives at a processor.
Args:
data: The frame processing event data.
"""
if self._startup_timing_reported:
return
if not isinstance(data.frame, StartFrame):
return
# Lock onto the first StartFrame.
if self._start_frame_id is None:
self._start_frame_id = data.frame.id
self._start_frame_arrival_ns = data.timestamp
self._start_wall_clock = time.time()
elif data.frame.id != self._start_frame_id:
return
if self._should_track(data.processor):
self._arrivals[data.processor.id] = _ArrivalInfo(
processor=data.processor, arrival_ts_ns=data.timestamp
)
async def on_push_frame(self, data: FramePushed):
"""Record when a StartFrame leaves a processor and compute the delta.
Also handles ``BotConnectedFrame`` and ``ClientConnectedFrame`` to
measure transport timing.
Args:
data: The frame push event data.
"""
if isinstance(data.frame, BotConnectedFrame):
self._handle_bot_connected(data)
return
if isinstance(data.frame, ClientConnectedFrame):
await self._handle_client_connected(data)
return
if self._startup_timing_reported:
return
if not isinstance(data.frame, StartFrame):
return
if self._start_frame_id is not None and data.frame.id != self._start_frame_id:
return
arrival = self._arrivals.pop(data.source.id, None)
if arrival is None:
return
duration_ns = data.timestamp - arrival.arrival_ts_ns
duration_secs = duration_ns / 1e9
start_offset_secs = (arrival.arrival_ts_ns - self._start_frame_arrival_ns) / 1e9
self._timings.append(
ProcessorStartupTiming(
processor_name=arrival.processor.name,
start_offset_secs=start_offset_secs,
duration_secs=duration_secs,
)
)
def _handle_bot_connected(self, data: FramePushed):
"""Record bot connected timing on first BotConnectedFrame."""
if self._bot_connected_secs is not None or self._start_frame_arrival_ns is None:
return
delta_ns = data.timestamp - self._start_frame_arrival_ns
self._bot_connected_secs = delta_ns / 1e9
async def _handle_client_connected(self, data: FramePushed):
"""Emit transport timing report on first ClientConnectedFrame."""
if self._transport_timing_reported or self._start_frame_arrival_ns is None:
return
self._transport_timing_reported = True
delta_ns = data.timestamp - self._start_frame_arrival_ns
client_connected_secs = delta_ns / 1e9
report = TransportTimingReport(
start_time=self._start_wall_clock or 0.0,
bot_connected_secs=self._bot_connected_secs,
client_connected_secs=client_connected_secs,
)
await self._call_event_handler("on_transport_timing_report", report)
async def _emit_report(self):
"""Build and emit the startup timing report."""
if self._startup_timing_reported:
return
self._startup_timing_reported = True
total = sum(t.duration_secs for t in self._timings)
report = StartupTimingReport(
start_time=self._start_wall_clock or 0.0,
total_duration_secs=total,
processor_timings=self._timings,
)
await self._call_event_handler("on_startup_timing_report", report)

2
src/pipecat/observers/turn_tracking_observer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

351
src/pipecat/observers/user_bot_latency_observer.py Normal file
View File

@@ -0,0 +1,351 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Observer for tracking user-to-bot response latency.
This module provides an observer that monitors the time between when a user
stops speaking and when the bot starts speaking, emitting events when latency
is measured. Optionally collects per-service latency breakdown metrics
(TTFB, text aggregation) when ``enable_metrics=True``.
"""
import time
from collections import deque
from typing import Dict, List, Optional
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
ClientConnectedFrame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
InterruptionFrame,
MetricsFrame,
UserStoppedSpeakingFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.metrics.metrics import (
TextAggregationMetricsData,
TTFBMetricsData,
)
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.frame_processor import FrameDirection
class TTFBBreakdownMetrics(BaseModel):
"""TTFB measurement with timestamp for timeline placement.
Parameters:
processor: Name of the processor that reported the TTFB.
model: Optional model name associated with the metric.
start_time: Unix timestamp when the TTFB measurement started.
duration_secs: TTFB duration in seconds.
"""
processor: str
model: Optional[str] = None
start_time: float
duration_secs: float
class TextAggregationBreakdownMetrics(BaseModel):
"""Text aggregation measurement with timestamp for timeline placement.
Parameters:
processor: Name of the processor that reported the metric.
start_time: Unix timestamp when text aggregation started.
duration_secs: Aggregation duration in seconds.
"""
processor: str
start_time: float
duration_secs: float
class FunctionCallMetrics(BaseModel):
"""Latency for a single function call execution.
Parameters:
function_name: Name of the function that was called.
start_time: Unix timestamp when execution started.
duration_secs: Time in seconds from execution start to result.
"""
function_name: str
start_time: float
duration_secs: float
class LatencyBreakdown(BaseModel):
"""Per-service latency breakdown for a single user-to-bot cycle.
Collected between ``VADUserStoppedSpeakingFrame`` and
``BotStartedSpeakingFrame`` when ``enable_metrics=True`` in
:class:`~pipecat.pipeline.task.PipelineParams`.
Parameters:
ttfb: Time-to-first-byte metrics from each service in the pipeline.
text_aggregation: First text aggregation measurement, representing
the latency cost of sentence aggregation in the TTS pipeline.
user_turn_start_time: Unix timestamp when the user turn started
(actual user silence, adjusted for VAD stop_secs). ``None`` if
no ``VADUserStoppedSpeakingFrame`` was observed.
user_turn_secs: Duration in seconds of the user's turn, measured
from when the user actually stopped speaking to when the turn
was released (``UserStoppedSpeakingFrame``). This includes
VAD silence detection, STT finalization, and any turn analyzer
wait. ``None`` if no ``UserStoppedSpeakingFrame`` was observed
(e.g. no turn analyzer configured).
function_calls: Latency for each function call executed during
this cycle. Empty if no function calls occurred.
"""
ttfb: List[TTFBBreakdownMetrics] = Field(default_factory=list)
text_aggregation: Optional[TextAggregationBreakdownMetrics] = None
user_turn_start_time: Optional[float] = None
user_turn_secs: Optional[float] = None
function_calls: List[FunctionCallMetrics] = Field(default_factory=list)
def chronological_events(self) -> List[str]:
"""Return human-readable event labels sorted by start time.
Collects all sub-metrics into a flat list, sorts by ``start_time``,
and returns formatted strings suitable for logging.
Returns:
List of formatted strings, one per event, in chronological order.
"""
events: List[tuple] = []
if self.user_turn_start_time is not None and self.user_turn_secs is not None:
events.append((self.user_turn_start_time, f"User turn: {self.user_turn_secs:.3f}s"))
for t in self.ttfb:
events.append((t.start_time, f"{t.processor}: TTFB {t.duration_secs:.3f}s"))
for fc in self.function_calls:
events.append((fc.start_time, f"{fc.function_name}: {fc.duration_secs:.3f}s"))
if self.text_aggregation:
ta = self.text_aggregation
events.append(
(ta.start_time, f"{ta.processor}: text aggregation {ta.duration_secs:.3f}s")
)
events.sort(key=lambda e: e[0])
return [label for _, label in events]
class UserBotLatencyObserver(BaseObserver):
"""Observer that tracks user-to-bot response latency.
Measures the time between when a user stops speaking (VADUserStoppedSpeakingFrame)
and when the bot starts speaking (BotStartedSpeakingFrame). Emits events when
latency is measured, allowing consumers to log, trace, or otherwise process
the latency data.
When ``enable_metrics=True`` in pipeline params, also collects per-service
latency breakdown (TTFB, text aggregation) and emits an
``on_latency_breakdown`` event alongside the existing latency measurement.
This observer follows the composition pattern used by TurnTrackingObserver,
acting as a reusable component for latency measurement.
Events:
on_latency_measured(observer, latency_seconds): Emitted when
time-to-first-bot-speech is calculated. Measures the time from
when the user stopped speaking to when the bot starts speaking.
on_latency_breakdown(observer, breakdown): Emitted at each
``BotStartedSpeakingFrame`` with a :class:`LatencyBreakdown`
containing per-service metrics collected during the user→bot cycle.
on_first_bot_speech_latency(observer, latency_seconds): Emitted once,
the first time ``BotStartedSpeakingFrame`` arrives after
``ClientConnectedFrame``. Measures the time from client connection
to the first bot speech.
"""
def __init__(self, *, max_frames=100, **kwargs):
"""Initialize the user-bot latency observer.
Sets up tracking for processed frames and user speech timing
to calculate response latencies.
Args:
max_frames: Maximum number of frame IDs to keep in history for
duplicate detection. Defaults to 100.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._user_stopped_time: Optional[float] = None
self._user_turn_start_time: Optional[float] = None
self._user_turn: Optional[float] = None
# First bot speech tracking
self._client_connected_time: Optional[float] = None
self._first_bot_speech_measured: bool = False
# Frame deduplication (bounded deque + set pattern)
self._processed_frames: set = set()
self._frame_history: deque = deque(maxlen=max_frames)
# Per-cycle metric accumulators
self._ttfb: List[TTFBBreakdownMetrics] = []
self._text_aggregation: Optional[TextAggregationBreakdownMetrics] = None
self._function_call_starts: Dict[str, tuple[str, float]] = {}
self._function_call_metrics: List[FunctionCallMetrics] = []
self._register_event_handler("on_latency_measured")
self._register_event_handler("on_latency_breakdown")
self._register_event_handler("on_first_bot_speech_latency")
async def on_push_frame(self, data: FramePushed):
"""Process frames to track speech timing and calculate latency.
Tracks VAD events and bot speaking events to measure the time between
user stopping speech and bot starting speech. Also accumulates metrics
from MetricsFrame for the latency breakdown.
Args:
data: Frame push event containing the frame and direction information.
"""
# Only process downstream frames
if data.direction != FrameDirection.DOWNSTREAM:
return
# Skip already processed frames (bounded deque + set)
if data.frame.id in self._processed_frames:
return
self._processed_frames.add(data.frame.id)
self._frame_history.append(data.frame.id)
if len(self._processed_frames) > len(self._frame_history):
self._processed_frames = set(self._frame_history)
# Track client connection (first occurrence only)
if isinstance(data.frame, ClientConnectedFrame):
if self._client_connected_time is None:
self._client_connected_time = time.time()
return
# Track speech and pipeline events for latency
if isinstance(data.frame, VADUserStartedSpeakingFrame):
# Reset when user starts speaking
self._user_stopped_time = None
self._user_turn_start_time = None
self._user_turn = None
self._reset_accumulators()
# If user speaks before the bot's first speech, abandon the
# first-bot-speech measurement — it's only meaningful for greetings.
self._first_bot_speech_measured = True
elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
# Record the actual time the user stopped speaking, which is
# the VAD determination time minus the stop_secs silence duration
# that had to elapse before the VAD confirmed speech ended.
self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
self._user_turn_start_time = self._user_stopped_time
elif isinstance(data.frame, UserStoppedSpeakingFrame):
# Measure the user turn duration: from actual user silence to
# turn release. Includes VAD silence detection, STT finalization,
# and any turn analyzer wait.
if self._user_stopped_time is not None:
self._user_turn = time.time() - self._user_stopped_time
elif isinstance(data.frame, InterruptionFrame):
# Discard stale metrics from cancelled LLM/TTS cycles
self._reset_accumulators()
elif isinstance(data.frame, FunctionCallInProgressFrame):
self._function_call_starts[data.frame.tool_call_id] = (
data.frame.function_name,
time.time(),
)
elif isinstance(data.frame, FunctionCallResultFrame):
start = self._function_call_starts.pop(data.frame.tool_call_id, None)
if start is not None:
function_name, start_time = start
self._function_call_metrics.append(
FunctionCallMetrics(
function_name=function_name,
start_time=start_time,
duration_secs=time.time() - start_time,
)
)
elif isinstance(data.frame, MetricsFrame):
self._handle_metrics_frame(data.frame)
elif isinstance(data.frame, BotStartedSpeakingFrame):
await self._handle_bot_started_speaking()
async def _handle_bot_started_speaking(self):
"""Handle BotStartedSpeakingFrame to emit latency and breakdown."""
emit_breakdown = False
# One-time first bot speech measurement (client connect → first speech)
if self._client_connected_time is not None and not self._first_bot_speech_measured:
self._first_bot_speech_measured = True
latency = time.time() - self._client_connected_time
await self._call_event_handler("on_first_bot_speech_latency", latency)
emit_breakdown = True
if self._user_stopped_time is not None:
latency = time.time() - self._user_stopped_time
self._user_stopped_time = None
await self._call_event_handler("on_latency_measured", latency)
emit_breakdown = True
if emit_breakdown:
breakdown = LatencyBreakdown(
ttfb=list(self._ttfb),
text_aggregation=self._text_aggregation,
user_turn_start_time=self._user_turn_start_time,
user_turn_secs=self._user_turn,
function_calls=list(self._function_call_metrics),
)
await self._call_event_handler("on_latency_breakdown", breakdown)
self._reset_accumulators()
def _handle_metrics_frame(self, frame: MetricsFrame):
"""Extract latency metrics from a MetricsFrame.
Accumulates metrics when a measurement is in progress: either a
user→bot cycle (after ``VADUserStoppedSpeakingFrame``) or the
first-bot-speech window (after ``ClientConnectedFrame``).
"""
waiting_for_first_speech = (
self._client_connected_time is not None and not self._first_bot_speech_measured
)
if self._user_stopped_time is None and not waiting_for_first_speech:
return
now = time.time()
for metrics_data in frame.data:
if isinstance(metrics_data, TTFBMetricsData) and metrics_data.value > 0:
self._ttfb.append(
TTFBBreakdownMetrics(
processor=metrics_data.processor,
model=metrics_data.model,
start_time=now - metrics_data.value,
duration_secs=metrics_data.value,
)
)
elif isinstance(metrics_data, TextAggregationMetricsData):
# Only keep the first measurement — it's the one that
# impacts the initial speaking latency.
if self._text_aggregation is None:
self._text_aggregation = TextAggregationBreakdownMetrics(
processor=metrics_data.processor,
start_time=now - metrics_data.value,
duration_secs=metrics_data.value,
)
def _reset_accumulators(self):
"""Clear per-cycle metric accumulators."""
self._ttfb = []
self._text_aggregation = None
self._user_turn_start_time = None
self._user_turn = None
self._function_call_starts = {}
self._function_call_metrics = []

2
src/pipecat/pipeline/base_pipeline.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/pipeline/base_task.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

34
src/pipecat/pipeline/llm_switcher.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -9,7 +9,11 @@
from typing import Any, List, Optional, Type
from pipecat.adapters.schemas.direct_function import DirectFunction
from pipecat.pipeline.service_switcher import ServiceSwitcher, StrategyType
from pipecat.pipeline.service_switcher import (
ServiceSwitcher,
ServiceSwitcherStrategyManual,
StrategyType,
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.services.llm_service import LLMService
@@ -19,18 +23,20 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
Example::
llm_switcher = LLMSwitcher(
llms=[openai_llm, anthropic_llm],
strategy_type=ServiceSwitcherStrategyManual
)
llm_switcher = LLMSwitcher(llms=[openai_llm, anthropic_llm])
"""
def __init__(self, llms: List[LLMService], strategy_type: Type[StrategyType]):
def __init__(
self,
llms: List[LLMService],
strategy_type: Type[StrategyType] = ServiceSwitcherStrategyManual,
):
"""Initialize the service switcher with a list of LLMs and a switching strategy.
Args:
llms: List of LLM services to switch between.
strategy_type: The strategy class to use for switching between LLMs.
Defaults to ``ServiceSwitcherStrategyManual``.
"""
super().__init__(llms, strategy_type)
@@ -44,7 +50,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
return self.services
@property
def active_llm(self) -> Optional[LLMService]:
def active_llm(self) -> LLMService:
"""Get the currently active LLM.
Returns:
@@ -52,17 +58,19 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
"""
return self.strategy.active_service
async def run_inference(self, context: LLMContext) -> Optional[str]:
async def run_inference(self, context: LLMContext, **kwargs) -> Optional[str]:
"""Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context, using the currently active LLM.
Args:
context: The LLM context containing conversation history.
**kwargs: Additional arguments forwarded to the active LLM's run_inference
(e.g. max_tokens, system_instruction).
Returns:
The LLM's response as a string, or None if no response is generated.
"""
if self.active_llm:
return await self.active_llm.run_inference(context=context)
return await self.active_llm.run_inference(context=context, **kwargs)
return None
def register_function(
@@ -72,6 +80,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
start_callback=None,
*,
cancel_on_interruption: bool = True,
timeout_secs: Optional[float] = None,
):
"""Register a function handler for LLM function calls, on all LLMs, active or not.
@@ -88,6 +97,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
cancel_on_interruption: Whether to cancel this function call when an
interruption occurs. Defaults to True.
timeout_secs: Optional timeout in seconds for the function call.
"""
for llm in self.llms:
llm.register_function(
@@ -95,6 +105,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
handler=handler,
start_callback=start_callback,
cancel_on_interruption=cancel_on_interruption,
timeout_secs=timeout_secs,
)
def register_direct_function(
@@ -102,6 +113,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
handler: DirectFunction,
*,
cancel_on_interruption: bool = True,
timeout_secs: Optional[float] = None,
):
"""Register a direct function handler for LLM function calls, on all LLMs, active or not.
@@ -109,9 +121,11 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
handler: The direct function to register. Must follow DirectFunction protocol.
cancel_on_interruption: Whether to cancel this function call when an
interruption occurs. Defaults to True.
timeout_secs: Optional timeout in seconds for the function call.
"""
for llm in self.llms:
llm.register_direct_function(
handler=handler,
cancel_on_interruption=cancel_on_interruption,
timeout_secs=timeout_secs,
)

45
src/pipecat/pipeline/parallel_pipeline.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -52,6 +52,8 @@ class ParallelPipeline(BasePipeline):
self._seen_ids = set()
self._frame_counter: Dict[int, int] = {}
self._synchronizing: bool = False
self._buffered_frames: list[tuple[Frame, FrameDirection]] = []
logger.debug(f"Creating {self} pipelines")
@@ -141,8 +143,22 @@ class ParallelPipeline(BasePipeline):
await super().process_frame(frame, direction)
# Parallel pipeline synchronized frames.
#
# - StartFrame: If a fast branch completes first, processors in
# other branches that haven't received StartFrame yet could
# receive other frames before it, causing errors.
#
# - EndFrame: If EndFrame escapes from a fast branch, downstream
# processors (e.g. output transport) begin shutting down while
# other branches still have frames to flush, causing lost output.
#
# - CancelFrame: PipelineTask waits for CancelFrame to reach the
# pipeline sink. If it escapes from a fast branch while slower
# branches are still running, the task considers cancellation
# complete prematurely.
if isinstance(frame, (StartFrame, EndFrame, CancelFrame)):
self._frame_counter[frame.id] = len(self._pipelines)
self._synchronizing = True
await self.pause_processing_system_frames()
await self.pause_processing_frames()
@@ -151,10 +167,18 @@ class ParallelPipeline(BasePipeline):
await p.queue_frame(frame, direction)
async def _parallel_push_frame(self, frame: Frame, direction: FrameDirection):
"""Push frames while avoiding duplicates using frame ID tracking."""
"""Push frames while avoiding duplicates using frame ID tracking.
During lifecycle frame synchronization, non-lifecycle frames are buffered
to prevent them from escaping the parallel pipeline before all branches
have finished processing the lifecycle frame.
"""
if frame.id not in self._seen_ids:
self._seen_ids.add(frame.id)
await self.push_frame(frame, direction)
if self._synchronizing:
self._buffered_frames.append((frame, direction))
else:
await self.push_frame(frame, direction)
async def _pipeline_sink_push_frame(self, frame: Frame, direction: FrameDirection):
# Parallel pipeline synchronized frames.
@@ -167,8 +191,21 @@ class ParallelPipeline(BasePipeline):
# Only push the frame when all pipelines have processed it.
if frame_counter == 0:
await self._parallel_push_frame(frame, direction)
self._synchronizing = False
# StartFrame should always go before any other frame.
if isinstance(frame, StartFrame):
await self._parallel_push_frame(frame, direction)
await self._flush_buffered_frames()
else:
await self._flush_buffered_frames()
await self._parallel_push_frame(frame, direction)
await self.resume_processing_system_frames()
await self.resume_processing_frames()
else:
await self._parallel_push_frame(frame, direction)
async def _flush_buffered_frames(self):
"""Flush frames that were buffered during lifecycle frame synchronization."""
while len(self._buffered_frames) > 0:
frame, direction = self._buffered_frames.pop(0)
await self.push_frame(frame, direction)

2
src/pipecat/pipeline/pipeline.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/pipeline/runner.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

316
src/pipecat/pipeline/service_switcher.py
View File

@@ -1,31 +1,45 @@
#
# Copyright (c) 2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Service switcher for switching between different services at runtime, with different switching strategies."""
from dataclasses import dataclass
from typing import Any, Generic, List, Optional, Type, TypeVar
from loguru import logger
from pipecat.frames.frames import (
ControlFrame,
ErrorFrame,
Frame,
ManuallySwitchServiceFrame,
ServiceMetadataFrame,
ServiceSwitcherFrame,
ServiceSwitcherRequestMetadataFrame,
)
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from pipecat.processors.filters.function_filter import FunctionFilter
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.base_object import BaseObject
class ServiceSwitcherStrategy:
class ServiceSwitcherStrategy(BaseObject):
"""Base class for service switching strategies.
Note:
Strategy classes are instantiated internally by ServiceSwitcher.
Developers should pass the strategy class (not an instance) to ServiceSwitcher.
Event handlers available:
- on_service_switched: Called when the active service changes.
Example::
@strategy.event_handler("on_service_switched")
async def on_service_switched(strategy, service):
...
"""
def __init__(self, services: List[FrameProcessor]):
@@ -37,20 +51,76 @@ class ServiceSwitcherStrategy:
Args:
services: List of frame processors to switch between.
"""
self.services = services
self.active_service: Optional[FrameProcessor] = None
super().__init__()
def handle_frame(self, frame: ServiceSwitcherFrame, direction: FrameDirection):
if len(services) == 0:
raise Exception(f"ServiceSwitcherStrategy needs at least one service")
self._services = services
self._active_service = services[0]
self._register_event_handler("on_service_switched")
@property
def services(self) -> List[FrameProcessor]:
"""Return the list of available services."""
return self._services
@property
def active_service(self) -> FrameProcessor:
"""Return the currently active service."""
return self._active_service
async def handle_frame(
self, frame: ServiceSwitcherFrame, direction: FrameDirection
) -> Optional[FrameProcessor]:
"""Handle a frame that controls service switching.
This method can be overridden by subclasses to implement specific logic
for handling frames that control service switching.
The base implementation returns ``None`` for all frames. Subclasses
override this to implement specific switching behaviors.
Args:
frame: The frame to handle.
direction: The direction of the frame (upstream or downstream).
Returns:
The newly active service if a switch occurred, or None otherwise.
"""
raise NotImplementedError("Subclasses must implement this method.")
return None
async def handle_error(self, error: ErrorFrame) -> Optional[FrameProcessor]:
"""Handle an error from the active service.
Called by ``ServiceSwitcher`` when a non-fatal ``ErrorFrame`` is pushed
upstream by the currently active service. Subclasses can override this
to implement automatic failover.
Args:
error: The error frame pushed by the active service.
Returns:
The newly active service if a switch occurred, or None otherwise.
"""
return None
async def _set_active_if_available(self, service: FrameProcessor) -> Optional[FrameProcessor]:
"""Set the active service to the given one, if it is in the list of available services.
If it's not in the list, the request is ignored, as it may have been
intended for another ServiceSwitcher in the pipeline.
Args:
service: The service to set as active.
Returns:
The newly active service, or None if the service was not found.
"""
if service in self.services:
self._active_service = service
await service.queue_frame(ServiceSwitcherRequestMetadataFrame(service=service))
await self._call_event_handler("on_service_switched", service)
return service
return None
class ServiceSwitcherStrategyManual(ServiceSwitcherStrategy):
@@ -67,103 +137,116 @@ class ServiceSwitcherStrategyManual(ServiceSwitcherStrategy):
)
"""
def __init__(self, services: List[FrameProcessor]):
"""Initialize the manual service switcher strategy with a list of services.
Note:
This is called internally by ServiceSwitcher. Do not instantiate directly.
Args:
services: List of frame processors to switch between.
"""
super().__init__(services)
self.active_service = services[0] if services else None
def handle_frame(self, frame: ServiceSwitcherFrame, direction: FrameDirection):
async def handle_frame(
self, frame: ServiceSwitcherFrame, direction: FrameDirection
) -> Optional[FrameProcessor]:
"""Handle a frame that controls service switching.
Args:
frame: The frame to handle.
direction: The direction of the frame (upstream or downstream).
Returns:
The newly active service if a switch occurred, or None otherwise.
"""
if isinstance(frame, ManuallySwitchServiceFrame):
self._set_active_if_available(frame.service)
else:
raise ValueError(f"Unsupported frame type: {type(frame)}")
return await self._set_active_if_available(frame.service)
def _set_active_if_available(self, service: FrameProcessor):
"""Set the active service to the given one, if it is in the list of available services.
return None
If it's not in the list, the request is ignored, as it may have been
intended for another ServiceSwitcher in the pipeline.
class ServiceSwitcherStrategyFailover(ServiceSwitcherStrategyManual):
"""A strategy that automatically switches to a backup service on failure.
When the active service produces a non-fatal error, this strategy switches
to the next available service in the list. Recovery and fallback policies
are left to application code via the ``on_service_switched`` event.
Event handlers available:
- on_service_switched: Called when the active service changes.
Example::
switcher = ServiceSwitcher(
services=[primary_stt, backup_stt],
strategy_type=ServiceSwitcherStrategyFailover,
)
@switcher.strategy.event_handler("on_service_switched")
async def on_switched(strategy, service):
# App decides when/how to recover the failed service
...
"""
async def handle_error(self, error: ErrorFrame) -> Optional[FrameProcessor]:
"""Handle an error from the active service by failing over.
Switches to the next service in the list. The failed service remains
in the list and can be switched back to manually or via application
logic in the ``on_service_switched`` event handler.
Args:
service: The service to set as active.
error: The error frame pushed by the active service.
Returns:
The newly active service if a switch occurred, or None if no
other service is available.
"""
if service in self.services:
self.active_service = service
service_name = error.processor.name if error.processor else self._active_service.name
logger.warning(f"Service {service_name} reported an error: {error.error}")
if len(self._services) <= 1:
logger.error("No other service available to switch to")
return None
current_idx = self._services.index(self._active_service)
next_idx = (current_idx + 1) % len(self._services)
return await self._set_active_if_available(self._services[next_idx])
StrategyType = TypeVar("StrategyType", bound=ServiceSwitcherStrategy)
class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]):
"""A pipeline that switches between different services at runtime."""
"""Parallel pipeline that routes frames to one active service at a time.
def __init__(self, services: List[FrameProcessor], strategy_type: Type[StrategyType]):
Wraps each service in a pair of filters that gate frame flow based on
which service is currently active. Switching is controlled by
`ServiceSwitcherFrame` frames and delegated to a pluggable
`ServiceSwitcherStrategy`.
Example::
switcher = ServiceSwitcher(services=[stt_1, stt_2])
"""
def __init__(
self,
services: List[FrameProcessor],
strategy_type: Type[StrategyType] = ServiceSwitcherStrategyManual,
):
"""Initialize the service switcher with a list of services and a switching strategy.
Args:
services: List of frame processors to switch between.
strategy_type: The strategy class to use for switching between services.
Defaults to ``ServiceSwitcherStrategyManual``.
"""
strategy = strategy_type(services)
super().__init__(*self._make_pipeline_definitions(services, strategy))
self.services = services
self.strategy = strategy
_strategy = strategy_type(services)
super().__init__(*self._make_pipeline_definitions(services, _strategy))
self._services = services
self._strategy = _strategy
class ServiceSwitcherFilter(FunctionFilter):
"""An internal filter that allows frames to pass through to the wrapped service only if it's the active service."""
@property
def strategy(self) -> StrategyType:
"""Return the active switching strategy."""
return self._strategy
def __init__(
self,
wrapped_service: FrameProcessor,
active_service: FrameProcessor,
direction: FrameDirection,
):
"""Initialize the service switcher filter with a strategy and direction.
Args:
wrapped_service: The service that this filter wraps.
active_service: The currently active service.
direction: The direction of frame flow to filter.
"""
self._wrapped_service = wrapped_service
self._active_service = active_service
async def filter(_: Frame) -> bool:
return self._wrapped_service == self._active_service
super().__init__(filter, direction, filter_system_frames=True)
async def process_frame(self, frame, direction):
"""Process a frame through the filter, handling special internal filter-updating frames."""
if isinstance(frame, ServiceSwitcher.ServiceSwitcherFilterFrame):
self._active_service = frame.active_service
# Two ServiceSwitcherFilters "sandwich" a service. Push the
# frame only to update the other side of the sandwich, but
# otherwise don't let it leave the sandwich.
if direction == self._direction:
await self.push_frame(frame, direction)
return
await super().process_frame(frame, direction)
@dataclass
class ServiceSwitcherFilterFrame(ControlFrame):
"""An internal frame used by ServiceSwitcher to filter frames based on active service."""
active_service: FrameProcessor
@property
def services(self) -> List[FrameProcessor]:
"""Return the list of available services."""
return self._services
@staticmethod
def _make_pipeline_definitions(
@@ -178,20 +261,66 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]):
def _make_pipeline_definition(
service: FrameProcessor, strategy: ServiceSwitcherStrategy
) -> Any:
async def filter(_: Frame) -> bool:
return service == strategy.active_service
# Layout: Filter → Service → Filter
#
# filter_system_frames: we want to run filter functions also on system
# frames.
#
# enable_direct_mode: filter functions are quick so we don't need
# additional tasks.
return [
ServiceSwitcher.ServiceSwitcherFilter(
wrapped_service=service,
active_service=strategy.active_service,
FunctionFilter(
filter=filter,
direction=FrameDirection.DOWNSTREAM,
filter_system_frames=True,
enable_direct_mode=True,
),
service,
ServiceSwitcher.ServiceSwitcherFilter(
wrapped_service=service,
active_service=strategy.active_service,
FunctionFilter(
filter=filter,
direction=FrameDirection.UPSTREAM,
filter_system_frames=True,
enable_direct_mode=True,
),
]
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame out of the service switcher.
Suppresses `ServiceSwitcherRequestMetadataFrame` targeting the active
service (since it has already been handled) and `ServiceMetadataFrame`
from inactive services so only the active service's metadata reaches
downstream processors. One case this happens is with `StartFrame` since
all the filters let it pass, and `StartFrame` causes the service to
generate `ServiceMetadataFrame`.
Non-fatal ``ErrorFrame`` instances are forwarded to the strategy via
``handle_error`` so strategies like ``ServiceSwitcherStrategyFailover``
can perform failover. The error frame is still propagated upstream so
that application-level error handlers can observe it.
"""
# Consume ServiceSwitcherRequestMetadataFrame once the targeted service
# has handled it (i.e. the active service).
if isinstance(frame, ServiceSwitcherRequestMetadataFrame):
if frame.service == self.strategy.active_service:
return
# Only let metadata from the active service escape.
if isinstance(frame, ServiceMetadataFrame):
if frame.service_name != self.strategy.active_service.name:
return
# Let the strategy react to non-fatal errors from the active service,
# ignoring errors just propagating upstream from other processors.
if isinstance(frame, ErrorFrame) and not frame.fatal:
if frame.processor and frame.processor == self.strategy.active_service:
await self.strategy.handle_error(frame)
await super().push_frame(frame, direction)
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame, handling frames which affect service switching.
@@ -199,11 +328,12 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]):
frame: The frame to process.
direction: The direction of the frame (upstream or downstream).
"""
await super().process_frame(frame, direction)
if isinstance(frame, ServiceSwitcherFrame):
self.strategy.handle_frame(frame, direction)
service_switcher_filter_frame = ServiceSwitcher.ServiceSwitcherFilterFrame(
active_service=self.strategy.active_service
)
await super().process_frame(service_switcher_filter_frame, direction)
service = await self.strategy.handle_frame(frame, direction)
# If we don't switch to a new service we need to keep processing the
# frame. If we switched, we just swallow the frame.
if not service:
await super().process_frame(frame, direction)
else:
await super().process_frame(frame, direction)

192
src/pipecat/pipeline/sync_parallel_pipeline.py
View File

@@ -1,18 +1,24 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Synchronous parallel pipeline implementation for concurrent frame processing.
"""Synchronized parallel pipeline that holds output until all branches finish.
This module provides a pipeline that processes frames through multiple parallel
pipelines simultaneously, synchronizing their output to maintain frame ordering
and prevent duplicate processing.
A SyncParallelPipeline fans each inbound frame out to multiple parallel pipelines
and waits for every pipeline to finish processing before releasing any of the
resulting output frames. This ensures that all frames produced in response to a
single input frame are emitted together.
System frames (except EndFrame) are exempt from this synchronization — they pass
straight through without waiting, since they are expected to race ahead of
regular data frames.
"""
import asyncio
from dataclasses import dataclass
from enum import Enum
from itertools import chain
from typing import List
@@ -24,22 +30,42 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
class FrameOrder(Enum):
"""Controls the order in which synchronized frames are pushed downstream.
When multiple parallel pipelines produce output for the same input frame,
this setting determines the order in which those output frames are pushed.
Attributes:
ARRIVAL: Frames are pushed in the order they arrive from any pipeline.
This is the default and matches the behavior of prior versions.
PIPELINE: Frames are pushed in pipeline definition order — all frames
from the first pipeline are pushed, then all frames from the second
pipeline, and so on. Useful when the relative ordering between
pipelines matters (e.g. ensuring image frames precede audio frames).
"""
ARRIVAL = "arrival"
PIPELINE = "pipeline"
@dataclass
class SyncFrame(ControlFrame):
"""Control frame used to synchronize parallel pipeline processing.
"""Sentinel frame used to detect when a parallel pipeline has finished processing.
This frame is sent through parallel pipelines to determine when the
internal pipelines have finished processing a batch of frames.
After sending a real frame into a parallel pipeline, a SyncFrame is sent
behind it. When the SyncFrame emerges from the pipeline's output, we know
all output frames for the preceding input have been produced.
"""
pass
class SyncParallelPipelineSource(FrameProcessor):
"""Source processor for synchronous parallel pipeline processing.
"""Bookend processor placed at the start of each parallel pipeline.
Routes frames to parallel pipelines and collects upstream responses
for synchronization purposes.
Forwards downstream frames into the pipeline and captures upstream frames
into a queue so the parent SyncParallelPipeline can release them later.
"""
def __init__(self, upstream_queue: asyncio.Queue):
@@ -68,10 +94,11 @@ class SyncParallelPipelineSource(FrameProcessor):
class SyncParallelPipelineSink(FrameProcessor):
"""Sink processor for synchronous parallel pipeline processing.
"""Bookend processor placed at the end of each parallel pipeline.
Collects downstream frames from parallel pipelines and routes
upstream frames back through the pipeline.
Captures downstream output frames into a queue so the parent
SyncParallelPipeline can release them later, and forwards upstream
frames back through the pipeline.
"""
def __init__(self, downstream_queue: asyncio.Queue):
@@ -100,29 +127,44 @@ class SyncParallelPipelineSink(FrameProcessor):
class SyncParallelPipeline(BasePipeline):
"""Pipeline that processes frames through multiple parallel pipelines synchronously.
"""Fans each input frame to parallel pipelines then holds output until every pipeline finishes.
Creates multiple parallel processing paths that all receive the same input frames
and produces synchronized output. Each parallel path is a separate pipeline that
processes frames independently, with synchronization points to ensure consistent
ordering and prevent duplicate frame processing.
For each inbound frame the pipeline:
The pipeline uses SyncFrame control frames to coordinate between parallel paths
and ensure all paths have completed processing before moving to the next frame.
1. Sends the frame into every parallel pipeline.
2. Sends a ``SyncFrame`` sentinel behind it in each pipeline.
3. Waits until every pipeline has produced its ``SyncFrame``, meaning all
output for that input is ready.
4. Releases the collected output frames (deduplicating by frame id, since
the same frame may emerge from more than one branch).
System frames (except ``EndFrame``) bypass this mechanism entirely — they are
forwarded through each pipeline and pushed immediately, since system frames
are expected to race ahead of regular data frames.
By default, output frames are pushed in the order they arrive from any pipeline
(``FrameOrder.ARRIVAL``). Set ``frame_order=FrameOrder.PIPELINE`` to push frames
in pipeline definition order instead — all output from the first pipeline, then
the second, and so on.
"""
def __init__(self, *args):
def __init__(self, *args, frame_order: FrameOrder = FrameOrder.ARRIVAL):
"""Initialize the synchronous parallel pipeline.
Args:
*args: Variable number of processor lists, each representing a parallel pipeline path.
Each argument should be a list of FrameProcessor instances.
*args: Variable number of processor lists, each representing a parallel
pipeline path. Each argument should be a list of FrameProcessor instances.
frame_order: Controls the order in which synchronized output frames are
pushed. ``FrameOrder.ARRIVAL`` (default) pushes frames in the order they arrive.
``FrameOrder.PIPELINE`` pushes all frames from the first pipeline
before the second, and so on.
Raises:
Exception: If no arguments are provided.
TypeError: If any argument is not a list of processors.
"""
super().__init__()
self._frame_order = frame_order
if len(args) == 0:
raise Exception(f"SyncParallelPipeline needs at least one argument")
@@ -184,7 +226,7 @@ class SyncParallelPipeline(BasePipeline):
Returns:
The list of entry processors.
"""
return self._sources
return [s["processor"] for s in self._sources]
def processors_with_metrics(self) -> List[FrameProcessor]:
"""Collect processors that can generate metrics from all parallel pipelines.
@@ -209,11 +251,11 @@ class SyncParallelPipeline(BasePipeline):
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames through all parallel pipelines with synchronization.
"""Send a frame through all parallel pipelines and release output once all finish.
Distributes frames to all parallel pipelines and synchronizes their output
to maintain proper ordering and prevent duplicate processing. Uses SyncFrame
control frames to coordinate between parallel paths.
System frames (except EndFrame) skip synchronization and pass straight
through. All other frames are fanned out to every pipeline, and output is
held until every pipeline signals completion (via SyncFrame).
Args:
frame: The frame to process.
@@ -221,60 +263,102 @@ class SyncParallelPipeline(BasePipeline):
"""
await super().process_frame(frame, direction)
# SystemFrames are simply passed through all internal pipelines without
# draining queued output. This avoids the race condition where a
# SystemFrame's wait_for_sync steals frames from a concurrent
# non-SystemFrame's wait_for_sync.
if isinstance(frame, SystemFrame):
if direction == FrameDirection.UPSTREAM:
for s in self._sinks:
await s["processor"].process_frame(frame, direction)
elif direction == FrameDirection.DOWNSTREAM:
for s in self._sources:
await s["processor"].process_frame(frame, direction)
await self.push_frame(frame, direction)
return
use_pipeline_order = self._frame_order == FrameOrder.PIPELINE
# The last processor of each pipeline needs to be synchronous otherwise
# this element won't work. Since, we know it should be synchronous we
# this element won't work. Since we know it should be synchronous we
# push a SyncFrame. Since frames are ordered we know this frame will be
# pushed after the synchronous processor has pushed its data allowing us
# to synchrnonize all the internal pipelines by waiting for the
# to synchronize all the internal pipelines by waiting for the
# SyncFrame in all of them.
#
# In ARRIVAL mode, output frames are put onto a shared main_queue as
# they arrive. In PIPELINE mode, they are accumulated in a per-pipeline
# list and returned so the caller can drain them in definition order.
async def wait_for_sync(
obj, main_queue: asyncio.Queue, frame: Frame, direction: FrameDirection
):
) -> list[Frame]:
processor = obj["processor"]
queue = obj["queue"]
output_frames: list[Frame] = []
await processor.process_frame(frame, direction)
if isinstance(frame, (SystemFrame, EndFrame)):
if isinstance(frame, EndFrame):
new_frame = await queue.get()
if isinstance(new_frame, (SystemFrame, EndFrame)):
await main_queue.put(new_frame)
else:
while not isinstance(new_frame, (SystemFrame, EndFrame)):
if isinstance(new_frame, EndFrame):
if use_pipeline_order:
output_frames.append(new_frame)
else:
await main_queue.put(new_frame)
else:
while not isinstance(new_frame, EndFrame):
if use_pipeline_order:
output_frames.append(new_frame)
else:
await main_queue.put(new_frame)
queue.task_done()
new_frame = await queue.get()
else:
await processor.process_frame(SyncFrame(), direction)
new_frame = await queue.get()
while not isinstance(new_frame, SyncFrame):
await main_queue.put(new_frame)
if use_pipeline_order:
output_frames.append(new_frame)
else:
await main_queue.put(new_frame)
queue.task_done()
new_frame = await queue.get()
return output_frames
if direction == FrameDirection.UPSTREAM:
# If we get an upstream frame we process it in each sink.
await asyncio.gather(
frames_per_pipeline = await asyncio.gather(
*[wait_for_sync(s, self._up_queue, frame, direction) for s in self._sinks]
)
elif direction == FrameDirection.DOWNSTREAM:
# If we get a downstream frame we process it in each source.
await asyncio.gather(
frames_per_pipeline = await asyncio.gather(
*[wait_for_sync(s, self._down_queue, frame, direction) for s in self._sources]
)
seen_ids = set()
while not self._up_queue.empty():
frame = await self._up_queue.get()
if frame.id not in seen_ids:
await self.push_frame(frame, FrameDirection.UPSTREAM)
seen_ids.add(frame.id)
self._up_queue.task_done()
if use_pipeline_order:
# Push frames in pipeline definition order, deduplicating by id.
seen_ids = set()
for pipeline_frames in frames_per_pipeline:
for f in pipeline_frames:
if f.id not in seen_ids:
await self.push_frame(f, direction)
seen_ids.add(f.id)
else:
# ARRIVAL mode: drain the shared queues in the order frames arrived.
seen_ids = set()
while not self._up_queue.empty():
frame = await self._up_queue.get()
if frame.id not in seen_ids:
await self.push_frame(frame, FrameDirection.UPSTREAM)
seen_ids.add(frame.id)
self._up_queue.task_done()
seen_ids = set()
while not self._down_queue.empty():
frame = await self._down_queue.get()
if frame.id not in seen_ids:
await self.push_frame(frame, FrameDirection.DOWNSTREAM)
seen_ids.add(frame.id)
self._down_queue.task_done()
seen_ids = set()
while not self._down_queue.empty():
frame = await self._down_queue.get()
if frame.id not in seen_ids:
await self.push_frame(frame, FrameDirection.DOWNSTREAM)
seen_ids.add(frame.id)
self._down_queue.task_done()

276
src/pipecat/pipeline/task.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -15,7 +15,7 @@ import asyncio
import importlib.util
import os
from pathlib import Path
from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type
from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Set, Tuple, Type, TypeVar
from loguru import logger
from pydantic import BaseModel, ConfigDict, Field
@@ -43,12 +43,17 @@ from pipecat.frames.frames import (
from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
from pipecat.observers.user_bot_latency_observer import UserBotLatencyObserver
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.base_task import BasePipelineTask, PipelineTaskParams
from pipecat.pipeline.pipeline import Pipeline, PipelineSink, PipelineSource
from pipecat.pipeline.task_observer import TaskObserver
from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIObserverParams, RTVIProcessor
from pipecat.utils.asyncio.task_manager import BaseTaskManager, TaskManager, TaskManagerParams
from pipecat.utils.tracing.setup import is_tracing_available
from pipecat.utils.tracing.tracing_context import TracingContext
from pipecat.utils.tracing.turn_trace_observer import TurnTraceObserver
HEARTBEAT_SECS = 1.0
@@ -59,6 +64,9 @@ IDLE_TIMEOUT_SECS = 300
CANCEL_TIMEOUT_SECS = 20.0
T = TypeVar("T")
class IdleFrameObserver(BaseObserver):
"""Idle timeout observer.
@@ -105,6 +113,10 @@ class PipelineParams(BaseModel):
Parameters:
allow_interruptions: Whether to allow pipeline interruptions.
.. deprecated:: 0.0.99
Use `LLMUserAggregator`'s new `user_turn_strategies` parameter instead.
audio_in_sample_rate: Input audio sample rate in Hz.
audio_out_sample_rate: Output audio sample rate in Hz.
enable_heartbeats: Whether to enable heartbeat monitoring.
@@ -113,7 +125,11 @@ class PipelineParams(BaseModel):
heartbeats_period_secs: Period between heartbeats in seconds.
heartbeats_monitor_secs: Timeout (in seconds) before warning about
missed heartbeats. Defaults to 10 seconds.
interruption_strategies: Strategies for bot interruption behavior.
interruption_strategies: [deprecated] Strategies for bot interruption behavior.
.. deprecated:: 0.0.99
Use `LLMUserAggregator`'s new `user_turn_strategies` parameter instead.
observers: [deprecated] Use `observers` arg in `PipelineTask` class.
.. deprecated:: 0.0.58
@@ -207,7 +223,7 @@ class PipelineTask(BasePipelineTask):
def __init__(
self,
pipeline: FrameProcessor,
pipeline: BasePipeline,
*,
params: Optional[PipelineParams] = None,
additional_span_attributes: Optional[dict] = None,
@@ -218,9 +234,12 @@ class PipelineTask(BasePipelineTask):
conversation_id: Optional[str] = None,
enable_tracing: bool = False,
enable_turn_tracking: bool = True,
enable_rtvi: bool = True,
idle_timeout_frames: Tuple[Type[Frame], ...] = (BotSpeakingFrame, UserSpeakingFrame),
idle_timeout_secs: Optional[float] = IDLE_TIMEOUT_SECS,
observers: Optional[List[BaseObserver]] = None,
rtvi_processor: Optional[RTVIProcessor] = None,
rtvi_observer_params: Optional[RTVIObserverParams] = None,
task_manager: Optional[BaseTaskManager] = None,
):
"""Initialize the PipelineTask.
@@ -237,6 +256,7 @@ class PipelineTask(BasePipelineTask):
check_dangling_tasks: Whether to check for processors' tasks finishing properly.
clock: Clock implementation for timing operations.
conversation_id: Optional custom ID for the conversation.
enable_rtvi: Whether to automatically add RTVI support to the pipeline.
enable_tracing: Whether to enable tracing.
enable_turn_tracking: Whether to enable turn tracking.
idle_timeout_frames: A tuple with the frames that should trigger an idle
@@ -245,6 +265,8 @@ class PipelineTask(BasePipelineTask):
None. If a pipeline is idle the pipeline task will be cancelled
automatically.
observers: List of observers for monitoring pipeline execution.
rtvi_observer_params: The RTVI observer parameter to use if RTVI is enabled.
rtvi_processor: The RTVI processor to add if RTVI is enabled.
task_manager: Optional task manager for handling asyncio tasks.
"""
super().__init__()
@@ -270,17 +292,28 @@ class PipelineTask(BasePipelineTask):
observers = self._params.observers
observers = observers or []
self._turn_tracking_observer: Optional[TurnTrackingObserver] = None
self._user_bot_latency_observer: Optional[UserBotLatencyObserver] = None
self._turn_trace_observer: Optional[TurnTraceObserver] = None
self._tracing_context: Optional[TracingContext] = None
if self._enable_turn_tracking:
self._turn_tracking_observer = TurnTrackingObserver()
observers.append(self._turn_tracking_observer)
if self._enable_tracing and self._turn_tracking_observer:
# Create pipeline-scoped tracing context
self._tracing_context = TracingContext()
# Create latency observer for tracing
self._user_bot_latency_observer = UserBotLatencyObserver()
observers.append(self._user_bot_latency_observer)
# Create turn trace observer with latency tracking
self._turn_trace_observer = TurnTraceObserver(
self._turn_tracking_observer,
latency_tracker=self._user_bot_latency_observer,
conversation_id=self._conversation_id,
additional_span_attributes=self._additional_span_attributes,
tracing_context=self._tracing_context,
)
observers.append(self._turn_trace_observer)
self._finished = False
self._cancelled = False
@@ -298,6 +331,39 @@ class PipelineTask(BasePipelineTask):
self._heartbeat_push_task: Optional[asyncio.Task] = None
self._heartbeat_monitor_task: Optional[asyncio.Task] = None
# RTVI support
self._rtvi = None
prepend_rtvi = False
external_rtvi = self._find_processor(pipeline, RTVIProcessor)
external_observer_found = any(isinstance(o, RTVIObserver) for o in observers)
if external_rtvi and not external_observer_found:
logger.error(
f"{self}: RTVIProcessor found in pipeline but no RTVIObserver in observers. "
"Make sure to add both."
)
elif not external_rtvi and external_observer_found:
logger.error(
f"{self}: RTVIObserver found in observers but no RTVIProcessor in pipeline. "
"Make sure to add both."
)
elif external_rtvi and external_observer_found:
logger.warning(
f"{self}: RTVIProcessor and RTVIObserver found, skipping default ones. "
"They are both added by default, no need to add them yourself."
)
self._rtvi = external_rtvi
elif enable_rtvi:
self._rtvi = rtvi_processor or RTVIProcessor()
observers.append(self._rtvi.create_rtvi_observer(params=rtvi_observer_params))
prepend_rtvi = True
if self._rtvi:
# Automatically call RTVIProcessor.set_bot_ready()
@self.rtvi.event_handler("on_client_ready")
async def on_client_ready(rtvi: RTVIProcessor):
await rtvi.set_bot_ready()
# This is the idle event. When selected frames are pushed from any
# processor we consider the pipeline is not idle. We use an observer
# which will be listening any part of the pipeline.
@@ -326,8 +392,12 @@ class PipelineTask(BasePipelineTask):
# source allows us to receive and react to upstream frames, and the sink
# allows us to receive and react to downstream frames.
source = PipelineSource(self._source_push_frame, name=f"{self}::Source")
sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
self._pipeline = Pipeline([pipeline], source=source, sink=sink)
self._sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
# Only prepend the RTVIProcessor if we created it ourselves. When the
# user already placed it inside their pipeline we must not insert it
# again or it will appear twice in the frame chain.
processors = [self._rtvi, pipeline] if prepend_rtvi else [pipeline]
self._pipeline = Pipeline(processors, source=source, sink=self._sink)
# The task observer acts as a proxy to the provided observers. This way,
# we only need to pass a single observer (using the StartFrame) which
@@ -340,8 +410,8 @@ class PipelineTask(BasePipelineTask):
# in. This is mainly for efficiency reason because each event handler
# creates a task and most likely you only care about one or two frame
# types.
self._reached_upstream_types: Tuple[Type[Frame], ...] = ()
self._reached_downstream_types: Tuple[Type[Frame], ...] = ()
self._reached_upstream_types: Set[Type[Frame]] = set()
self._reached_downstream_types: Set[Type[Frame]] = set()
self._register_event_handler("on_frame_reached_upstream")
self._register_event_handler("on_frame_reached_downstream")
self._register_event_handler("on_idle_timeout")
@@ -361,6 +431,17 @@ class PipelineTask(BasePipelineTask):
"""
return self._params
@property
def pipeline(self) -> BasePipeline:
"""Get the full pipeline managed by this pipeline task.
This will also include any internal processors added by the pipeline task.
Returns:
The pipeline managed by the pipeline task.
"""
return self._pipeline
@property
def turn_tracking_observer(self) -> Optional[TurnTrackingObserver]:
"""Get the turn tracking observer if enabled.
@@ -379,6 +460,35 @@ class PipelineTask(BasePipelineTask):
"""
return self._turn_trace_observer
@property
def rtvi(self) -> RTVIProcessor:
"""Get the RTVI processor if RTVI is enabled.
Returns:
The RTVI processor added to the pipeline when RTVI is enabled.
"""
if not self._rtvi:
raise Exception(f"{self} RTVI is not enabled.")
return self._rtvi
@property
def reached_upstream_types(self) -> Tuple[Type[Frame], ...]:
"""Get the currently configured upstream frame type filters.
Returns:
Tuple of frame types that trigger the on_frame_reached_upstream event.
"""
return tuple(self._reached_upstream_types)
@property
def reached_downstream_types(self) -> Tuple[Type[Frame], ...]:
"""Get the currently configured downstream frame type filters.
Returns:
Tuple of frame types that trigger the on_frame_reached_downstream event.
"""
return tuple(self._reached_downstream_types)
def event_handler(self, event_name: str):
"""Decorator for registering event handlers.
@@ -422,7 +532,7 @@ class PipelineTask(BasePipelineTask):
Args:
types: Tuple of frame types to monitor for upstream events.
"""
self._reached_upstream_types = types
self._reached_upstream_types = set(types)
def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
"""Set which frame types trigger the on_frame_reached_downstream event.
@@ -430,7 +540,23 @@ class PipelineTask(BasePipelineTask):
Args:
types: Tuple of frame types to monitor for downstream events.
"""
self._reached_downstream_types = types
self._reached_downstream_types = set(types)
def add_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]):
"""Add frame types to trigger the on_frame_reached_upstream event.
Args:
types: Tuple of frame types to add to upstream monitoring.
"""
self._reached_upstream_types.update(types)
def add_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
"""Add frame types to trigger the on_frame_reached_downstream event.
Args:
types: Tuple of frame types to add to downstream monitoring.
"""
self._reached_downstream_types.update(types)
def has_finished(self) -> bool:
"""Check if the pipeline task has finished execution.
@@ -502,26 +628,43 @@ class PipelineTask(BasePipelineTask):
self._finished = True
logger.debug(f"Pipeline task {self} has finished")
async def queue_frame(self, frame: Frame):
"""Queue a single frame to be pushed down the pipeline.
async def queue_frame(
self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
):
"""Queue a single frame to be pushed through the pipeline.
Downstream frames are pushed from the beginning of the pipeline.
Upstream frames are pushed from the end of the pipeline.
Args:
frame: The frame to be processed.
direction: The direction to push the frame. Defaults to downstream.
"""
await self._push_queue.put(frame)
if direction == FrameDirection.DOWNSTREAM:
await self._push_queue.put(frame)
else:
await self._sink.queue_frame(frame, direction)
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
"""Queues multiple frames to be pushed down the pipeline.
async def queue_frames(
self,
frames: Iterable[Frame] | AsyncIterable[Frame],
direction: FrameDirection = FrameDirection.DOWNSTREAM,
):
"""Queue multiple frames to be pushed through the pipeline.
Downstream frames are pushed from the beginning of the pipeline.
Upstream frames are pushed from the end of the pipeline.
Args:
frames: An iterable or async iterable of frames to be processed.
direction: The direction to push the frames. Defaults to downstream.
"""
if isinstance(frames, AsyncIterable):
async for frame in frames:
await self.queue_frame(frame)
await self.queue_frame(frame, direction)
elif isinstance(frames, Iterable):
for frame in frames:
await self.queue_frame(frame)
await self.queue_frame(frame, direction)
async def _cancel(self, *, reason: Optional[str] = None):
"""Internal cancellation logic for the pipeline task.
@@ -647,6 +790,9 @@ class PipelineTask(BasePipelineTask):
async def _setup(self, params: PipelineTaskParams):
"""Set up the pipeline task and all processors."""
# Do any additional pipeline task setup externally.
await self._load_setup_files()
# Load additional observers.
await self._load_observer_files()
@@ -697,8 +843,9 @@ class PipelineTask(BasePipelineTask):
enable_usage_metrics=self._params.enable_usage_metrics,
report_only_initial_ttfb=self._params.report_only_initial_ttfb,
interruption_strategies=self._params.interruption_strategies,
tracing_context=self._tracing_context,
)
start_frame.metadata = self._params.start_metadata
start_frame.metadata = self._create_start_metadata()
await self._pipeline.queue_frame(start_frame)
# Wait for the pipeline to be started before pushing any other frame.
@@ -727,27 +874,27 @@ class PipelineTask(BasePipelineTask):
pipeline to be stopped (e.g. EndTaskFrame) in which case we would send
an EndFrame down the pipeline.
"""
if isinstance(frame, self._reached_upstream_types):
if isinstance(frame, tuple(self._reached_upstream_types)):
await self._call_event_handler("on_frame_reached_upstream", frame)
if isinstance(frame, EndTaskFrame):
# Tell the task we should end nicely.
logger.debug(f"{self}: received end task frame {frame}")
logger.debug(f"{self}: received end task frame upstream {frame}")
await self.queue_frame(EndFrame(reason=frame.reason))
elif isinstance(frame, CancelTaskFrame):
# Tell the task we should end right away.
logger.debug(f"{self}: received cancel task frame {frame}")
logger.debug(f"{self}: received cancel task frame upstream {frame}")
await self.queue_frame(CancelFrame(reason=frame.reason))
elif isinstance(frame, StopTaskFrame):
# Tell the task we should stop nicely.
logger.debug(f"{self}: received stop task frame {frame}")
logger.debug(f"{self}: received stop task frame upstream {frame}")
await self.queue_frame(StopFrame())
elif isinstance(frame, InterruptionTaskFrame):
# Tell the task we should interrupt the pipeline. Note that we are
# bypassing the push queue and directly queue into the
# pipeline. This is in case the push task is blocked waiting for a
# pipeline-ending frame to finish traversing the pipeline.
logger.debug(f"{self}: received interruption task frame {frame}")
logger.debug(f"{self}: received interruption task frame upstream {frame}")
await self._pipeline.queue_frame(InterruptionFrame())
elif isinstance(frame, ErrorFrame):
await self._call_event_handler("on_pipeline_error", frame)
@@ -766,11 +913,12 @@ class PipelineTask(BasePipelineTask):
processors have handled the EndFrame and therefore we can exit the task
cleanly.
"""
if isinstance(frame, self._reached_downstream_types):
if isinstance(frame, tuple(self._reached_downstream_types)):
await self._call_event_handler("on_frame_reached_downstream", frame)
if isinstance(frame, StartFrame):
await self._call_event_handler("on_pipeline_started", frame)
await self._observer.on_pipeline_started()
# Start heartbeat tasks now that StartFrame has been processed
# by all processors in the pipeline
@@ -789,6 +937,18 @@ class PipelineTask(BasePipelineTask):
self._pipeline_end_event.set()
elif isinstance(frame, HeartbeatFrame):
await self._heartbeat_queue.put(frame)
elif isinstance(frame, EndTaskFrame):
logger.debug(f"{self}: received end task frame downstream {frame}")
await self.queue_frame(EndTaskFrame(reason=frame.reason), FrameDirection.UPSTREAM)
elif isinstance(frame, StopTaskFrame):
logger.debug(f"{self}: received stop task frame downstream {frame}")
await self.queue_frame(StopTaskFrame(), FrameDirection.UPSTREAM)
elif isinstance(frame, CancelTaskFrame):
logger.debug(f"{self}: received cancel task frame downstream {frame}")
await self.queue_frame(CancelTaskFrame(reason=frame.reason), FrameDirection.UPSTREAM)
elif isinstance(frame, InterruptionTaskFrame):
logger.debug(f"{self}: received interruption task frame downstream {frame}")
await self.queue_frame(InterruptionTaskFrame(), FrameDirection.UPSTREAM)
async def _heartbeat_push_handler(self):
"""Push heartbeat frames at regular intervals."""
@@ -853,9 +1013,51 @@ class PipelineTask(BasePipelineTask):
return False
return True
async def _load_setup_files(self):
"""Dynamically setup pipeline task from files listed in PIPECAT_SETUP_FILES.
Each file should contain a `setup_pipeline_task(task)` async function
that receives the `PipelineTask` instance and can perform any custom
setup (e.g., adding event handlers, observers, or modifying task
configuration).
"""
setup_files = [f for f in os.environ.get("PIPECAT_SETUP_FILES", "").split(":") if f]
for f in setup_files:
try:
path = Path(f).resolve()
module_name = path.stem
spec = importlib.util.spec_from_file_location(module_name, str(path))
if spec and spec.loader:
logger.debug(f"{self} running setup from {path}")
# Load module.
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# Run setup function.
if hasattr(module, "setup_pipeline_task"):
await module.setup_pipeline_task(self)
else:
logger.warning(
f"{self} setup file {path} has no setup_pipeline_task function"
)
except Exception as e:
logger.error(f"{self} error running external setup from {f}: {e}")
async def _load_observer_files(self):
observer_files = os.environ.get("PIPECAT_OBSERVER_FILES", "").split(":")
"""Dynamically load observers from files listed in PIPECAT_OBSERVER_FILES."""
observer_files = [f for f in os.environ.get("PIPECAT_OBSERVER_FILES", "").split(":") if f]
for f in observer_files:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Observer files (and environment variable `PIPECAT_OBSERVER_FILES`) is deprecated, use setup files instead (and `PIPECAT_SETUP_FILES`) instead.",
DeprecationWarning,
)
try:
path = Path(f).resolve()
module_name = path.stem
@@ -879,3 +1081,27 @@ class PipelineTask(BasePipelineTask):
tasks = [t.get_name() for t in self._task_manager.current_tasks()]
if tasks:
logger.warning(f"Dangling tasks detected: {tasks}")
def _create_start_metadata(self) -> Dict[str, Any]:
"""Build and return start metadata including user-provided values."""
start_metadata = {}
# NOTE(aleix): Remove when OpenAILLMContext/LLMUserContextAggregator is removed.
if self._find_processor(self._pipeline, LLMUserContextAggregator):
start_metadata["deprecated_openaillmcontext"] = True
# Update with user provided metadata.
start_metadata.update(self._params.start_metadata)
return start_metadata
def _find_processor(self, processor: FrameProcessor, processor_type: Type[T]) -> Optional[T]:
"""Recursively find a processor of the given type in the pipeline."""
if isinstance(processor, processor_type):
return processor
for p in processor.processors:
found = self._find_processor(p, processor_type)
if found:
return found
return None

16
src/pipecat/pipeline/task_observer.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -39,6 +39,12 @@ class Proxy:
observer: BaseObserver
class _PipelineStartedSignal:
"""Internal sentinel queued to observers when the pipeline has started."""
pass
class TaskObserver(BaseObserver):
"""Proxy observer that manages multiple observers without blocking the pipeline.
@@ -129,6 +135,10 @@ class TaskObserver(BaseObserver):
for proxy in self._proxies:
await proxy.cleanup()
async def on_pipeline_started(self):
"""Forward pipeline started signal to all managed observers."""
await self._send_to_proxy(_PipelineStartedSignal())
async def on_process_frame(self, data: FrameProcessed):
"""Queue frame data for all managed observers.
@@ -186,7 +196,9 @@ class TaskObserver(BaseObserver):
while True:
data = await queue.get()
if isinstance(data, FramePushed):
if isinstance(data, _PipelineStartedSignal):
await observer.on_pipeline_started()
elif isinstance(data, FramePushed):
if on_push_frame_deprecated:
await observer.on_push_frame(
data.source, data.destination, data.frame, data.direction, data.timestamp

53
src/pipecat/pipeline/to_be_updated/merge_pipeline.py
View File

@@ -1,53 +0,0 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Sequential pipeline merging for Pipecat.
This module provides a pipeline implementation that sequentially merges
the output from multiple pipelines, processing them one after another
in a specified order.
"""
from typing import List
from pipecat.frames.frames import EndFrame, EndPipeFrame
from pipecat.pipeline.pipeline import Pipeline
class SequentialMergePipeline(Pipeline):
"""Pipeline that sequentially merges output from multiple pipelines.
This pipeline merges the sink queues from a list of pipelines by processing
frames from each pipeline's sink sequentially in the order specified. Each
pipeline runs to completion before the next one begins processing.
"""
def __init__(self, pipelines: List[Pipeline]):
"""Initialize the sequential merge pipeline.
Args:
pipelines: List of pipelines to merge sequentially. Pipelines will
be processed in the order they appear in this list.
"""
super().__init__([])
self.pipelines = pipelines
async def run_pipeline(self):
"""Run all pipelines sequentially and merge their output.
Processes each pipeline in order, consuming all frames from each
pipeline's sink until an EndFrame or EndPipeFrame is encountered,
then moves to the next pipeline. After all pipelines complete,
sends a final EndFrame to signal completion.
"""
for idx, pipeline in enumerate(self.pipelines):
while True:
frame = await pipeline.sink.get()
if isinstance(frame, EndFrame) or isinstance(frame, EndPipeFrame):
break
await self.sink.put(frame)
await self.sink.put(EndFrame())

4
src/pipecat/processors/aggregators/dtmf_aggregator.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -104,7 +104,7 @@ class DTMFAggregator(FrameProcessor):
# For first digit, schedule interruption.
if is_first_digit:
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
# Check for immediate flush conditions
if frame.button == self._termination_digit:

2
src/pipecat/processors/aggregators/gated.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

4
src/pipecat/processors/aggregators/gated_llm_context.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -9,7 +9,7 @@
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, LLMContextFrame, StartFrame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.sync.base_notifier import BaseNotifier
from pipecat.utils.sync.base_notifier import BaseNotifier
class GatedLLMContextAggregator(FrameProcessor):

2
src/pipecat/processors/aggregators/gated_open_ai_llm_context.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

60
src/pipecat/processors/aggregators/llm_context.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -78,12 +78,29 @@ class LLMContext:
from OpenAILLMContext to LLMContext. New user code should use
LLMContext directly.
.. deprecated:: 0.0.99
`from_openai_context()` is deprecated and will be removed in a future version.
Directly use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
Args:
openai_context: The OpenAI LLM context to convert.
Returns:
New LLMContext instance with converted messages and settings.
"""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"from_openai_context() (likely invoked by create_context_aggregator()) is deprecated and will be removed in a future version. "
"Directly use the universal LLMContext and LLMContextAggregatorPair instead. "
"See OpenAILLMContext docstring for migration guide.",
DeprecationWarning,
stacklevel=2,
)
# Convert tools to ToolsSchema if needed.
# If the tools are already a ToolsSchema, this is a no-op.
# Otherwise, we wrap them in a shim ToolsSchema.
@@ -150,21 +167,29 @@ class LLMContext:
Args:
role: The role of this message (defaults to "user").
format: Image format (e.g., 'RGB', 'RGBA').
format: Image format (e.g., 'RGB', 'RGBA', or, if already encoded,
the MIME type like 'image/jpeg').
size: Image dimensions as (width, height) tuple.
image: Raw image bytes.
text: Optional text to include with the image.
"""
# Format is a mime type: image is already encoded
image_already_encoded = format.startswith("image/")
def encode_image():
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
if image_already_encoded:
bytes = image
else:
# Encode to JPEG
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
bytes = buffer.getvalue()
encoded_image = base64.b64encode(bytes).decode("utf-8")
return encoded_image
encoded_image = await asyncio.to_thread(encode_image)
url = f"data:image/jpeg;base64,{encoded_image}"
url = f"data:{format if image_already_encoded else 'image/jpeg'};base64,{encoded_image}"
return LLMContext.create_image_url_message(role=role, url=url, text=text)
@@ -179,13 +204,12 @@ class LLMContext:
audio_frames: List of audio frame objects to include.
text: Optional text to include with the audio.
"""
content = [{"type": "text", "text": text}]
async def encode_audio():
def encode_audio():
sample_rate = audio_frames[0].sample_rate
num_channels = audio_frames[0].num_channels
content = []
content.append({"type": "text", "text": text})
data = b"".join(frame.audio for frame in audio_frames)
with io.BytesIO() as buffer:
@@ -195,7 +219,7 @@ class LLMContext:
wf.setframerate(sample_rate)
wf.writeframes(data)
encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8")
encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8")
return encoded_audio
encoded_audio = await asyncio.to_thread(encode_audio)
@@ -231,7 +255,7 @@ class LLMContext:
this method, which is part of the public API of OpenAILLMContext but
doesn't need to be for LLMContext.
.. deprecated::
.. deprecated:: 0.0.92
Use `get_messages()` instead.
Returns:
@@ -334,18 +358,26 @@ class LLMContext:
self._tool_choice = tool_choice
async def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None
self,
*,
format: str,
size: tuple[int, int],
image: bytes,
text: Optional[str] = None,
role: str = "user",
):
"""Add a message containing an image frame.
Args:
format: Image format (e.g., 'RGB', 'RGBA').
format: Image format (e.g., 'RGB', 'RGBA', or, if already encoded,
the MIME type like 'image/jpeg').
size: Image dimensions as (width, height) tuple.
image: Raw image bytes.
text: Optional text to include with the image.
role: The role of this message (defaults to "user").
"""
message = await LLMContext.create_image_message(
format=format, size=size, image=image, text=text
role=role, format=format, size=size, image=image, text=text
)
self.add_message(message)

480
src/pipecat/processors/aggregators/llm_context_summarizer.py Normal file
View File

@@ -0,0 +1,480 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""This module defines a summarizer for managing LLM context summarization."""
import asyncio
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from loguru import logger
from pipecat.frames.frames import (
Frame,
InterruptionFrame,
LLMContextSummaryRequestFrame,
LLMContextSummaryResultFrame,
LLMFullResponseStartFrame,
LLMSummarizeContextFrame,
)
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject
from pipecat.utils.context.llm_context_summarization import (
DEFAULT_SUMMARIZATION_TIMEOUT,
LLMAutoContextSummarizationConfig,
LLMContextSummarizationUtil,
LLMContextSummaryConfig,
)
if TYPE_CHECKING:
from pipecat.services.llm_service import LLMService
@dataclass
class SummaryAppliedEvent:
"""Event data emitted when context summarization completes successfully.
Parameters:
original_message_count: Number of messages before summarization.
new_message_count: Number of messages after summarization.
summarized_message_count: Number of messages that were compressed
into the summary.
preserved_message_count: Number of recent messages preserved
uncompressed.
"""
original_message_count: int
new_message_count: int
summarized_message_count: int
preserved_message_count: int
class LLMContextSummarizer(BaseObject):
"""Summarizer for managing LLM context summarization.
This class manages context summarization, either automatically when token or
message limits are reached, or on-demand when an ``LLMSummarizeContextFrame``
is received. It monitors the LLM context size, triggers summarization requests,
and applies the results to compress conversation history.
When ``auto_trigger=True`` (the default), summarization is triggered
automatically based on the configured thresholds in
``LLMAutoContextSummarizationConfig``. When ``auto_trigger=False``,
threshold checks are skipped and summarization only happens when an
``LLMSummarizeContextFrame`` is explicitly pushed into the pipeline.
Both modes can coexist: set ``auto_trigger=True`` and also push
``LLMSummarizeContextFrame`` at any time to force an immediate summarization
(subject to the ``_summarization_in_progress`` guard).
Event handlers available:
- on_request_summarization: Emitted when summarization should be triggered.
The aggregator should broadcast this frame to the LLM service.
- on_summary_applied: Emitted after a summary has been successfully applied
to the context. Receives a SummaryAppliedEvent with metrics about the
compression.
Example::
@summarizer.event_handler("on_request_summarization")
async def on_request_summarization(summarizer, frame: LLMContextSummaryRequestFrame):
await aggregator.broadcast_frame(
LLMContextSummaryRequestFrame,
request_id=frame.request_id,
context=frame.context,
...
)
@summarizer.event_handler("on_summary_applied")
async def on_summary_applied(summarizer, event: SummaryAppliedEvent):
logger.info(f"Compressed {event.original_message_count} -> {event.new_message_count} messages")
"""
def __init__(
self,
*,
context: LLMContext,
config: Optional[LLMAutoContextSummarizationConfig] = None,
auto_trigger: bool = True,
):
"""Initialize the context summarizer.
Args:
context: The LLM context to monitor and summarize.
config: Auto-summarization configuration controlling both trigger
thresholds and default summary generation parameters. If None,
uses default ``LLMAutoContextSummarizationConfig`` values.
auto_trigger: Whether to automatically trigger summarization when
thresholds are reached. When False, summarization only happens
when an ``LLMSummarizeContextFrame`` is pushed into the pipeline.
Defaults to True.
"""
super().__init__()
self._context = context
self._auto_config = config or LLMAutoContextSummarizationConfig()
self._auto_trigger = auto_trigger
self._task_manager: Optional[BaseTaskManager] = None
self._summarization_in_progress = False
self._pending_summary_request_id: Optional[str] = None
self._register_event_handler("on_request_summarization", sync=True)
self._register_event_handler("on_summary_applied")
@property
def task_manager(self) -> BaseTaskManager:
"""Returns the configured task manager."""
if not self._task_manager:
raise RuntimeError(f"{self} context summarizer was not properly setup")
return self._task_manager
async def setup(self, task_manager: BaseTaskManager):
"""Initialize the summarizer with the given task manager.
Args:
task_manager: The task manager to be associated with this instance.
"""
self._task_manager = task_manager
async def cleanup(self):
"""Cleanup the summarizer."""
await super().cleanup()
await self._clear_summarization_state()
async def process_frame(self, frame: Frame):
"""Process an incoming frame to detect when summarization is needed.
Args:
frame: The frame to be processed.
"""
if isinstance(frame, LLMFullResponseStartFrame):
await self._handle_llm_response_start(frame)
elif isinstance(frame, LLMSummarizeContextFrame):
await self._handle_manual_summarization_request(frame)
elif isinstance(frame, LLMContextSummaryResultFrame):
await self._handle_summary_result(frame)
elif isinstance(frame, InterruptionFrame):
await self._handle_interruption()
async def _handle_llm_response_start(self, frame: LLMFullResponseStartFrame):
"""Handle LLM response start to check if summarization is needed.
Args:
frame: The LLM response start frame.
"""
if self._should_summarize():
await self._request_summarization()
async def _handle_manual_summarization_request(self, frame: LLMSummarizeContextFrame):
"""Handle an explicit on-demand summarization request.
Reuses the same ``_request_summarization()`` code path as auto mode,
so bookkeeping (``_summarization_in_progress``,
``_pending_summary_request_id``) is always updated correctly.
Args:
frame: The manual summarization request frame, optionally carrying
a per-request :class:`~pipecat.utils.context.llm_context_summarization.LLMContextSummaryConfig`.
"""
if self._summarization_in_progress:
logger.debug(f"{self}: Summarization already in progress, ignoring manual request")
return
await self._request_summarization(config_override=frame.config)
async def _handle_interruption(self):
"""Handle interruption by canceling summarization in progress."""
# Reset summarization state to allow new requests. This is necessary because
# the request frame (LLMContextSummaryRequestFrame) may have been cancelled
# during interruption. We preserve _pending_summary_request_id to handle the
# response frame (LLMContextSummaryResultFrame), which is uninterruptible and
# will still be delivered.
self._summarization_in_progress = False
async def _clear_summarization_state(self):
"""Cancel pending summarization."""
if self._summarization_in_progress:
logger.debug(f"{self}: Clearing pending summarization")
self._summarization_in_progress = False
self._pending_summary_request_id = None
def _should_summarize(self) -> bool:
"""Determine if context summarization should be triggered.
Evaluates whether the current context has reached either the token
threshold or message count threshold that warrants compression.
Either threshold can be ``None`` to disable that check; at least one
must be set (enforced at config construction time).
Returns:
True if all conditions are met:
- ``auto_trigger`` is enabled
- No summarization currently in progress
- AND either:
- Token count exceeds ``max_context_tokens`` (when set)
- OR message count exceeds ``max_unsummarized_messages`` since last summary (when set)
"""
logger.trace(f"{self}: Checking if context summarization is needed")
if not self._auto_trigger:
return False
if self._summarization_in_progress:
logger.debug(f"{self}: Summarization already in progress")
return False
# Estimate tokens in context
total_tokens = LLMContextSummarizationUtil.estimate_context_tokens(self._context)
num_messages = len(self._context.messages)
# Check if we've reached the token limit
token_limit = self._auto_config.max_context_tokens
token_limit_exceeded = token_limit is not None and total_tokens >= token_limit
# Check if we've exceeded max unsummarized messages
messages_since_summary = len(self._context.messages) - 1
message_threshold = self._auto_config.max_unsummarized_messages
message_threshold_exceeded = (
message_threshold is not None and messages_since_summary >= message_threshold
)
logger.trace(
f"{self}: Context has {num_messages} messages, "
f"~{total_tokens} tokens (limit: {token_limit if token_limit is not None else 'disabled'}), "
f"{messages_since_summary} messages since last summary "
f"(message threshold: {message_threshold if message_threshold is not None else 'disabled'})"
)
# Trigger if either limit is exceeded
if not token_limit_exceeded and not message_threshold_exceeded:
logger.trace(
f"{self}: Neither token limit nor message threshold exceeded, skipping summarization"
)
return False
reason = []
if token_limit_exceeded:
reason.append(f"~{total_tokens} tokens (>={token_limit} limit)")
if message_threshold_exceeded:
reason.append(f"{messages_since_summary} messages (>={message_threshold} threshold)")
logger.debug(f"{self}: ✓ Summarization needed - {', '.join(reason)}")
return True
async def _request_summarization(
self, config_override: Optional[LLMContextSummaryConfig] = None
):
"""Request context summarization from LLM service.
Creates a summarization request frame and either handles it directly
using a dedicated LLM (if configured) or emits it via event handler
for the pipeline's primary LLM.
Tracks the request ID to match async responses and prevent race conditions.
Args:
config_override: Optional per-request summary configuration. If provided,
overrides the default summary generation settings from
``self._auto_config.summary_config``.
"""
# Generate unique request ID
request_id = str(uuid.uuid4())
summary_config = config_override or self._auto_config.summary_config
# Mark summarization in progress
self._summarization_in_progress = True
self._pending_summary_request_id = request_id
logger.debug(f"{self}: Sending summarization request (request_id={request_id})")
# Create the request frame
request_frame = LLMContextSummaryRequestFrame(
request_id=request_id,
context=self._context,
min_messages_to_keep=summary_config.min_messages_after_summary,
target_context_tokens=summary_config.target_context_tokens,
summarization_prompt=summary_config.summary_prompt,
summarization_timeout=summary_config.summarization_timeout,
)
if summary_config.llm:
# Use dedicated LLM directly — no need to involve the pipeline
self.task_manager.create_task(
self._generate_summary_with_dedicated_llm(summary_config.llm, request_frame),
f"{self}-dedicated-llm-summary",
)
else:
# Emit event for aggregator to broadcast to the pipeline LLM
await self._call_event_handler("on_request_summarization", request_frame)
async def _generate_summary_with_dedicated_llm(
self, llm: "LLMService", frame: LLMContextSummaryRequestFrame
):
"""Generate summary using a dedicated LLM service.
Calls the dedicated LLM's _generate_summary directly and feeds the
result back through _handle_summary_result, bypassing the pipeline.
Args:
llm: The dedicated LLM service to use for summarization.
frame: The summarization request frame.
"""
timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
try:
summary, last_index = await asyncio.wait_for(
llm._generate_summary(frame),
timeout=timeout,
)
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary=summary,
last_summarized_index=last_index,
)
except asyncio.TimeoutError:
error = f"Context summarization timed out after {timeout}s"
logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary="",
last_summarized_index=-1,
error=error,
)
except Exception as e:
error = f"Error generating context summary: {e}"
logger.error(f"{self}: {error}")
result_frame = LLMContextSummaryResultFrame(
request_id=frame.request_id,
summary="",
last_summarized_index=-1,
error=error,
)
await self._handle_summary_result(result_frame)
async def _handle_summary_result(self, frame: LLMContextSummaryResultFrame):
"""Handle context summarization result from LLM service.
Processes the summary result by validating the request ID, checking for
errors, validating context state, and applying the summary.
Args:
frame: The summary result frame containing the generated summary.
"""
logger.debug(f"{self}: Received summary result (request_id={frame.request_id})")
# Check if this is the result we're waiting for. Both auto and manual
# summarization set _pending_summary_request_id via _request_summarization(),
# so this check always applies.
if frame.request_id != self._pending_summary_request_id:
logger.debug(f"{self}: Ignoring stale summary result (request_id={frame.request_id})")
return
# Clear pending state
await self._clear_summarization_state()
# Check for errors
if frame.error:
logger.error(f"{self}: Context summarization failed: {frame.error}")
return
# Validate context state
if not self._validate_summary_context(frame.last_summarized_index):
logger.warning(f"{self}: Context state changed, skipping summary application")
return
# Apply summary
await self._apply_summary(frame.summary, frame.last_summarized_index)
def _validate_summary_context(self, last_summarized_index: int) -> bool:
"""Validate that context state is still valid for applying summary.
Args:
last_summarized_index: The index of the last summarized message.
Returns:
True if the context state is still consistent with the summary.
"""
if last_summarized_index < 0:
return False
# Check if we still have enough messages
if last_summarized_index >= len(self._context.messages):
return False
min_keep = self._auto_config.summary_config.min_messages_after_summary
remaining = len(self._context.messages) - 1 - last_summarized_index
if remaining < min_keep:
return False
return True
async def _apply_summary(self, summary: str, last_summarized_index: int):
"""Apply summary to compress the conversation context.
Reconstructs the context with:
[first_system_message] + [summary_message] + [recent_messages]
Args:
summary: The generated summary text.
last_summarized_index: Index of the last message that was summarized.
"""
config = self._auto_config.summary_config
messages = self._context.messages
# Find the first system message to preserve. LLMSpecificMessage instances are excluded
# because they are not dict-like and never represent a system message; they hold
# service-specific metadata (e.g. thinking blocks) that is always paired with a
# standard message.
first_system_msg = next(
(
msg
for msg in messages
if not isinstance(msg, LLMSpecificMessage) and msg.get("role") == "system"
),
None,
)
# Get recent messages to keep
recent_messages = messages[last_summarized_index + 1 :]
# Create summary message as a user message (the summary is context
# provided *to* the assistant, not something the assistant said)
summary_content = config.summary_message_template.format(summary=summary)
summary_message = {"role": "user", "content": summary_content}
# Reconstruct context
new_messages = []
if first_system_msg:
new_messages.append(first_system_msg)
new_messages.append(summary_message)
new_messages.extend(recent_messages)
# Update context
original_message_count = len(messages)
num_system_preserved = 1 if first_system_msg else 0
self._context.set_messages(new_messages)
# Messages actually summarized = index range minus the preserved system message
summarized_count = last_summarized_index + 1 - num_system_preserved
logger.info(
f"{self}: Applied context summary, compressed {summarized_count} messages "
f"into summary. Context now has {len(new_messages)} messages (was {original_message_count})"
)
# Emit event for observability
event = SummaryAppliedEvent(
original_message_count=original_message_count,
new_message_count=len(new_messages),
summarized_message_count=summarized_count,
preserved_message_count=len(recent_messages) + num_system_preserved,
)
await self._call_event_handler("on_summary_applied", event)

88
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -12,6 +12,7 @@ LLM processing, and text-to-speech components in conversational AI pipelines.
"""
import asyncio
import warnings
from abc import abstractmethod
from dataclasses import dataclass
from typing import Dict, List, Literal, Optional, Set
@@ -66,6 +67,10 @@ from pipecat.utils.time import time_now_iso8601
class LLMUserAggregatorParams:
"""Parameters for configuring LLM user aggregation behavior.
.. deprecated:: 0.0.99
This class is deprecated, use the new universal `LLMContext` and
`LLMContextAggregatorPair`.
Parameters:
aggregation_timeout: Maximum time in seconds to wait for additional
transcription content before pushing aggregated result. This
@@ -87,6 +92,10 @@ class LLMUserAggregatorParams:
class LLMAssistantAggregatorParams:
"""Parameters for configuring LLM assistant aggregation behavior.
.. deprecated:: 0.0.99
This class is deprecated, use the new universal `LLMContext` and
`LLMContextAggregatorPair`.
Parameters:
expect_stripped_words: Whether to expect and handle stripped words
in text frames by adding spaces between tokens. This parameter is
@@ -175,6 +184,11 @@ class BaseLLMResponseAggregator(FrameProcessor):
The aggregators keep a store (e.g. message list or LLM context) of the current
conversation, storing messages from both users and the bot.
.. deprecated:: 0.0.99
`BaseLLMResponseAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
def __init__(self, **kwargs):
@@ -182,7 +196,21 @@ class BaseLLMResponseAggregator(FrameProcessor):
Args:
**kwargs: Additional arguments passed to parent FrameProcessor.
.. deprecated:: 0.0.99
`BaseLLMResponseAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
f"{self.__class__.__name__} (likely created with create_context_aggregator()) is deprecated and will be removed in a future version. "
"Use the universal LLMContext and LLMContextAggregatorPair instead. "
"See OpenAILLMContext docstring for migration guide.",
DeprecationWarning,
stacklevel=2,
)
super().__init__(**kwargs)
@property
@@ -274,6 +302,11 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator):
This aggregator maintains conversation state using an OpenAILLMContext and
pushes OpenAILLMContextFrame objects as aggregation frames. It provides
common functionality for context-based conversation management.
.. deprecated:: 0.0.99
`LLMContextResponseAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
def __init__(self, *, context: OpenAILLMContext, role: str, **kwargs):
@@ -283,7 +316,13 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator):
context: The OpenAI LLM context to use for conversation storage.
role: The role this aggregator represents (e.g. "user", "assistant").
**kwargs: Additional arguments passed to parent class.
.. deprecated:: 0.0.99
`LLMContextResponseAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMUserAggregator` and `LLMAssistantAggregator` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
# Super handles deprecation warning
super().__init__(**kwargs)
self._context = context
self._role = role
@@ -326,8 +365,6 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator):
Returns:
LLMContextFrame containing the current context.
"""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -400,6 +437,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
The aggregator uses timeouts to handle cases where transcriptions arrive
after VAD events or when no VAD is available.
.. deprecated:: 0.0.99
`LLMUserContextAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
def __init__(
@@ -415,15 +457,19 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
context: The OpenAI LLM context for conversation storage.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments. Supports deprecated 'aggregation_timeout'.
.. deprecated:: 0.0.99
`LLMUserContextAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
# Super handles deprecation warning
super().__init__(context=context, role="user", **kwargs)
self._params = params or LLMUserAggregatorParams()
self._vad_params: Optional[VADParams] = None
self._turn_params: Optional[SmartTurnParams] = None
if "aggregation_timeout" in kwargs:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -535,7 +581,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
logger.debug(
"Interruption conditions met - pushing interruption and aggregation"
)
await self.push_interruption_task_frame_and_wait()
await self.broadcast_interruption()
await self._process_aggregation()
else:
logger.debug("Interruption conditions not met - not pushing aggregation")
@@ -746,6 +792,11 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
The aggregator manages function calls in progress and coordinates between
text generation and tool execution phases of LLM responses.
.. deprecated:: 0.0.99
`LLMAssistantContextAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
def __init__(
@@ -761,13 +812,17 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
context: The OpenAI LLM context for conversation storage.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments. Supports deprecated 'expect_stripped_words'.
.. deprecated:: 0.0.99
`LLMAssistantContextAggregator` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
# Super handles deprecation warning
super().__init__(context=context, role="assistant", **kwargs)
self._params = params or LLMAssistantAggregatorParams()
if "expect_stripped_words" in kwargs:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -969,10 +1024,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
logger.debug(
f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]"
)
if frame.tool_call_id not in self._function_calls_in_progress:
return
if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption:
function_call = self._function_calls_in_progress.get(frame.tool_call_id)
if function_call and function_call.cancel_on_interruption:
await self.handle_function_call_cancel(frame)
del self._function_calls_in_progress[frame.tool_call_id]
@@ -989,6 +1042,11 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
del self._function_calls_in_progress[frame.request.tool_call_id]
# Call the result_callback if provided. This signals that the image
# has been retrieved and the function call can now complete.
if frame.request and frame.request.result_callback:
await frame.request.result_callback(None)
await self.handle_user_image_frame(frame)
await self.push_aggregation()
await self.push_context_frame(FrameDirection.UPSTREAM)
@@ -1001,7 +1059,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
await self.push_aggregation()
async def _handle_text(self, frame: TextFrame):
if not self._started:
if not frame.append_to_context:
return
if self._params.expect_stripped_words:
@@ -1039,8 +1097,6 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments passed to parent class.
"""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
@@ -1086,8 +1142,6 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments passed to parent class.
"""
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(

1086
src/pipecat/processors/aggregators/llm_response_universal.py
View File

File diff suppressed because it is too large Load Diff

103
src/pipecat/processors/aggregators/llm_text_processor.py Normal file
View File

@@ -0,0 +1,103 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""LLM text processor module for processing and aggregating raw LLM output text.
This processor will convert LLMTextFrames into AggregatedTextFrames based on the
configured text aggregator. Using the customizable aggregator, it provides
functionality to handle or manipulate LLM text frames before they are sent to other
components such as TTS services or context aggregators. It can be used to pre-aggregate
and categorize, modify, or filter direct output tokens from the LLM.
"""
from typing import Optional
from pipecat.frames.frames import (
AggregatedTextFrame,
EndFrame,
Frame,
InterruptionFrame,
LLMFullResponseEndFrame,
LLMTextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator
class LLMTextProcessor(FrameProcessor):
"""A processor for handling or manipulating LLM text frames before they are processed further.
This processor will convert LLMTextFrames into AggregatedTextFrames based on the configured
text aggregator. Using the customizable aggregator, it provides functionality to handle or
manipulate LLM text frames before they are sent to other components such as TTS services or
context aggregators. It can be used to pre-aggregate and categorize, modify, or filter direct
output tokens from the LLM.
"""
def __init__(self, *, text_aggregator: Optional[BaseTextAggregator] = None, **kwargs):
"""Initialize the LLM text processor.
Args:
text_aggregator: An optional text aggregator to use for processing LLM text frames. By
default, a SimpleTextAggregator aggregating by sentence will be used.
**kwargs: Additional arguments passed to parent class.
TODO: Allow transformations per aggregation type or all (and deprecate the TTS filters).
"""
super().__init__(**kwargs)
self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process an LLMTextFrames using the aggregator to generate AggregatedTextFrames.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
if isinstance(frame, InterruptionFrame):
await self._handle_interruption(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, LLMTextFrame):
await self._handle_llm_text(frame)
elif isinstance(frame, LLMFullResponseEndFrame):
await self._handle_llm_end(frame.skip_tts)
await self.push_frame(frame, direction)
elif isinstance(frame, EndFrame):
await self._handle_llm_end()
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
async def _handle_interruption(self, _):
"""Handle interruptions by resetting the text aggregator."""
await self._text_aggregator.handle_interruption()
async def reset(self):
"""Reset the internal state of the text processor and its aggregator."""
await self._text_aggregator.reset()
async def _handle_llm_text(self, in_frame: LLMTextFrame):
async for aggregation in self._text_aggregator.aggregate(in_frame.text):
out_frame = AggregatedTextFrame(
text=aggregation.text,
aggregated_by=aggregation.type,
)
out_frame.skip_tts = in_frame.skip_tts
await self.push_frame(out_frame)
async def _handle_llm_end(self, skip_tts: Optional[bool] = None):
# Flush any remaining text
remaining = await self._text_aggregator.flush()
if remaining:
out_frame = AggregatedTextFrame(
text=remaining.text,
aggregated_by=remaining.type,
)
out_frame.skip_tts = skip_tts
await self.push_frame(out_frame)

54
src/pipecat/processors/aggregators/openai_llm_context.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -8,12 +8,18 @@
This module provides classes for managing OpenAI-specific conversation contexts,
including message handling, tool management, and image/audio processing capabilities.
.. deprecated:: 0.0.99
This module is deprecated.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
import base64
import copy
import io
import json
import warnings
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
@@ -28,7 +34,6 @@ from PIL import Image
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.frames.frames import AudioRawFrame, Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
# JSON custom encoder to handle bytes arrays so that we can log contexts
# with images to the console.
@@ -62,6 +67,20 @@ class OpenAILLMContext:
Handles message history, tool definitions, tool choices, and multimedia content
for OpenAI API conversations. Provides methods for message manipulation,
content formatting, and integration with various LLM adapters.
.. deprecated:: 0.0.99
`OpenAILLMContext` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
**Before:**
context = OpenAILLMContext(messages, tools)
context_aggregator = llm.create_context_aggregator(context)
**After:**
context = LLMContext(messages, tools)
context_aggregator = LLMContextAggregatorPair(context)
"""
def __init__(
@@ -76,7 +95,21 @@ class OpenAILLMContext:
messages: Initial list of conversation messages.
tools: Available tools for the LLM to use.
tool_choice: Tool selection strategy for the LLM.
.. deprecated:: 0.0.99
`OpenAILLMContext` is deprecated and will be removed in a future version.
Use the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
"""
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"OpenAILLMContext is deprecated and will be removed in a future version. "
"Use the universal LLMContext and LLMContextAggregatorPair instead. "
"See OpenAILLMContext docstring for migration guide.",
DeprecationWarning,
stacklevel=2,
)
self._messages: List[ChatCompletionMessageParam] = messages if messages else []
self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice
self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools
@@ -356,8 +389,25 @@ class OpenAILLMContextFrame(Frame):
API. The context in this message is also mutable, and will be changed by the
OpenAIContextAggregator frame processor.
.. deprecated:: 0.0.99
`OpenAILLMContextFrame` is deprecated and will be removed in a future version.
Use `LLMContextFrame` with the universal `LLMContext` and `LLMContextAggregatorPair` instead.
See `OpenAILLMContext` docstring for migration guide.
Parameters:
context: The OpenAI LLM context containing messages, tools, and configuration.
"""
context: OpenAILLMContext
def __post_init__(self):
super().__post_init__()
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"OpenAILLMContextFrame is deprecated and will be removed in a future version. "
"Use LLMContextFrame with the universal `LLMContext` and `LLMContextAggregatorPair` instead. "
"See OpenAILLMContext docstring for migration guide.",
DeprecationWarning,
stacklevel=2,
)

2
src/pipecat/processors/aggregators/sentence.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/processors/aggregators/user_response.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/processors/aggregators/vision_image_frame.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/processors/async_generator.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

118
src/pipecat/processors/audio/audio_buffer_processor.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -11,9 +11,10 @@ of audio from both user input and bot output sources, with support for various a
configurations and event-driven processing.
"""
import time
from typing import Optional
from loguru import logger
from pipecat.audio.utils import create_stream_resampler, interleave_stereo_audio, mix_audio
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
@@ -104,10 +105,6 @@ class AudioBufferProcessor(FrameProcessor):
self._user_turn_audio_buffer = bytearray()
self._bot_turn_audio_buffer = bytearray()
# Intermittent (non continous user stream variables)
self._last_user_frame_at = 0
self._last_bot_frame_at = 0
self._recording = False
self._input_resampler = create_stream_resampler()
@@ -209,25 +206,55 @@ class AudioBufferProcessor(FrameProcessor):
async def _process_recording(self, frame: Frame):
"""Process audio frames for recording."""
# Track speaking state here (not just in _process_turn_recording) so the
# silence-injection guards below work regardless of enable_turn_audio.
if isinstance(frame, UserStartedSpeakingFrame):
self._user_speaking = True
elif isinstance(frame, UserStoppedSpeakingFrame):
self._user_speaking = False
elif isinstance(frame, BotStartedSpeakingFrame):
self._bot_speaking = True
elif isinstance(frame, BotStoppedSpeakingFrame):
self._bot_speaking = False
resampled = None
if isinstance(frame, InputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_user_frame_at)
self._user_audio_buffer.extend(silence)
# Add user audio.
resampled = await self._resample_input_audio(frame)
self._user_audio_buffer.extend(resampled)
# Save time of frame so we can compute silence.
self._last_user_frame_at = time.time()
elif self._recording and isinstance(frame, OutputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_bot_frame_at)
self._bot_audio_buffer.extend(silence)
# Add bot audio.
# Ignoring in case we don't have audio
if len(resampled) > 0:
# Sync bot buffer to current user position before adding user audio.
# We sync BEFORE extending to align both buffers at the same starting timestamp.
# For example, user buffer is at 100 bytes, and you receive 20 bytes of new audio
# - Bot buffer sees User is at 100. Bot pads itself to 100.
# - User buffer adds 20. User is now at 120.
# - Outcome: At index 100-120, we have User Audio and (potentially) Bot Audio or silence. They are aligned
# This gives the opportunity to the bot to send audio.
#
# If we synced AFTER, we'd pad the bot buffer with silence for the same
# window we just gave to the user, effectively "overwriting" that time slot
# with silence and causing the bot's audio to flicker or cut out.
#
# Skip silence injection if the bot is actively speaking to avoid
# inserting silence in the middle of a bot utterance (causes crackling).
if not self._bot_speaking:
self._sync_buffer_to_position(
self._bot_audio_buffer, len(self._user_audio_buffer)
)
# Add user audio.
self._user_audio_buffer.extend(resampled)
elif isinstance(frame, OutputAudioRawFrame):
resampled = await self._resample_output_audio(frame)
self._bot_audio_buffer.extend(resampled)
# Save time of frame so we can compute silence.
self._last_bot_frame_at = time.time()
# Ignoring in case we don't have audio
if len(resampled) > 0:
# Sync user buffer to current bot position before adding bot audio.
# Skip silence injection if the user is actively speaking to avoid
# inserting silence in the middle of a user utterance (causes crackling).
if not self._user_speaking:
self._sync_buffer_to_position(
self._user_audio_buffer, len(self._bot_audio_buffer)
)
# Add bot audio.
self._bot_audio_buffer.extend(resampled)
if self._buffer_size > 0 and (
len(self._user_audio_buffer) >= self._buffer_size
@@ -240,23 +267,34 @@ class AudioBufferProcessor(FrameProcessor):
if self._enable_turn_audio:
await self._process_turn_recording(frame, resampled)
def _sync_buffer_to_position(self, buffer: bytearray, target_position: int):
"""Pad buffer with silence if it's behind the target position.
This ensures both buffers stay synchronized by padding the lagging
buffer before new audio is added to the other buffer.
Args:
buffer: The buffer to potentially pad.
target_position: The position (in bytes) the buffer should reach.
"""
current_len = len(buffer)
if current_len < target_position:
silence_needed = target_position - current_len
buffer.extend(b"\x00" * silence_needed)
async def _process_turn_recording(self, frame: Frame, resampled_audio: Optional[bytes] = None):
"""Process frames for turn-based audio recording."""
if isinstance(frame, UserStartedSpeakingFrame):
self._user_speaking = True
elif isinstance(frame, UserStoppedSpeakingFrame):
# Speaking state (_user_speaking / _bot_speaking) is maintained by
# _process_recording so it is always up-to-date here.
if isinstance(frame, UserStoppedSpeakingFrame):
await self._call_event_handler(
"on_user_turn_audio_data", self._user_turn_audio_buffer, self.sample_rate, 1
)
self._user_speaking = False
self._user_turn_audio_buffer = bytearray()
elif isinstance(frame, BotStartedSpeakingFrame):
self._bot_speaking = True
elif isinstance(frame, BotStoppedSpeakingFrame):
await self._call_event_handler(
"on_bot_turn_audio_data", self._bot_turn_audio_buffer, self.sample_rate, 1
)
self._bot_speaking = False
self._bot_turn_audio_buffer = bytearray()
if isinstance(frame, InputAudioRawFrame) and resampled_audio:
@@ -281,8 +319,8 @@ class AudioBufferProcessor(FrameProcessor):
if len(self._user_audio_buffer) == 0 and len(self._bot_audio_buffer) == 0:
return
# Final alignment before we send the audio
self._align_track_buffers()
flush_time = time.time()
# Call original handler with merged audio
merged_audio = self.merge_audio_buffers()
@@ -299,9 +337,6 @@ class AudioBufferProcessor(FrameProcessor):
self._num_channels,
)
self._last_user_frame_at = flush_time
self._last_bot_frame_at = flush_time
def _buffer_has_audio(self, buffer: bytearray) -> bool:
"""Check if a buffer contains audio data."""
return buffer is not None and len(buffer) > 0
@@ -309,8 +344,6 @@ class AudioBufferProcessor(FrameProcessor):
def _reset_recording(self):
"""Reset recording state and buffers."""
self._reset_all_audio_buffers()
self._last_user_frame_at = time.time()
self._last_bot_frame_at = time.time()
def _reset_all_audio_buffers(self):
"""Reset all audio buffers to empty state."""
@@ -336,11 +369,9 @@ class AudioBufferProcessor(FrameProcessor):
target_len = max(user_len, bot_len)
if user_len < target_len:
self._user_audio_buffer.extend(b"\x00" * (target_len - user_len))
self._last_user_frame_at = max(self._last_user_frame_at, self._last_bot_frame_at)
self._sync_buffer_to_position(self._user_audio_buffer, target_len)
if bot_len < target_len:
self._bot_audio_buffer.extend(b"\x00" * (target_len - bot_len))
self._last_bot_frame_at = max(self._last_bot_frame_at, self._last_user_frame_at)
self._sync_buffer_to_position(self._bot_audio_buffer, target_len)
async def _resample_input_audio(self, frame: InputAudioRawFrame) -> bytes:
"""Resample audio frame to the target sample rate."""
@@ -353,14 +384,3 @@ class AudioBufferProcessor(FrameProcessor):
return await self._output_resampler.resample(
frame.audio, frame.sample_rate, self._sample_rate
)
def _compute_silence(self, from_time: float) -> bytes:
"""Compute silence to insert based on time gap."""
quiet_time = time.time() - from_time
# We should get audio frames very frequently. We introduce silence only
# if there's a big enough gap of 1s.
if from_time == 0 or quiet_time < 1.0:
return b""
num_bytes = int(quiet_time * self._sample_rate) * 2
silence = b"\x00" * num_bytes
return silence

108
src/pipecat/processors/audio/vad_processor.py Normal file
View File

@@ -0,0 +1,108 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Voice Activity Detection processor for detecting speech in audio streams.
This module provides a VADProcessor that wraps a VADController to process
audio frames and push VAD-related frames into the pipeline.
"""
from typing import Type
from loguru import logger
from pipecat.audio.vad.vad_analyzer import VADAnalyzer
from pipecat.audio.vad.vad_controller import VADController
from pipecat.frames.frames import (
Frame,
UserSpeakingFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class VADProcessor(FrameProcessor):
"""Processes audio frames through voice activity detection.
This processor wraps a VADController to detect speech in audio streams
and push VAD frames into the pipeline:
- ``VADUserStartedSpeakingFrame``: Pushed when speech begins.
- ``VADUserStoppedSpeakingFrame``: Pushed when speech ends.
- ``UserSpeakingFrame``: Pushed periodically while speech is detected.
Example::
vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer())
"""
def __init__(
self,
*,
vad_analyzer: VADAnalyzer,
speech_activity_period: float = 0.2,
**kwargs,
):
"""Initialize the VAD processor.
Args:
vad_analyzer: The VADAnalyzer instance for processing audio.
speech_activity_period: Minimum interval in seconds between
UserSpeakingFrame pushes. Defaults to 0.2.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs)
self._vad_controller = VADController(
vad_analyzer, speech_activity_period=speech_activity_period
)
# Push VAD frames when speech events are detected
@self._vad_controller.event_handler("on_speech_started")
async def on_speech_started(_controller):
logger.debug(f"{self}: User started speaking")
await self.broadcast_frame(
VADUserStartedSpeakingFrame,
start_secs=_controller._vad_analyzer.params.start_secs,
)
@self._vad_controller.event_handler("on_speech_stopped")
async def on_speech_stopped(_controller):
logger.debug(f"{self}: User stopped speaking")
await self.broadcast_frame(
VADUserStoppedSpeakingFrame,
stop_secs=_controller._vad_analyzer.params.stop_secs,
)
@self._vad_controller.event_handler("on_speech_activity")
async def on_speech_activity(_controller):
await self.broadcast_frame(UserSpeakingFrame)
# Wire up frame pushing from controller to processor
@self._vad_controller.event_handler("on_push_frame")
async def on_push_frame(_controller, frame: Frame, direction: FrameDirection):
await self.push_frame(frame, direction)
@self._vad_controller.event_handler("on_broadcast_frame")
async def on_broadcast_frame(_controller, frame_cls: Type[Frame], **kwargs):
await self.broadcast_frame(frame_cls, **kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame through VAD and forward it.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction)
# Forward the frame first, then let VAD controller process. This ensures:
# 1. StartFrame reaches downstream before SpeechControlParamsFrame is broadcast
# 2. Audio flows through immediately while VAD detection happens after
await self.push_frame(frame, direction)
# Let the VAD controller handle the frame
await self._vad_controller.process_frame(frame)

2
src/pipecat/processors/consumer_processor.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/processors/filters/frame_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

20
src/pipecat/processors/filters/function_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -10,11 +10,13 @@ This module provides a processor that filters frames based on a custom function,
allowing for flexible frame filtering logic in processing pipelines.
"""
from typing import Awaitable, Callable
from typing import Awaitable, Callable, Optional
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
FilterType = Callable[[Frame], Awaitable[bool]]
class FunctionFilter(FrameProcessor):
"""A frame processor that filters frames using a custom function.
@@ -26,9 +28,10 @@ class FunctionFilter(FrameProcessor):
def __init__(
self,
filter: Callable[[Frame], Awaitable[bool]],
direction: FrameDirection = FrameDirection.DOWNSTREAM,
filter: FilterType,
direction: Optional[FrameDirection] = FrameDirection.DOWNSTREAM,
filter_system_frames: bool = False,
**kwargs,
):
"""Initialize the function filter.
@@ -36,10 +39,13 @@ class FunctionFilter(FrameProcessor):
filter: An async function that takes a Frame and returns True if the
frame should pass through, False otherwise.
direction: The direction to apply filtering. Only frames moving in
this direction will be filtered. Defaults to DOWNSTREAM.
this direction will be filtered; frames in the other direction
pass through unfiltered. If None, frames in both directions
are filtered. Defaults to DOWNSTREAM.
filter_system_frames: Whether to filter system frames. Defaults to False.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__()
super().__init__(**kwargs)
self._filter = filter
self._direction = direction
self._filter_system_frames = filter_system_frames
@@ -51,7 +57,7 @@ class FunctionFilter(FrameProcessor):
def _should_passthrough_frame(self, frame, direction):
"""Check if a frame should pass through without filtering."""
# Always passthrough frames in the wrong direction
if direction != self._direction:
if self._direction and direction != self._direction:
return True
# Always passthrough lifecycle frames

2
src/pipecat/processors/filters/identity_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

2
src/pipecat/processors/filters/null_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

38
src/pipecat/processors/filters/stt_mute_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -21,8 +21,9 @@ from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
Frame,
FunctionCallInProgressFrame,
FunctionCallCancelFrame,
FunctionCallResultFrame,
FunctionCallsStartedFrame,
InputAudioRawFrame,
InterimTranscriptionFrame,
InterruptionFrame,
@@ -50,6 +51,10 @@ class STTMuteStrategy(Enum):
FUNCTION_CALL: Mute STT during function calls to prevent interruptions.
ALWAYS: Always mute STT when the bot is speaking.
CUSTOM: Use a custom callback to determine muting logic dynamically.
.. deprecated:: 0.0.99
`STTMuteStrategy` is deprecated and will be removed in a future version.
Use `LLMUserAggregator`'s new `user_mute_strategies` instead.
"""
FIRST_SPEECH = "first_speech"
@@ -76,6 +81,10 @@ class STTMuteConfig:
Note:
MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together
as they handle the first bot speech differently.
.. deprecated:: 0.0.99
`STTMuteConfig` is deprecated and will be removed in a future version.
Use `LLMUserAggregator`'s new `user_mute_strategies` instead.
"""
strategies: set[STTMuteStrategy]
@@ -103,6 +112,10 @@ class STTMuteFilter(FrameProcessor):
feature. When STT is muted, interruptions are automatically disabled by
suppressing VAD-related frames. This prevents unwanted speech detection
during bot speech, function calls, or other specified conditions.
.. deprecated:: 0.0.99
`STTMuteFilter` is deprecated and will be removed in a future version.
Use `LLMUserAggregator`'s new `user_mute_strategies` instead.
"""
def __init__(self, *, config: STTMuteConfig, **kwargs):
@@ -116,9 +129,19 @@ class STTMuteFilter(FrameProcessor):
self._config = config
self._first_speech_handled = False
self._bot_is_speaking = False
self._function_call_in_progress = False
self._function_call_in_progress = set()
self._is_muted = False
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"`STTMuteFilter` is deprecated and will be removed in a future version. "
"Use `LLMUserAggregator`'s new `user_mute_strategies` instead.",
DeprecationWarning,
)
async def _handle_mute_state(self, should_mute: bool):
"""Handle STT muting and interruption control state changes."""
if should_mute != self._is_muted:
@@ -176,11 +199,12 @@ class STTMuteFilter(FrameProcessor):
# Process frames to determine mute state
if isinstance(frame, StartFrame):
should_mute = await self._should_mute()
elif isinstance(frame, FunctionCallInProgressFrame):
self._function_call_in_progress = True
elif isinstance(frame, FunctionCallsStartedFrame):
for f in frame.function_calls:
self._function_call_in_progress.add(f.tool_call_id)
should_mute = await self._should_mute()
elif isinstance(frame, FunctionCallResultFrame):
self._function_call_in_progress = False
elif isinstance(frame, (FunctionCallCancelFrame, FunctionCallResultFrame)):
self._function_call_in_progress.remove(frame.tool_call_id)
should_mute = await self._should_mute()
elif isinstance(frame, BotStartedSpeakingFrame):
self._bot_is_speaking = True

26
src/pipecat/processors/filters/wake_check_filter.py
View File

@@ -1,11 +1,14 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Wake phrase detection filter for Pipecat transcription processing.
.. deprecated:: 0.0.106
Use :class:`~pipecat.turns.user_start.WakePhraseUserTurnStartStrategy` instead.
This module provides a frame processor that filters transcription frames,
only allowing them through after wake phrases have been detected. Includes
keepalive functionality to maintain conversation flow after wake detection.
@@ -13,18 +16,24 @@ keepalive functionality to maintain conversation flow after wake detection.
import re
import time
import warnings
from enum import Enum
from typing import List
from loguru import logger
from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
from pipecat.frames.frames import Frame, TranscriptionFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class WakeCheckFilter(FrameProcessor):
"""Frame processor that filters transcription frames based on wake phrase detection.
.. deprecated:: 0.0.106
Use :class:`~pipecat.turns.user_start.WakePhraseUserTurnStartStrategy` instead,
which integrates with the user turn strategy system and supports configurable
timeouts and single-activation mode.
This filter monitors transcription frames for configured wake phrases and only
passes frames through after a wake phrase has been detected. Maintains a
keepalive timeout to allow continued conversation after wake detection.
@@ -65,12 +74,21 @@ class WakeCheckFilter(FrameProcessor):
def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3):
"""Initialize the wake phrase filter.
.. deprecated:: 0.0.106
Use :class:`~pipecat.turns.user_start.WakePhraseUserTurnStartStrategy` instead.
Args:
wake_phrases: List of wake phrases to detect in transcriptions.
keepalive_timeout: Duration in seconds to keep passing frames after
wake detection. Defaults to 3 seconds.
"""
super().__init__()
warnings.warn(
"WakeCheckFilter is deprecated since v0.0.106. "
"Use WakePhraseUserTurnStartStrategy instead.",
DeprecationWarning,
stacklevel=2,
)
self._participant_states = {}
self._keepalive_timeout = keepalive_timeout
self._wake_patterns = []
@@ -126,6 +144,4 @@ class WakeCheckFilter(FrameProcessor):
else:
await self.push_frame(frame, direction)
except Exception as e:
error_msg = f"Error in wake word filter: {e}"
logger.exception(error_msg)
await self.push_error(ErrorFrame(error_msg))
await self.push_error(error_msg=f"Error in wake word filter: {e}", exception=e)

4
src/pipecat/processors/filters/wake_notifier_filter.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2024–2025, Daily
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
@@ -10,7 +10,7 @@ from typing import Awaitable, Callable, Tuple, Type
from pipecat.frames.frames import Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.sync.base_notifier import BaseNotifier
from pipecat.utils.sync.base_notifier import BaseNotifier
class WakeNotifierFilter(FrameProcessor):

Some files were not shown because too many files have changed in this diff Show More