From d1d74c571c9edf20c086d5e1fca4c9ef1ebb3e8b Mon Sep 17 00:00:00 2001 From: Nimo Beeren Date: Fri, 17 Oct 2025 10:38:06 +0200 Subject: [PATCH 01/93] add ellipsis character to sentence ending punctuation list --- src/pipecat/utils/string.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index c9cb05142..11964d23f 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -47,6 +47,7 @@ SENTENCE_ENDING_PUNCTUATION: FrozenSet[str] = frozenset( "!", "?", ";", + "…", # East Asian punctuation (Chinese (Traditional & Simplified), Japanese, Korean) "。", # Ideographic full stop "？", # Full-width question mark From ae5c5ed7f6e9d8555871089357270b482238e9ab Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 25 Sep 2025 09:46:33 -0400 Subject: [PATCH 02/93] Update `AWSNovaSonicLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` --- CHANGELOG.md | 12 + .../20a-persistent-context-openai.py | 4 +- .../20c-persistent-context-anthropic.py | 4 +- .../20d-persistent-context-gemini.py | 5 +- .../20e-persistent-context-aws-nova-sonic.py | 6 +- examples/foundational/40-aws-nova-sonic.py | 9 +- .../services/aws_nova_sonic_adapter.py | 122 +++++- .../processors/aggregators/llm_context.py | 41 +- .../services/aws/nova_sonic/context.py | 367 ------------------ src/pipecat/services/aws/nova_sonic/llm.py | 288 +++++++++----- tests/test_llm_context.py | 208 ++++++++++ 11 files changed, 580 insertions(+), 486 deletions(-) delete mode 100644 src/pipecat/services/aws/nova_sonic/context.py create mode 100644 tests/test_llm_context.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 71e5300c0..feb86e5d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Expanded support for universal `LLMContext` to `AWSNovaSonicLLMService`. + As a reminder, the context-setup pattern when using `LLMContext` is: + + ```python + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + ``` + + (Note that even though `AWSNovaSonicLLMService` now supports the universal + `LLMContext`, it is not meant to be swapped out for another LLM service at + runtime.) + - Added support for `bulbul:v3` model in `SarvamTTSService` and `SarvamHttpTTSService`. - Added `keyterms_prompt` parameter to `AssemblyAIConnectionParams`. diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 93c1fa438..a3f9f005a 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -67,11 +67,11 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" ) try: with open(filename, "w") as file: - messages = params.context.get_messages() + messages = params.context.get_messages_for_persistent_storage() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 411a976b8..7a0953290 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -68,12 +68,12 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" ) try: with open(filename, "w") as file: # todo: extract 'system' into the first message in the list - messages = params.context.get_messages() + messages = params.context.get_messages_for_persistent_storage() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 8dad8148d..0e52c0308 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -86,12 +86,11 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" ) try: with open(filename, "w") as file: - # todo: extract 'system' into the first message in the list - messages = params.context.get_messages() + messages = params.context.get_messages_for_persistent_storage() # remove the last message (the instruction to save the context) messages.pop() json.dump(messages, file, indent=2) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index 2161aff8f..4ff31dcc0 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -20,6 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -223,13 +225,13 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext( + context = LLMContext( messages=[ {"role": "system", "content": f"{system_instruction}"}, ], tools=tools, ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/40-aws-nova-sonic.py b/examples/foundational/40-aws-nova-sonic.py index d9bd2257d..e5e36e404 100644 --- a/examples/foundational/40-aws-nova-sonic.py +++ b/examples/foundational/40-aws-nova-sonic.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService @@ -119,9 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_current_weather", fetch_weather_from_api) # Set up context and context management. - # AWSNovaSonicService will adapt OpenAI LLM context objects with standard message format to - # what's expected by Nova Sonic. - context = OpenAILLMContext( + context = LLMContext( messages=[ {"role": "system", "content": f"{system_instruction}"}, { @@ -131,7 +130,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ], tools=tools, ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) # Build the pipeline pipeline = Pipeline( diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py index 64319d266..60f12798b 100644 --- a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -6,13 +6,47 @@ """AWS Nova Sonic LLM adapter for Pipecat.""" +import copy import json -from typing import Any, Dict, List, TypedDict +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, List, Optional, TypedDict + +from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage + + +class Role(Enum): + """Roles supported in AWS Nova Sonic conversations. + + Parameters: + SYSTEM: System-level messages (not used in conversation history). + USER: Messages sent by the user. + ASSISTANT: Messages sent by the assistant. + TOOL: Messages sent by tools (not used in conversation history). + """ + + SYSTEM = "SYSTEM" + USER = "USER" + ASSISTANT = "ASSISTANT" + TOOL = "TOOL" + + +@dataclass +class AWSNovaSonicConversationHistoryMessage: + """A single message in AWS Nova Sonic conversation history. + + Parameters: + role: The role of the message sender (USER or ASSISTANT only). + text: The text content of the message. + """ + + role: Role # only USER and ASSISTANT + text: str class AWSNovaSonicLLMInvocationParams(TypedDict): @@ -21,7 +55,9 @@ class AWSNovaSonicLLMInvocationParams(TypedDict): This is a placeholder until support for universal LLMContext machinery is added for AWS Nova Sonic. """ - pass + system_instruction: Optional[str] + messages: List[AWSNovaSonicConversationHistoryMessage] + tools: List[Dict[str, Any]] class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): @@ -34,7 +70,7 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): @property def id_for_llm_specific_messages(self) -> str: """Get the identifier used in LLMSpecificMessage instances for AWS Nova Sonic.""" - raise NotImplementedError("Universal LLMContext is not yet supported for AWS Nova Sonic.") + return "aws-nova-sonic" def get_llm_invocation_params(self, context: LLMContext) -> AWSNovaSonicLLMInvocationParams: """Get AWS Nova Sonic-specific LLM invocation parameters from a universal LLM context. @@ -47,7 +83,13 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): Returns: Dictionary of parameters for invoking AWS Nova Sonic's LLM API. """ - raise NotImplementedError("Universal LLMContext is not yet supported for AWS Nova Sonic.") + messages = self._from_universal_context_messages(self.get_messages(context)) + return { + "system_instruction": messages.system_instruction, + "messages": messages.messages, + # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) + "tools": self.from_standard_tools(context.tools) or [], + } def get_messages_for_logging(self, context) -> List[Dict[str, Any]]: """Get messages from a universal LLM context in a format ready for logging about AWS Nova Sonic. @@ -62,7 +104,75 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): Returns: List of messages in a format ready for logging about AWS Nova Sonic. """ - raise NotImplementedError("Universal LLMContext is not yet supported for AWS Nova Sonic.") + return self._from_universal_context_messages(self.get_messages(context)).messages + + @dataclass + class ConvertedMessages: + """Container for Google-formatted messages converted from universal context.""" + + messages: List[AWSNovaSonicConversationHistoryMessage] + system_instruction: Optional[str] = None + + def _from_universal_context_messages( + self, universal_context_messages: List[LLMContextMessage] + ) -> ConvertedMessages: + system_instruction = None + messages = [] + + # Bail if there are no messages + if not universal_context_messages: + return self.ConvertedMessages() + + universal_context_messages = copy.deepcopy(universal_context_messages) + + # If we have a "system" message as our first message, let's pull that out into "instruction" + if universal_context_messages[0].get("role") == "system": + system = universal_context_messages.pop(0) + content = system.get("content") + if isinstance(content, str): + system_instruction = content + elif isinstance(content, list): + system_instruction = content[0].get("text") + if system_instruction: + self._system_instruction = system_instruction + + # Process remaining messages to fill out conversation history. + # Nova Sonic supports "user" and "assistant" messages in history. + for universal_context_message in universal_context_messages: + message = self._from_universal_context_message(universal_context_message) + if message: + messages.append(message) + + return self.ConvertedMessages(messages=messages, system_instruction=system_instruction) + + def _from_universal_context_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + """Convert standard message format to Nova Sonic format. + + Args: + message: Standard message dictionary to convert. + + Returns: + Nova Sonic conversation history message, or None if not convertible. + """ + role = message.get("role") + if message.get("role") == "user" or message.get("role") == "assistant": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + # There won't be content if this is an assistant tool call entry. + # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation + # history + if content: + return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova + # Sonic conversation history @staticmethod def _to_aws_nova_sonic_function_format(function: FunctionSchema) -> Dict[str, Any]: diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 8b677cf02..e98bc743e 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -15,9 +15,10 @@ service-specific adapter. """ import base64 +import copy import io from dataclasses import dataclass -from typing import Any, List, Optional, TypeAlias, Union +from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union from loguru import logger from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN @@ -31,6 +32,9 @@ from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.frames.frames import AudioRawFrame +if TYPE_CHECKING: + from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext + # "Re-export" types from OpenAI that we're using as universal context types. # NOTE: if universal message types need to someday diverge from OpenAI's, we # should consider managing our own definitions. But we should do so carefully, @@ -65,6 +69,26 @@ class LLMContext: and content formatting. """ + @staticmethod + def from_openai_context(openai_context: "OpenAILLMContext") -> "LLMContext": + """Create a universal LLM context from an OpenAI-specific context. + + NOTE: this should only be used internally, for facilitating migration + from OpenAILLMContext to LLMContext. New user code should use + LLMContext directly. + + Args: + openai_context: The OpenAI LLM context to convert. + + Returns: + New LLMContext instance with converted messages and settings. + """ + return LLMContext( + messages=openai_context.get_messages(), + tools=openai_context.tools, + tool_choice=openai_context.tool_choice, + ) + def __init__( self, messages: Optional[List[LLMContextMessage]] = None, @@ -107,6 +131,21 @@ class LLMContext: ) return filtered_messages + def get_messages_for_persistent_storage( + self, system_instruction: Optional[str] = None + ) -> List[LLMContextMessage]: + """Get messages formatted for persistent storage. + + Args: + system_instruction: Optional system instruction to ensure is + included as the first message in the returned list, if not + already present. + """ + messages = copy.deepcopy(self.get_messages()) + if system_instruction and (not messages or messages[0].get("role") != "system"): + messages.insert(0, {"role": "system", "content": system_instruction}) + return messages + @property def tools(self) -> ToolsSchema | NotGiven: """Get the tools list. diff --git a/src/pipecat/services/aws/nova_sonic/context.py b/src/pipecat/services/aws/nova_sonic/context.py deleted file mode 100644 index 86aa0f0b5..000000000 --- a/src/pipecat/services/aws/nova_sonic/context.py +++ /dev/null @@ -1,367 +0,0 @@ -# -# Copyright (c) 2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -"""Context management for AWS Nova Sonic LLM service. - -This module provides specialized context aggregators and message handling for AWS Nova Sonic, -including conversation history management and role-specific message processing. -""" - -import copy -from dataclasses import dataclass, field -from enum import Enum - -from loguru import logger - -from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, - DataFrame, - Frame, - FunctionCallResultFrame, - InterruptionFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMMessagesAppendFrame, - LLMMessagesUpdateFrame, - LLMSetToolChoiceFrame, - LLMSetToolsFrame, - TextFrame, - UserImageRawFrame, -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) - - -class Role(Enum): - """Roles supported in AWS Nova Sonic conversations. - - Parameters: - SYSTEM: System-level messages (not used in conversation history). - USER: Messages sent by the user. - ASSISTANT: Messages sent by the assistant. - TOOL: Messages sent by tools (not used in conversation history). - """ - - SYSTEM = "SYSTEM" - USER = "USER" - ASSISTANT = "ASSISTANT" - TOOL = "TOOL" - - -@dataclass -class AWSNovaSonicConversationHistoryMessage: - """A single message in AWS Nova Sonic conversation history. - - Parameters: - role: The role of the message sender (USER or ASSISTANT only). - text: The text content of the message. - """ - - role: Role # only USER and ASSISTANT - text: str - - -@dataclass -class AWSNovaSonicConversationHistory: - """Complete conversation history for AWS Nova Sonic initialization. - - Parameters: - system_instruction: System-level instruction for the conversation. - messages: List of conversation messages between user and assistant. - """ - - system_instruction: str = None - messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) - - -class AWSNovaSonicLLMContext(OpenAILLMContext): - """Specialized LLM context for AWS Nova Sonic service. - - Extends OpenAI context with Nova Sonic-specific message handling, - conversation history management, and text buffering capabilities. - """ - - def __init__(self, messages=None, tools=None, **kwargs): - """Initialize AWS Nova Sonic LLM context. - - Args: - messages: Initial messages for the context. - tools: Available tools for the context. - **kwargs: Additional arguments passed to parent class. - """ - super().__init__(messages=messages, tools=tools, **kwargs) - self.__setup_local() - - def __setup_local(self, system_instruction: str = ""): - self._assistant_text = "" - self._user_text = "" - self._system_instruction = system_instruction - - @staticmethod - def upgrade_to_nova_sonic( - obj: OpenAILLMContext, system_instruction: str - ) -> "AWSNovaSonicLLMContext": - """Upgrade an OpenAI context to AWS Nova Sonic context. - - Args: - obj: The OpenAI context to upgrade. - system_instruction: System instruction for the context. - - Returns: - The upgraded AWS Nova Sonic context. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): - obj.__class__ = AWSNovaSonicLLMContext - obj.__setup_local(system_instruction) - return obj - - # NOTE: this method has the side-effect of updating _system_instruction from messages - def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: - """Get conversation history for initializing AWS Nova Sonic session. - - Processes stored messages and extracts system instruction and conversation - history in the format expected by AWS Nova Sonic. - - Returns: - Formatted conversation history with system instruction and messages. - """ - history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) - - # Bail if there are no messages - if not self.messages: - return history - - messages = copy.deepcopy(self.messages) - - # If we have a "system" message as our first message, let's pull that out into "instruction" - if messages[0].get("role") == "system": - system = messages.pop(0) - content = system.get("content") - if isinstance(content, str): - history.system_instruction = content - elif isinstance(content, list): - history.system_instruction = content[0].get("text") - if history.system_instruction: - self._system_instruction = history.system_instruction - - # Process remaining messages to fill out conversation history. - # Nova Sonic supports "user" and "assistant" messages in history. - for message in messages: - history_message = self.from_standard_message(message) - if history_message: - history.messages.append(history_message) - - return history - - def get_messages_for_persistent_storage(self): - """Get messages formatted for persistent storage. - - Returns: - List of messages including system instruction if present. - """ - messages = super().get_messages_for_persistent_storage() - # If we have a system instruction and messages doesn't already contain it, add it - if self._system_instruction and not (messages and messages[0].get("role") == "system"): - messages.insert(0, {"role": "system", "content": self._system_instruction}) - return messages - - def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: - """Convert standard message format to Nova Sonic format. - - Args: - message: Standard message dictionary to convert. - - Returns: - Nova Sonic conversation history message, or None if not convertible. - """ - role = message.get("role") - if message.get("role") == "user" or message.get("role") == "assistant": - content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): - if c.get("type") == "text": - content += " " + c.get("text") - else: - logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" - ) - # There won't be content if this is an assistant tool call entry. - # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation - # history - if content: - return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) - # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova - # Sonic conversation history - - def buffer_user_text(self, text): - """Buffer user text for later flushing to context. - - Args: - text: User text to buffer. - """ - self._user_text += f" {text}" if self._user_text else text - # logger.debug(f"User text buffered: {self._user_text}") - - def flush_aggregated_user_text(self) -> str: - """Flush buffered user text to context as a complete message. - - Returns: - The flushed user text, or empty string if no text was buffered. - """ - if not self._user_text: - return "" - user_text = self._user_text - message = { - "role": "user", - "content": [{"type": "text", "text": user_text}], - } - self._user_text = "" - self.add_message(message) - # logger.debug(f"Context updated (user): {self.get_messages_for_logging()}") - return user_text - - def buffer_assistant_text(self, text): - """Buffer assistant text for later flushing to context. - - Args: - text: Assistant text to buffer. - """ - self._assistant_text += text - # logger.debug(f"Assistant text buffered: {self._assistant_text}") - - def flush_aggregated_assistant_text(self): - """Flush buffered assistant text to context as a complete message.""" - if not self._assistant_text: - return - message = { - "role": "assistant", - "content": [{"type": "text", "text": self._assistant_text}], - } - self._assistant_text = "" - self.add_message(message) - # logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}") - - -@dataclass -class AWSNovaSonicMessagesUpdateFrame(DataFrame): - """Frame containing updated AWS Nova Sonic context. - - Parameters: - context: The updated AWS Nova Sonic LLM context. - """ - - context: AWSNovaSonicLLMContext - - -class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): - """Context aggregator for user messages in AWS Nova Sonic conversations. - - Extends the OpenAI user context aggregator to emit Nova Sonic-specific - context update frames. - """ - - async def process_frame( - self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM - ): - """Process frames and emit Nova Sonic-specific context updates. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - await super().process_frame(frame, direction) - - # Parent does not push LLMMessagesUpdateFrame - if isinstance(frame, LLMMessagesUpdateFrame): - await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) - - -class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Context aggregator for assistant messages in AWS Nova Sonic conversations. - - Provides specialized handling for assistant responses and function calls - in AWS Nova Sonic context, with custom frame processing logic. - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames with Nova Sonic-specific logic. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - # HACK: For now, disable the context aggregator by making it just pass through all frames - # that the parent handles (except the function call stuff, which we still need). - # For an explanation of this hack, see - # AWSNovaSonicLLMService._report_assistant_response_text_added. - if isinstance( - frame, - ( - InterruptionFrame, - LLMFullResponseStartFrame, - LLMFullResponseEndFrame, - TextFrame, - LLMMessagesAppendFrame, - LLMMessagesUpdateFrame, - LLMSetToolsFrame, - LLMSetToolChoiceFrame, - UserImageRawFrame, - BotStoppedSpeakingFrame, - ), - ): - await self.push_frame(frame, direction) - else: - await super().process_frame(frame, direction) - - async def handle_function_call_result(self, frame: FunctionCallResultFrame): - """Handle function call results for AWS Nova Sonic. - - Args: - frame: The function call result frame to handle. - """ - await super().handle_function_call_result(frame) - - # The standard function callback code path pushes the FunctionCallResultFrame from the LLM - # itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side - # context. Let's push a special frame to do that. - await self.push_frame( - AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) - - -@dataclass -class AWSNovaSonicContextAggregatorPair: - """Pair of user and assistant context aggregators for AWS Nova Sonic. - - Parameters: - _user: The user context aggregator. - _assistant: The assistant context aggregator. - """ - - _user: AWSNovaSonicUserContextAggregator - _assistant: AWSNovaSonicAssistantContextAggregator - - def user(self) -> AWSNovaSonicUserContextAggregator: - """Get the user context aggregator. - - Returns: - The user context aggregator instance. - """ - return self._user - - def assistant(self) -> AWSNovaSonicAssistantContextAggregator: - """Get the assistant context aggregator. - - Returns: - The assistant context aggregator instance. - """ - return self._assistant diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 2801de688..c1f941db4 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -25,7 +25,7 @@ from loguru import logger from pydantic import BaseModel, Field from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter +from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter, Role from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, @@ -33,35 +33,30 @@ from pipecat.frames.frames import ( Frame, FunctionCallFromLLM, InputAudioRawFrame, - InterimTranscriptionFrame, + InterruptionFrame, LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, - LLMTextFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, ) +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, ) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.aws.nova_sonic.context import ( - AWSNovaSonicAssistantContextAggregator, - AWSNovaSonicContextAggregatorPair, - AWSNovaSonicLLMContext, - AWSNovaSonicUserContextAggregator, - Role, -) -from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame from pipecat.services.llm_service import LLMService from pipecat.utils.time import time_now_iso8601 @@ -70,7 +65,7 @@ try: BedrockRuntimeClient, InvokeModelWithBidirectionalStreamOperationInput, ) - from aws_sdk_bedrock_runtime.config import Config + from aws_sdk_bedrock_runtime.config import Config, HTTPAuthSchemeResolver, SigV4AuthScheme from aws_sdk_bedrock_runtime.models import ( BidirectionalInputPayloadPart, InvokeModelWithBidirectionalStreamInput, @@ -78,8 +73,8 @@ try: InvokeModelWithBidirectionalStreamOperationOutput, InvokeModelWithBidirectionalStreamOutput, ) - from smithy_aws_core.auth.sigv4 import SigV4AuthScheme - from smithy_aws_core.identity.static import StaticCredentialsResolver + from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver + from smithy_aws_core.identity import AWSCredentialsIdentity from smithy_core.aio.eventstream import DuplexEventStream except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -231,7 +226,7 @@ class AWSNovaSonicLLMService(LLMService): self._system_instruction = system_instruction self._tools = tools self._send_transcription_frames = send_transcription_frames - self._context: Optional[AWSNovaSonicLLMContext] = None + self._context: Optional[LLMContext] = None self._stream: Optional[ DuplexEventStream[ InvokeModelWithBidirectionalStreamInput, @@ -244,14 +239,19 @@ class AWSNovaSonicLLMService(LLMService): self._input_audio_content_name: Optional[str] = None self._content_being_received: Optional[CurrentContent] = None self._assistant_is_responding = False + self._needs_re_push_assistant_text = False self._ready_to_send_context = False self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False + self._waiting_for_trigger_transcription = False self._disconnecting = False self._connected_time: Optional[float] = None self._wants_connection = False + self._user_text_buffer = "" + self._assistant_text_buffer = "" + self._completed_tool_calls = set() - file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav") + file_path = files("pipecat.services.aws_nova_sonic").joinpath("ready.wav") with wave.open(file_path.open("rb"), "rb") as wav_file: self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) @@ -302,12 +302,12 @@ class AWSNovaSonicLLMService(LLMService): logger.debug("Resetting conversation") await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False) - # Carry over previous context through disconnect + # Grab context to carry through disconnect/reconnect context = self._context - await self._disconnect() - self._context = context + await self._disconnect() await self._start_connecting() + await self._handle_context(context) # # frame processing @@ -322,28 +322,35 @@ class AWSNovaSonicLLMService(LLMService): """ await super().process_frame(frame, direction) - if isinstance(frame, OpenAILLMContextFrame): - await self._handle_context(frame.context) - elif isinstance(frame, LLMContextFrame): - raise NotImplementedError( - "Universal LLMContext is not yet supported for AWS Nova Sonic." + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + context = ( + frame.context + if isinstance(frame, LLMContextFrame) + else LLMContext.from_openai_context(frame.context) ) + await self._handle_context(context) elif isinstance(frame, InputAudioRawFrame): await self._handle_input_audio_frame(frame) elif isinstance(frame, BotStoppedSpeakingFrame): await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=True) - elif isinstance(frame, AWSNovaSonicFunctionCallResultFrame): - await self._handle_function_call_result(frame) + elif isinstance(frame, InterruptionFrame): + await self._handle_interruption_frame() await self.push_frame(frame, direction) - async def _handle_context(self, context: OpenAILLMContext): + async def _handle_context(self, context: LLMContext): + if self._disconnecting: + return + if not self._context: - # We got our initial context - try to finish connecting - self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic( - context, self._system_instruction - ) + # We got our initial context + # Try to finish connecting + self._context = context await self._finish_connecting_if_context_available() + else: + # We got an updated context + # Send results for any newly-completed function calls + await self._process_completed_function_calls(send_new_results=True) async def _handle_input_audio_frame(self, frame: InputAudioRawFrame): # Wait until we're done sending the assistant response trigger audio before sending audio @@ -393,9 +400,9 @@ class AWSNovaSonicLLMService(LLMService): else: await finalize_assistant_response() - async def _handle_function_call_result(self, frame: AWSNovaSonicFunctionCallResultFrame): - result = frame.result_frame - await self._send_tool_result(tool_call_id=result.tool_call_id, result=result.result) + async def _handle_interruption_frame(self): + if self._assistant_is_responding: + self._needs_re_push_assistant_text = True # # LLM communication: lifecycle @@ -429,7 +436,18 @@ class AWSNovaSonicLLMService(LLMService): await self._finish_connecting_if_context_available() except Exception as e: logger.error(f"{self} initialization error: {e}") - await self._disconnect() + self._disconnect() + + async def _process_completed_function_calls(self, send_new_results: bool): + # Check for set of completed function calls in the context + for message in self._context.get_messages(): + if message.get("role") and message.get("content") != "IN_PROGRESS": + tool_call_id = message.get("tool_call_id") + if tool_call_id and tool_call_id not in self._completed_tool_calls: + # Found a newly-completed function call - send the result to the service + if send_new_results: + await self._send_tool_result(tool_call_id, message.get("content")) + self._completed_tool_calls.add(tool_call_id) async def _finish_connecting_if_context_available(self): # We can only finish connecting once we've gotten our initial context and we're ready to @@ -439,30 +457,38 @@ class AWSNovaSonicLLMService(LLMService): logger.info("Finishing connecting (setting up session)...") + # Initialize our bookkeeping of already-completed tool calls in the + # context + await self._process_completed_function_calls(send_new_results=False) + # Read context - history = self._context.get_messages_for_initializing_history() + adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter() + llm_connection_params = adapter.get_llm_invocation_params(self._context) # Send prompt start event, specifying tools. # Tools from context take priority over self._tools. tools = ( - self._context.tools - if self._context.tools - else self.get_llm_adapter().from_standard_tools(self._tools) + llm_connection_params["tools"] + if llm_connection_params["tools"] + else adapter.from_standard_tools(self._tools) ) logger.debug(f"Using tools: {tools}") await self._send_prompt_start_event(tools) # Send system instruction. # Instruction from context takes priority over self._system_instruction. - # (NOTE: this prioritizing occurred automatically behind the scenes: the context was - # initialized with self._system_instruction and then updated itself from its messages when - # get_messages_for_initializing_history() was called). - logger.debug(f"Using system instruction: {history.system_instruction}") - if history.system_instruction: - await self._send_text_event(text=history.system_instruction, role=Role.SYSTEM) + system_instruction = ( + llm_connection_params["system_instruction"] + if llm_connection_params["system_instruction"] + else self._system_instruction + ) + logger.debug(f"Using system instruction: {system_instruction}") + if system_instruction: + await self._send_text_event(text=system_instruction, role=Role.SYSTEM) # Send conversation history - for message in history.messages: + for message in llm_connection_params["messages"]: + # logger.debug(f"Seeding conversation history with message: {message}") await self._send_text_event(text=message.text, role=message.role) # Start audio input @@ -492,9 +518,12 @@ class AWSNovaSonicLLMService(LLMService): await self._send_session_end_events() self._client = None + # Clean up context + self._context = None + # Clean up stream if self._stream: - await self._stream.input_stream.close() + await self._stream.close() self._stream = None # NOTE: see explanation of HACK, below @@ -510,15 +539,23 @@ class AWSNovaSonicLLMService(LLMService): self._receive_task = None # Reset remaining connection-specific state + # Should be all private state except: + # - _wants_connection + # - _assistant_response_trigger_audio self._prompt_name = None self._input_audio_content_name = None self._content_being_received = None self._assistant_is_responding = False + self._needs_re_push_assistant_text = False self._ready_to_send_context = False self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False + self._waiting_for_trigger_transcription = False self._disconnecting = False self._connected_time = None + self._user_text_buffer = "" + self._assistant_text_buffer = "" + self._completed_tool_calls = set() logger.info("Finished disconnecting") except Exception as e: @@ -528,11 +565,15 @@ class AWSNovaSonicLLMService(LLMService): config = Config( endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", region=self._region, - aws_access_key_id=self._access_key_id, - aws_secret_access_key=self._secret_access_key, - aws_session_token=self._session_token, - aws_credentials_identity_resolver=StaticCredentialsResolver(), - auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="bedrock")}, + aws_credentials_identity_resolver=StaticCredentialsResolver( + credentials=AWSCredentialsIdentity( + access_key_id=self._access_key_id, + secret_access_key=self._secret_access_key, + session_token=self._session_token, + ) + ), + http_auth_scheme_resolver=HTTPAuthSchemeResolver(), + http_auth_schemes={"aws.auth#sigv4": SigV4AuthScheme()}, ) return BedrockRuntimeClient(config=config) @@ -826,6 +867,10 @@ class AWSNovaSonicLLMService(LLMService): # Handle the LLM completion ending await self._handle_completion_end_event(event_json) except Exception as e: + if self._disconnecting: + # Errors are kind of expected while disconnecting, so just + # ignore them and do nothing + return logger.error(f"{self} error processing responses: {e}") if self._wants_connection: await self.reset_conversation() @@ -956,7 +1001,7 @@ class AWSNovaSonicLLMService(LLMService): async def _report_assistant_response_started(self): logger.debug("Assistant response started") - # Report that the assistant has started their response. + # Report the start of the assistant response. await self.push_frame(LLMFullResponseStartFrame()) # Report that equivalent of TTS (this is a speech-to-speech model) started @@ -968,23 +1013,16 @@ class AWSNovaSonicLLMService(LLMService): logger.debug(f"Assistant response text added: {text}") - # Report some text added to the ongoing assistant response - await self.push_frame(LLMTextFrame(text)) - - # Report some text added to the *equivalent* of TTS (this is a speech-to-speech model) + # Report the text of the assistant response. await self.push_frame(TTSTextFrame(text)) - # TODO: this is a (hopefully temporary) HACK. Here we directly manipulate the context rather - # than relying on the frames pushed to the assistant context aggregator. The pattern of - # receiving full-sentence text after the assistant has spoken does not easily fit with the - # Pipecat expectation of chunks of text streaming in while the assistant is speaking. - # Interruption handling was especially challenging. Rather than spend days trying to fit a - # square peg in a round hole, I decided on this hack for the time being. We can most cleanly - # abandon this hack if/when AWS Nova Sonic implements streaming smaller text chunks - # interspersed with audio. Note that when we move away from this hack, we need to make sure - # that on an interruption we avoid sending LLMFullResponseEndFrame, which gets the - # LLMAssistantContextAggregator into a bad state. - self._context.buffer_assistant_text(text) + # HACK: here we're also buffering the assistant text ourselves as a + # backup rather than relying solely on the assistant context aggregator + # to do it, because the text arrives from Nova Sonic only after all the + # assistant audio frames have been pushed, meaning that if an + # interruption frame were to arrive we would lose all of it (the text + # frames sitting in the queue would be wiped). + self._assistant_text_buffer += text async def _report_assistant_response_ended(self): if not self._context: # should never happen @@ -992,14 +1030,25 @@ class AWSNovaSonicLLMService(LLMService): logger.debug("Assistant response ended") - # Report that the assistant has finished their response. + # If an interruption frame arrived while the assistant was responding + # we probably lost all of the assistant text (see HACK, above), so + # re-push it downstream to the aggregator now. + if self._needs_re_push_assistant_text: + # We also need to re-push the LLMFullResponseStartFrame since the + # TTSTextFrame would be ignored otherwise (the interruption frame + # would have cleared the assistant aggregator state). + await self.push_frame(LLMFullResponseStartFrame()) + await self.push_frame(TTSTextFrame(self._assistant_text_buffer)) + self._needs_re_push_assistant_text = False + + # Report the end of the assistant response. await self.push_frame(LLMFullResponseEndFrame()) # Report that equivalent of TTS (this is a speech-to-speech model) stopped. await self.push_frame(TTSStoppedFrame()) - # For an explanation of this hack, see _report_assistant_response_text_added. - self._context.flush_aggregated_assistant_text() + # Clear out the buffered assistant text + self._assistant_text_buffer = "" # # user transcription reporting @@ -1016,33 +1065,71 @@ class AWSNovaSonicLLMService(LLMService): logger.debug(f"User transcription text added: {text}") - # Manually add new user transcription text to context. - # We can't rely on the user context aggregator to do this since it's upstream from the LLM. - self._context.buffer_user_text(text) - - # Report that some new user transcription text is available. - if self._send_transcription_frames: - await self.push_frame( - InterimTranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) - ) + # HACK: here we're buffering the user text ourselves rather than + # relying on the upstream user context aggregator to do it, because the + # text arrives in fairly large chunks spaced fairly far apart in time. + # That means the user text would be split between different messages in + # context. Even if we sent placeholder InterimTranscriptionFrames in + # between each TranscriptionFrame to tell the aggregator to hold off on + # finalizing the user message, the aggregator would likely get the last + # chunk too late. + self._user_text_buffer += f" {text}" if self._user_text_buffer else text async def _report_user_transcription_ended(self): if not self._context: # should never happen return - # Manually add user transcription to context (if any has been buffered). - # We can't rely on the user context aggregator to do this since it's upstream from the LLM. - transcription = self._context.flush_aggregated_user_text() - - if not transcription: - return - logger.debug(f"User transcription ended") - if self._send_transcription_frames: - await self.push_frame( - TranscriptionFrame(text=transcription, user_id="", timestamp=time_now_iso8601()) + # Report to the upstream user context aggregator that some new user + # transcription text is available. + + # HACK: Check if this transcription was triggered by our own + # assistant response trigger. If so, we need to wrap it with + # UserStarted/StoppedSpeakingFrames; otherwise the user aggregator + # would fire an EmulatedUserStartedSpeakingFrame, which would + # trigger an interruption, which would prevent us from writing the + # assistant response to context. + # + # Sending an EmulateUserStartedSpeakingFrame ourselves doesn't + # work: it just causes the interruption we're trying to avoid. + # + # Setting enable_emulated_vad_interruptions also doesn't work: at + # the time the user aggregator receives the TranscriptionFrame, it + # doesn't yet know the assistant has started responding, so it + # doesn't know that emulating the user starting to speak would + # cause an interruption. + should_wrap_in_user_started_stopped_speaking_frames = ( + self._waiting_for_trigger_transcription + and self._user_text_buffer.strip().lower() == "ready" + ) + + # Start wrapping the upstream transcription in UserStarted/StoppedSpeakingFrames if needed + if should_wrap_in_user_started_stopped_speaking_frames: + logger.debug( + "Wrapping assistant response trigger transcription with upstream UserStarted/StoppedSpeakingFrames" ) + await self.push_frame(UserStartedSpeakingFrame(), direction=FrameDirection.UPSTREAM) + + # Send the transcription upstream for the user context aggregator + frame = TranscriptionFrame( + text=self._user_text_buffer, user_id="", timestamp=time_now_iso8601() + ) + await self.push_frame(frame, direction=FrameDirection.UPSTREAM) + + # Finish wrapping the upstream transcription in UserStarted/StoppedSpeakingFrames if needed + if should_wrap_in_user_started_stopped_speaking_frames: + await self.push_frame(UserStoppedSpeakingFrame(), direction=FrameDirection.UPSTREAM) + + # Also send the transcription downstream if requested + if self._send_transcription_frames: + await self.push_frame(frame, direction=FrameDirection.DOWNSTREAM) + + # Clear out the buffered user text + self._user_text_buffer = "" + + # We're no longer waiting for a trigger transcription + self._waiting_for_trigger_transcription = False # # context @@ -1054,23 +1141,26 @@ class AWSNovaSonicLLMService(LLMService): *, user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), - ) -> AWSNovaSonicContextAggregatorPair: + ) -> LLMContextAggregatorPair: """Create context aggregator pair for managing conversation context. + NOTE: this method exists only for backward compatibility. New code + should instead do: + context = LLMContext(...) + context_aggregator = LLMContextAggregatorPair(context) + Args: - context: The OpenAI LLM context to upgrade. + context: The OpenAI LLM context. user_params: Parameters for the user context aggregator. assistant_params: Parameters for the assistant context aggregator. Returns: A pair of user and assistant context aggregators. """ - context.set_llm_adapter(self.get_llm_adapter()) - - user = AWSNovaSonicUserContextAggregator(context=context, params=user_params) - assistant = AWSNovaSonicAssistantContextAggregator(context=context, params=assistant_params) - - return AWSNovaSonicContextAggregatorPair(user, assistant) + context = LLMContext.from_openai_context(context) + return LLMContextAggregatorPair( + context, user_params=user_params, assistant_params=assistant_params + ) # # assistant response trigger (HACK) @@ -1108,6 +1198,8 @@ class AWSNovaSonicLLMService(LLMService): try: logger.debug("Sending assistant response trigger...") + self._waiting_for_trigger_transcription = True + chunk_duration = 0.02 # what we might get from InputAudioRawFrame chunk_size = int( chunk_duration diff --git a/tests/test_llm_context.py b/tests/test_llm_context.py new file mode 100644 index 000000000..3dd84bd2c --- /dev/null +++ b/tests/test_llm_context.py @@ -0,0 +1,208 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage + + +class TestGetMessagesForPersistentStorage(unittest.TestCase): + """Test suite for LLMContext.get_messages_for_persistent_storage method.""" + + def test_no_system_instruction_returns_messages_as_is(self): + """Test that without system instruction, messages are returned unchanged.""" + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"}, + ] + context = LLMContext(messages=messages) + + result = context.get_messages_for_persistent_storage() + + self.assertEqual(result, messages) + self.assertEqual(len(result), 2) + + def test_empty_messages_with_system_instruction_adds_system_message(self): + """Test that system instruction is added when messages list is empty.""" + context = LLMContext() + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[0]["content"], system_instruction) + + def test_non_system_first_message_prepends_system_instruction(self): + """Test that system instruction is prepended when first message is not system.""" + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"}, + ] + context = LLMContext(messages=messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + self.assertEqual(len(result), 3) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[0]["content"], system_instruction) + self.assertEqual(result[1], messages[0]) + self.assertEqual(result[2], messages[1]) + + def test_existing_system_message_not_duplicated(self): + """Test that system instruction is not added when first message is already system.""" + messages = [ + {"role": "system", "content": "Existing system message"}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"}, + ] + context = LLMContext(messages=messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + self.assertEqual(len(result), 3) + self.assertEqual(result, messages) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[0]["content"], "Existing system message") + + def test_empty_system_instruction_does_not_add_message(self): + """Test that empty system instruction does not add a system message.""" + messages = [{"role": "user", "content": "Hello"}] + context = LLMContext(messages=messages) + + result = context.get_messages_for_persistent_storage("") + + self.assertEqual(result, messages) + self.assertEqual(len(result), 1) + + def test_none_system_instruction_does_not_add_message(self): + """Test that None system instruction does not add a system message.""" + messages = [{"role": "user", "content": "Hello"}] + context = LLMContext(messages=messages) + + result = context.get_messages_for_persistent_storage(None) + + self.assertEqual(result, messages) + self.assertEqual(len(result), 1) + + def test_whitespace_only_system_instruction_adds_message(self): + """Test that whitespace-only system instruction still adds a system message.""" + messages = [{"role": "user", "content": "Hello"}] + context = LLMContext(messages=messages) + system_instruction = " " + + result = context.get_messages_for_persistent_storage(system_instruction) + + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[0]["content"], system_instruction) + + def test_with_llm_specific_messages(self): + """Test that method works correctly with LLMSpecificMessage objects.""" + llm_specific = LLMSpecificMessage( + llm="test-llm", message={"role": "user", "content": "Specific"} + ) + messages = [{"role": "user", "content": "Standard message"}, llm_specific] + context = LLMContext(messages=messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + self.assertEqual(len(result), 3) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[0]["content"], system_instruction) + self.assertEqual(result[1], messages[0]) + self.assertEqual(result[2], llm_specific) + + def test_system_message_detection_case_sensitivity(self): + """Test that system message detection is case sensitive.""" + messages = [ + {"role": "System", "content": "Mixed case system"}, # Capital S + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + # Should prepend because "System" != "system" + self.assertEqual(len(result), 3) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[0]["content"], system_instruction) + self.assertEqual(result[1], messages[0]) + + def test_message_without_role_key_does_not_crash(self): + """Test that messages without 'role' key are handled gracefully.""" + messages = [{"content": "Message without role"}, {"role": "user", "content": "Hello"}] + context = LLMContext(messages=messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + # Should prepend system instruction since first message doesn't have role="system" + self.assertEqual(len(result), 3) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[0]["content"], system_instruction) + + def test_original_messages_not_modified(self): + """Test that the original messages list is not modified.""" + original_messages = [{"role": "user", "content": "Hello"}] + context = LLMContext(messages=original_messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + # Original messages should remain unchanged + self.assertEqual(len(original_messages), 1) + self.assertEqual(original_messages[0]["role"], "user") + + # Result should have system message prepended + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[1], original_messages[0]) + + def test_complex_message_structure_preserved(self): + """Test that complex message structures are preserved.""" + complex_message = { + "role": "user", + "content": [ + {"type": "text", "text": "Complex message"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}, + ], + } + messages = [complex_message] + context = LLMContext(messages=messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["role"], "system") + self.assertEqual(result[1], complex_message) + self.assertEqual(result[1]["content"], complex_message["content"]) + + def test_deep_copy_prevents_nested_mutation(self): + """Test that deep copy prevents mutation of nested message content.""" + nested_content = {"nested": {"data": "original"}} + complex_message = {"role": "user", "content": nested_content} + messages = [complex_message] + context = LLMContext(messages=messages) + system_instruction = "You are a helpful assistant." + + result = context.get_messages_for_persistent_storage(system_instruction) + + # Modify the nested content in the result + result[1]["content"]["nested"]["data"] = "modified" + + # Original message should remain unchanged + self.assertEqual(complex_message["content"]["nested"]["data"], "original") + self.assertEqual(context.get_messages()[0]["content"]["nested"]["data"], "original") + + +if __name__ == "__main__": + unittest.main() From 860c39d1b1adbbab2a1168b74a5f5ae20b6ced09 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 29 Sep 2025 10:56:00 -0400 Subject: [PATCH 03/93] Get rid of `LLMContext.get_messages_for_persistent_storage()`. The reason for its `system_instruction` argument was to support usage with LLMs where you might pass the system instruction as a parameter to the `LLMService` rather than specifying it in the context. But as I thought about it more I became unconvinced that the `system_instruction` argument was really beneficial: - If you specified your system instruction in your context in the first place, it'll still be there when you read messages for persistent storage - If you didn't specify your system instruction in the context and instead passed it in as an `LLMService` parameter, you most likely *don't* want it to be in the context when you read messages for persistent storage - ...and if you really really do need to inject it at the start of the context, it's quite easy to do anyway And if we remove the `system_instruction` argument from `get_messages_for_persistent_storage()`, then it's essentially just `get_messages()`. --- .../20a-persistent-context-openai.py | 4 +- .../20c-persistent-context-anthropic.py | 5 +- .../20d-persistent-context-gemini.py | 4 +- .../20e-persistent-context-aws-nova-sonic.py | 2 +- .../processors/aggregators/llm_context.py | 15 -- tests/test_llm_context.py | 208 ------------------ 6 files changed, 7 insertions(+), 231 deletions(-) delete mode 100644 tests/test_llm_context.py diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index a3f9f005a..93c1fa438 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -67,11 +67,11 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 7a0953290..e8822bbc6 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -68,12 +68,11 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: - # todo: extract 'system' into the first message in the list - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 0e52c0308..b32c2fd5b 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -86,11 +86,11 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message (the instruction to save the context) messages.pop() json.dump(messages, file, indent=2) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index 4ff31dcc0..0bc3a0d4e 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -77,7 +77,7 @@ async def save_conversation(params: FunctionCallParams): filename = f"{BASE_FILENAME}{timestamp}.json" try: with open(filename, "w") as file: - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last few messages. in reverse order, they are: # - the in progress save tool call # - the invocation of the save tool call diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index e98bc743e..b60c029cb 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -131,21 +131,6 @@ class LLMContext: ) return filtered_messages - def get_messages_for_persistent_storage( - self, system_instruction: Optional[str] = None - ) -> List[LLMContextMessage]: - """Get messages formatted for persistent storage. - - Args: - system_instruction: Optional system instruction to ensure is - included as the first message in the returned list, if not - already present. - """ - messages = copy.deepcopy(self.get_messages()) - if system_instruction and (not messages or messages[0].get("role") != "system"): - messages.insert(0, {"role": "system", "content": system_instruction}) - return messages - @property def tools(self) -> ToolsSchema | NotGiven: """Get the tools list. diff --git a/tests/test_llm_context.py b/tests/test_llm_context.py deleted file mode 100644 index 3dd84bd2c..000000000 --- a/tests/test_llm_context.py +++ /dev/null @@ -1,208 +0,0 @@ -# -# Copyright (c) 2024-2025 Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import unittest - -from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage - - -class TestGetMessagesForPersistentStorage(unittest.TestCase): - """Test suite for LLMContext.get_messages_for_persistent_storage method.""" - - def test_no_system_instruction_returns_messages_as_is(self): - """Test that without system instruction, messages are returned unchanged.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - context = LLMContext(messages=messages) - - result = context.get_messages_for_persistent_storage() - - self.assertEqual(result, messages) - self.assertEqual(len(result), 2) - - def test_empty_messages_with_system_instruction_adds_system_message(self): - """Test that system instruction is added when messages list is empty.""" - context = LLMContext() - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - self.assertEqual(len(result), 1) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[0]["content"], system_instruction) - - def test_non_system_first_message_prepends_system_instruction(self): - """Test that system instruction is prepended when first message is not system.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - context = LLMContext(messages=messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - self.assertEqual(len(result), 3) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[0]["content"], system_instruction) - self.assertEqual(result[1], messages[0]) - self.assertEqual(result[2], messages[1]) - - def test_existing_system_message_not_duplicated(self): - """Test that system instruction is not added when first message is already system.""" - messages = [ - {"role": "system", "content": "Existing system message"}, - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - context = LLMContext(messages=messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - self.assertEqual(len(result), 3) - self.assertEqual(result, messages) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[0]["content"], "Existing system message") - - def test_empty_system_instruction_does_not_add_message(self): - """Test that empty system instruction does not add a system message.""" - messages = [{"role": "user", "content": "Hello"}] - context = LLMContext(messages=messages) - - result = context.get_messages_for_persistent_storage("") - - self.assertEqual(result, messages) - self.assertEqual(len(result), 1) - - def test_none_system_instruction_does_not_add_message(self): - """Test that None system instruction does not add a system message.""" - messages = [{"role": "user", "content": "Hello"}] - context = LLMContext(messages=messages) - - result = context.get_messages_for_persistent_storage(None) - - self.assertEqual(result, messages) - self.assertEqual(len(result), 1) - - def test_whitespace_only_system_instruction_adds_message(self): - """Test that whitespace-only system instruction still adds a system message.""" - messages = [{"role": "user", "content": "Hello"}] - context = LLMContext(messages=messages) - system_instruction = " " - - result = context.get_messages_for_persistent_storage(system_instruction) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[0]["content"], system_instruction) - - def test_with_llm_specific_messages(self): - """Test that method works correctly with LLMSpecificMessage objects.""" - llm_specific = LLMSpecificMessage( - llm="test-llm", message={"role": "user", "content": "Specific"} - ) - messages = [{"role": "user", "content": "Standard message"}, llm_specific] - context = LLMContext(messages=messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - self.assertEqual(len(result), 3) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[0]["content"], system_instruction) - self.assertEqual(result[1], messages[0]) - self.assertEqual(result[2], llm_specific) - - def test_system_message_detection_case_sensitivity(self): - """Test that system message detection is case sensitive.""" - messages = [ - {"role": "System", "content": "Mixed case system"}, # Capital S - {"role": "user", "content": "Hello"}, - ] - context = LLMContext(messages=messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - # Should prepend because "System" != "system" - self.assertEqual(len(result), 3) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[0]["content"], system_instruction) - self.assertEqual(result[1], messages[0]) - - def test_message_without_role_key_does_not_crash(self): - """Test that messages without 'role' key are handled gracefully.""" - messages = [{"content": "Message without role"}, {"role": "user", "content": "Hello"}] - context = LLMContext(messages=messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - # Should prepend system instruction since first message doesn't have role="system" - self.assertEqual(len(result), 3) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[0]["content"], system_instruction) - - def test_original_messages_not_modified(self): - """Test that the original messages list is not modified.""" - original_messages = [{"role": "user", "content": "Hello"}] - context = LLMContext(messages=original_messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - # Original messages should remain unchanged - self.assertEqual(len(original_messages), 1) - self.assertEqual(original_messages[0]["role"], "user") - - # Result should have system message prepended - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[1], original_messages[0]) - - def test_complex_message_structure_preserved(self): - """Test that complex message structures are preserved.""" - complex_message = { - "role": "user", - "content": [ - {"type": "text", "text": "Complex message"}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}, - ], - } - messages = [complex_message] - context = LLMContext(messages=messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["role"], "system") - self.assertEqual(result[1], complex_message) - self.assertEqual(result[1]["content"], complex_message["content"]) - - def test_deep_copy_prevents_nested_mutation(self): - """Test that deep copy prevents mutation of nested message content.""" - nested_content = {"nested": {"data": "original"}} - complex_message = {"role": "user", "content": nested_content} - messages = [complex_message] - context = LLMContext(messages=messages) - system_instruction = "You are a helpful assistant." - - result = context.get_messages_for_persistent_storage(system_instruction) - - # Modify the nested content in the result - result[1]["content"]["nested"]["data"] = "modified" - - # Original message should remain unchanged - self.assertEqual(complex_message["content"]["nested"]["data"], "original") - self.assertEqual(context.get_messages()[0]["content"]["nested"]["data"], "original") - - -if __name__ == "__main__": - unittest.main() From 4820f1c059037e39a515d63a97af14ff7f1041a2 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 29 Sep 2025 11:46:56 -0400 Subject: [PATCH 04/93] Address some `AWSNovaSonicLLMService` context-recording edge cases --- src/pipecat/services/aws/nova_sonic/llm.py | 54 +++++++++++++++------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index c1f941db4..658bdbd89 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -212,6 +212,11 @@ class AWSNovaSonicLLMService(LLMService): system_instruction: System-level instruction for the model. tools: Available tools/functions for the model to use. send_transcription_frames: Whether to emit transcription frames. + + .. deprecated:: 0.0.87 + This parameter is deprecated and will be removed in a future version. + Transcription frames are always sent. + **kwargs: Additional arguments passed to the parent LLMService. """ super().__init__(**kwargs) @@ -225,7 +230,19 @@ class AWSNovaSonicLLMService(LLMService): self._params = params or Params() self._system_instruction = system_instruction self._tools = tools - self._send_transcription_frames = send_transcription_frames + + if not send_transcription_frames: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "`send_transcription_frames` is deprecated and will be removed in a future version. " + "Transcription frames are always sent.", + DeprecationWarning, + stacklevel=2, + ) + self._context: Optional[LLMContext] = None self._stream: Optional[ DuplexEventStream[ @@ -239,7 +256,7 @@ class AWSNovaSonicLLMService(LLMService): self._input_audio_content_name: Optional[str] = None self._content_being_received: Optional[CurrentContent] = None self._assistant_is_responding = False - self._needs_re_push_assistant_text = False + self._may_need_repush_assistant_text = False self._ready_to_send_context = False self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False @@ -402,7 +419,7 @@ class AWSNovaSonicLLMService(LLMService): async def _handle_interruption_frame(self): if self._assistant_is_responding: - self._needs_re_push_assistant_text = True + self._may_need_repush_assistant_text = True # # LLM communication: lifecycle @@ -546,7 +563,7 @@ class AWSNovaSonicLLMService(LLMService): self._input_audio_content_name = None self._content_being_received = None self._assistant_is_responding = False - self._needs_re_push_assistant_text = False + self._may_need_repush_assistant_text = False self._ready_to_send_context = False self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False @@ -1031,15 +1048,24 @@ class AWSNovaSonicLLMService(LLMService): logger.debug("Assistant response ended") # If an interruption frame arrived while the assistant was responding - # we probably lost all of the assistant text (see HACK, above), so + # we may have lost all of the assistant text (see HACK, above), so # re-push it downstream to the aggregator now. - if self._needs_re_push_assistant_text: - # We also need to re-push the LLMFullResponseStartFrame since the - # TTSTextFrame would be ignored otherwise (the interruption frame - # would have cleared the assistant aggregator state). - await self.push_frame(LLMFullResponseStartFrame()) - await self.push_frame(TTSTextFrame(self._assistant_text_buffer)) - self._needs_re_push_assistant_text = False + if self._may_need_repush_assistant_text: + # Just in case, check that assistant text hasn't already made it + # into the context (sometimes it does, despite the interruption). + messages = self._context.get_messages() + last_message = messages[-1] if messages else None + if ( + not last_message + or last_message.get("role") != "assistant" + or last_message.get("content") != self._assistant_text_buffer + ): + # We also need to re-push the LLMFullResponseStartFrame since the + # TTSTextFrame would be ignored otherwise (the interruption frame + # would have cleared the assistant aggregator state). + await self.push_frame(LLMFullResponseStartFrame()) + await self.push_frame(TTSTextFrame(self._assistant_text_buffer)) + self._may_need_repush_assistant_text = False # Report the end of the assistant response. await self.push_frame(LLMFullResponseEndFrame()) @@ -1121,10 +1147,6 @@ class AWSNovaSonicLLMService(LLMService): if should_wrap_in_user_started_stopped_speaking_frames: await self.push_frame(UserStoppedSpeakingFrame(), direction=FrameDirection.UPSTREAM) - # Also send the transcription downstream if requested - if self._send_transcription_frames: - await self.push_frame(frame, direction=FrameDirection.DOWNSTREAM) - # Clear out the buffered user text self._user_text_buffer = "" From 87d9e8c9cd78b0a73459cb7e4ba20c6e85d7364a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 16 Oct 2025 14:51:27 -0400 Subject: [PATCH 05/93] Re-apply a couple of recent changes to `AWSNovaSonicLLMService` that were lost in a rebase --- src/pipecat/services/aws/nova_sonic/llm.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 658bdbd89..2731c1b42 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -65,7 +65,7 @@ try: BedrockRuntimeClient, InvokeModelWithBidirectionalStreamOperationInput, ) - from aws_sdk_bedrock_runtime.config import Config, HTTPAuthSchemeResolver, SigV4AuthScheme + from aws_sdk_bedrock_runtime.config import Config from aws_sdk_bedrock_runtime.models import ( BidirectionalInputPayloadPart, InvokeModelWithBidirectionalStreamInput, @@ -73,8 +73,8 @@ try: InvokeModelWithBidirectionalStreamOperationOutput, InvokeModelWithBidirectionalStreamOutput, ) - from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver - from smithy_aws_core.identity import AWSCredentialsIdentity + from smithy_aws_core.auth.sigv4 import SigV4AuthScheme + from smithy_aws_core.identity.static import StaticCredentialsResolver from smithy_core.aio.eventstream import DuplexEventStream except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -453,7 +453,7 @@ class AWSNovaSonicLLMService(LLMService): await self._finish_connecting_if_context_available() except Exception as e: logger.error(f"{self} initialization error: {e}") - self._disconnect() + await self._disconnect() async def _process_completed_function_calls(self, send_new_results: bool): # Check for set of completed function calls in the context @@ -582,15 +582,11 @@ class AWSNovaSonicLLMService(LLMService): config = Config( endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", region=self._region, - aws_credentials_identity_resolver=StaticCredentialsResolver( - credentials=AWSCredentialsIdentity( - access_key_id=self._access_key_id, - secret_access_key=self._secret_access_key, - session_token=self._session_token, - ) - ), - http_auth_scheme_resolver=HTTPAuthSchemeResolver(), - http_auth_schemes={"aws.auth#sigv4": SigV4AuthScheme()}, + aws_access_key_id=self._access_key_id, + aws_secret_access_key=self._secret_access_key, + aws_session_token=self._session_token, + aws_credentials_identity_resolver=StaticCredentialsResolver(), + auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="bedrock")}, ) return BedrockRuntimeClient(config=config) From f13e006db2b18a58e56a7313bd7753af95a7b366 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 16 Oct 2025 14:52:10 -0400 Subject: [PATCH 06/93] Bump version in deprecation message in docstring --- src/pipecat/services/aws/nova_sonic/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 2731c1b42..32d6d4975 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -213,7 +213,7 @@ class AWSNovaSonicLLMService(LLMService): tools: Available tools/functions for the model to use. send_transcription_frames: Whether to emit transcription frames. - .. deprecated:: 0.0.87 + .. deprecated:: 0.0.91 This parameter is deprecated and will be removed in a future version. Transcription frames are always sent. From 4d499324d100a5c6055788cc487aa5237437b35c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 16 Oct 2025 14:57:03 -0400 Subject: [PATCH 07/93] Re-apply a change to `AWSNovaSonicLLMService` that was lost in a rebase --- src/pipecat/services/aws/nova_sonic/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 32d6d4975..5df3bbd21 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -268,7 +268,7 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_text_buffer = "" self._completed_tool_calls = set() - file_path = files("pipecat.services.aws_nova_sonic").joinpath("ready.wav") + file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav") with wave.open(file_path.open("rb"), "rb") as wav_file: self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) From ff0b38859bc422144f1322541ad6081021994d0b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 16 Oct 2025 15:13:21 -0400 Subject: [PATCH 08/93] Remove AWS Nova Sonic's context.py, which was always concerned with types for internal use only. Now those types are either gone or moved elsewhere. --- .../services/aws_nova_sonic/context.py | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 src/pipecat/services/aws_nova_sonic/context.py diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py deleted file mode 100644 index 05a24f337..000000000 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -"""Context management for AWS Nova Sonic LLM service. - -This module provides specialized context aggregators and message handling for AWS Nova Sonic, -including conversation history management and role-specific message processing. -""" - -import warnings - -from pipecat.services.aws.nova_sonic.context import * - -with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Types in pipecat.services.aws_nova_sonic.context are deprecated. " - "Please use the equivalent types from " - "pipecat.services.aws.nova_sonic.context instead.", - DeprecationWarning, - stacklevel=2, - ) From 3b40079120e9a92a5fada0f9e03a41d9b0c2f871 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 16 Oct 2025 17:36:59 -0400 Subject: [PATCH 09/93] Add a detailed warning when trying to import things from pipecat.services.aws_nova_sonic.context or pipecat.services.aws.nova_sonic.context --- .../services/aws/nova_sonic/context.py | 87 +++++++++++++++++++ .../services/aws_nova_sonic/context.py | 87 +++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 src/pipecat/services/aws/nova_sonic/context.py create mode 100644 src/pipecat/services/aws_nova_sonic/context.py diff --git a/src/pipecat/services/aws/nova_sonic/context.py b/src/pipecat/services/aws/nova_sonic/context.py new file mode 100644 index 000000000..864ae783e --- /dev/null +++ b/src/pipecat/services/aws/nova_sonic/context.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Context management for AWS Nova Sonic LLM service. + +This module provides specialized context aggregators and message handling for AWS Nova Sonic, +including conversation history management and role-specific message processing. + +.. deprecated:: 0.0.91 + AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. + Using the new patterns should allow you to not need types from this module. + + BEFORE: + ``` + # Setup + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: AWSNovaSonicLLMContext + # or + context: OpenAILLMContext + + # Reading messages from context + messages = context.messages + ``` + + AFTER: + ``` + # Setup + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + + # Reading messages from context + messages = context.get_messages() + ``` +""" + +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws.nova_sonic.context are deprecated. \n" + "AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. \n" + "Using the new patterns should allow you to not need types from this module.\n\n" + "BEFORE:\n" + "```\n" + "# Setup\n" + "context = OpenAILLMContext(messages, tools)\n" + "context_aggregator = llm.create_context_aggregator(context)\n\n" + "# Context frame type\n" + "frame: OpenAILLMContextFrame\n\n" + "# Context type\n" + "context: AWSNovaSonicLLMContext\n" + "# or\n" + "context: OpenAILLMContext\n\n" + "# Reading messages from context\n" + "messages = context.messages\n" + "```\n\n" + "AFTER:\n" + "```\n" + "# Setup\n" + "context = LLMContext(messages, tools)\n" + "context_aggregator = LLMContextAggregatorPair(context)\n\n" + "# Context frame type\n" + "frame: LLMContextFrame\n\n" + "# Context type\n" + "context: LLMContext\n\n" + "# Reading messages from context\n" + "messages = context.messages\n" + "```", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py new file mode 100644 index 000000000..b54f80185 --- /dev/null +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Context management for AWS Nova Sonic LLM service. + +This module provides specialized context aggregators and message handling for AWS Nova Sonic, +including conversation history management and role-specific message processing. + +.. deprecated:: 0.0.91 + AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. + Using the new patterns should allow you to not need types from this module. + + BEFORE: + ``` + # Setup + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: AWSNovaSonicLLMContext + # or + context: OpenAILLMContext + + # Reading messages from context + messages = context.messages + ``` + + AFTER: + ``` + # Setup + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + + # Reading messages from context + messages = context.get_messages() + ``` +""" + +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic.context are deprecated. \n" + "AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. \n" + "Using the new patterns should allow you to not need types from this module.\n\n" + "BEFORE:\n" + "```\n" + "# Setup\n" + "context = OpenAILLMContext(messages, tools)\n" + "context_aggregator = llm.create_context_aggregator(context)\n\n" + "# Context frame type\n" + "frame: OpenAILLMContextFrame\n\n" + "# Context type\n" + "context: AWSNovaSonicLLMContext\n" + "# or\n" + "context: OpenAILLMContext\n\n" + "# Reading messages from context\n" + "messages = context.messages\n" + "```\n\n" + "AFTER:\n" + "```\n" + "# Setup\n" + "context = LLMContext(messages, tools)\n" + "context_aggregator = LLMContextAggregatorPair(context)\n\n" + "# Context frame type\n" + "frame: LLMContextFrame\n\n" + "# Context type\n" + "context: LLMContext\n\n" + "# Reading messages from context\n" + "messages = context.messages\n" + "```", + DeprecationWarning, + stacklevel=2, + ) From 6cae61f2cc6a92f17cafa1be89046473ef315f66 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 09:46:32 -0400 Subject: [PATCH 10/93] Add a bit more detail to CHANGELOG entry about AWSNovaSonicLLMService's support for LLMContext --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index feb86e5d2..11ee10764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `LLMContext`, it is not meant to be swapped out for another LLM service at runtime.) + Worth noting: whether or not you use the new context-setup pattern with + `AWSNovaSonicLLMService`, some types have changed under the hood: + + ```python + ## BEFORE: + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: AWSNovaSonicLLMContext + # or + context: OpenAILLMContext + + # Reading messages from context + messages = context.messages + + ## AFTER: + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + + # Reading messages from context + messages = context.get_messages() + ``` + - Added support for `bulbul:v3` model in `SarvamTTSService` and `SarvamHttpTTSService`. - Added `keyterms_prompt` parameter to `AssemblyAIConnectionParams`. From 02a88022ddd0d2b625441a6f7be18c2cace7c49e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 10:06:09 -0400 Subject: [PATCH 11/93] Add a bit more detail to CHANGELOG related to AWSNovaSonicLLMService's support for LLMContext --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11ee10764..fb4209309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ```python ## BEFORE: + # Context aggregator type + context_aggregator: AWSNovaSonicContextAggregatorPair + # Context frame type frame: OpenAILLMContextFrame @@ -39,6 +42,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 messages = context.messages ## AFTER: + + # Context aggregator type + context_aggregator: LLMContextAggregatorPair + # Context frame type frame: LLMContextFrame @@ -85,6 +92,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `SpeechmaticsSTTService` updated dependencies for `speechmatics-rt>=0.5.0`. +### Deprecated + +- The `send_transcription_frames` argument to `AWSNovaSonicLLMService` is + deprecated. Transcription frames are now always sent. They go upstream, to be + handled by the user context aggregator. See "Changed" section for details. + +- Types in `pipecat.services.aws.nova_sonic.context` have been deprecated due + to changes to support `LLMContext`. See "Changed" section for details. + ### Fixed - Fixed an issue in `RivaSegmentedSTTService` where a runtime error occurred due From ec3cd241823467271431d88fba4bf630a22db3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 20 Oct 2025 06:56:41 -0700 Subject: [PATCH 12/93] pyproject: update daily-python to 0.20.0 --- CHANGELOG.md | 3 ++- pyproject.toml | 2 +- uv.lock | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71e5300c0..234b9ce90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `keyterms_prompt` parameter to `AssemblyAIConnectionParams`. - Added `speech_model` parameter to `AssemblyAIConnectionParams` to access the multilingual model. -- + - Added support for trickle ICE to the `SmallWebRTCTransport`. - Added support for updating `OpenAITTSService` settings (`instructions` and @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Package upgrades: + - `daily-python` upgraded to 0.20.0. - `openai` upgraded to support up to 2.x.x. - `openpipe` upgraded to support up to 5.x.x. diff --git a/pyproject.toml b/pyproject.toml index 65546311c..c3bac258a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.19.9" ] +daily = [ "daily-python~=0.20.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] diff --git a/uv.lock b/uv.lock index dec765912..96761c015 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,13 +1282,13 @@ wheels = [ [[package]] name = "daily-python" -version = "0.19.9" +version = "0.20.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/85/6064c3225e5b190e522e8f3bc6a460efc5e3e6632f16fd5f9799c44ba57a/daily_python-0.19.9-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:cbc558ad7d49e79b550bf7567b9ceae75e2864d4fcaf41c90377b620e38a2461", size = 13365213, upload-time = "2025-09-06T00:31:00.224Z" }, - { url = "https://files.pythonhosted.org/packages/23/58/af986c6881180a46a7b60dd418ce58d6d7c0c4ffc48d261748067c679317/daily_python-0.19.9-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:446bb9ee848d88bc68ca29a2216793c9b5ebaf5991bf604daf76f7c5a53d5919", size = 11711673, upload-time = "2025-09-06T00:31:02.526Z" }, - { url = "https://files.pythonhosted.org/packages/9d/48/1cad4c3e92cdb5ef06467d972c76a510fe5e807513334b10ad7f8c21bf74/daily_python-0.19.9-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2facaf82b614404c642c70bbf0874fb045d8ad46400acb051470cd4df93cb4db", size = 13679393, upload-time = "2025-09-06T00:31:04.999Z" }, - { url = "https://files.pythonhosted.org/packages/3c/e9/354f4699619e83d13e266256b2352b21741ac527e3e5ab5f2264d5c482cd/daily_python-0.19.9-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ffc205efca7b47739efd358febab17577248c8db2ebc4d17d819307a83b9eefc", size = 14221932, upload-time = "2025-09-06T00:31:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/ce81ebf11a04cd133a5539e08f85060574711fff05a1d6ad29705f0755c1/daily_python-0.20.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:7da3f1df8cd9ef7f7fcc96ce688348dc903f62d82b6dd155a53bc64b7a74f3a7", size = 13259887, upload-time = "2025-10-16T22:14:12.262Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1e/51f06f3486c978e1184af2271e800ce6a6e8a8f95d61ee6624bae88ae9cd/daily_python-0.20.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:d02fd7b8c8079ceaa550ef23db052cdf70a8ffaf8ab6a8bc1a1e97bf0b939464", size = 11642453, upload-time = "2025-10-16T22:14:14.477Z" }, + { url = "https://files.pythonhosted.org/packages/71/c9/f767f0b479abd39330569ad61fb9db4661aae56cd74bb27c6f3483595463/daily_python-0.20.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a5c8718982c221dc18b41fb0692c9f8435f115f72e74994c94d3b9c6dad7c534", size = 13634216, upload-time = "2025-10-16T22:14:16.235Z" }, + { url = "https://files.pythonhosted.org/packages/e8/10/5c6d7b000bee36c2a0587a092a34c7486d2de831fc8e44ed42b16a6bd99f/daily_python-0.20.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ca9132aef1bdb5be663d1894b440dab1f998ebb3f45dfc31d44effabded4bc08", size = 14282189, upload-time = "2025-10-16T22:14:18.229Z" }, ] [[package]] @@ -4637,7 +4637,7 @@ requires-dist = [ { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, - { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.19.9" }, + { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.20.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = "~=4.7.0" }, { name = "docstring-parser", specifier = "~=0.16" }, { name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" }, @@ -4708,7 +4708,7 @@ requires-dist = [ { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=0.1.10" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.0" }, { name = "soxr", specifier = "~=0.5.0" }, - { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.4.0" }, + { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.5.0" }, { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, @@ -6619,14 +6619,14 @@ wheels = [ [[package]] name = "speechmatics-rt" -version = "0.4.2" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/2e/d694390d58b9b6807280441d1275856f5a316c3e8a815c2037502636bbea/speechmatics_rt-0.4.2.tar.gz", hash = "sha256:c0f7ed34442b0f505a12d1b19c8cc8dc2cc0b1a423aeb5669ca0738fc5e59f0d", size = 26142, upload-time = "2025-09-30T10:50:36.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/57/26/10359e1f16c2aa6a198eb11a9056f4a86a8bb8d4e610bbbe4a118b227b59/speechmatics_rt-0.5.0.tar.gz", hash = "sha256:ca974a186a012f946fd997deeaf3bf1c4f203f6d6e05a866172d27709183afc8", size = 26832, upload-time = "2025-10-15T15:54:25.695Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/c7/2cd551c71e14256ca463f31feec17f466b57c2730d636e20803e7a541104/speechmatics_rt-0.4.2-py3-none-any.whl", hash = "sha256:70b91ff750e2f7516eaf1839d39f7a8ac65ff6665638b837cf67bab9cc9967bc", size = 32131, upload-time = "2025-09-30T10:50:35.656Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/9931ebe9360e9d385c68826b33137c2c9a4cfa361cd929d1ac6e72ebfe53/speechmatics_rt-0.5.0-py3-none-any.whl", hash = "sha256:58151488f891fa00cf7054f0cfab1b1eb94b55c3441be587f7941c726caef991", size = 32850, upload-time = "2025-10-15T15:54:24.5Z" }, ] [[package]] From a0c93ab6dec68b98f79eccf9e0e98947c951d1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 20 Oct 2025 09:07:50 -0700 Subject: [PATCH 13/93] update CHANGELOG cosmetics --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 815057f6b..f13b3cece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,11 +56,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 messages = context.get_messages() ``` -- Added support for `bulbul:v3` model in `SarvamTTSService` and `SarvamHttpTTSService`. +- Added support for `bulbul:v3` model in `SarvamTTSService` and + `SarvamHttpTTSService`. - Added `keyterms_prompt` parameter to `AssemblyAIConnectionParams`. -- Added `speech_model` parameter to `AssemblyAIConnectionParams` to access the multilingual model. +- Added `speech_model` parameter to `AssemblyAIConnectionParams` to access the + multilingual model. - Added support for trickle ICE to the `SmallWebRTCTransport`. @@ -97,7 +99,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `send_transcription_frames` argument to `AWSNovaSonicLLMService` is deprecated. Transcription frames are now always sent. They go upstream, to be - handled by the user context aggregator. See "Changed" section for details. + handled by the user context aggregator. See "Added" section for details. - Types in `pipecat.services.aws.nova_sonic.context` have been deprecated due to changes to support `LLMContext`. See "Changed" section for details. From abf015026177d4c9f5d60485af89ce30a3d7b61b Mon Sep 17 00:00:00 2001 From: vipyne Date: Wed, 15 Oct 2025 10:45:36 -0500 Subject: [PATCH 14/93] add 47-custom-frame-processor.py to foundational examples --- .../foundational/47-custom-frame-processor.py | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 examples/foundational/47-custom-frame-processor.py diff --git a/examples/foundational/47-custom-frame-processor.py b/examples/foundational/47-custom-frame-processor.py new file mode 100644 index 000000000..7de01ab17 --- /dev/null +++ b/examples/foundational/47-custom-frame-processor.py @@ -0,0 +1,223 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import io +import os +import re + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + Frame, + FunctionCallResultFrame, + InputAudioRawFrame, + InterruptionFrame, + LLMRunFrame, + LLMTextFrame, + StartFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, + VADUserStartedSpeakingFrame, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +class CustomFrameProcessor(FrameProcessor): + """CustomFrameProcessor does 3 things: + + 1. keeps count of `InputAudioRawFrame` frames and logs count + when a `UserStoppedSpeakingFrame` is emitted. + + 2. Filters `LLMTextFrame` frames and replaces "the" with "the pumpkin". + + 3. Logs the following frames: + BotStartedSpeakingFrame + BotStoppedSpeakingFrame + CancelFrame + EndFrame + InterruptionFrame + StartFrame + UserStartedSpeakingFrame + VADUserStartedSpeakingFrame + + 4. Always pushes all frames + + """ + + def __init__(self): + super().__init__() + self._raw_audio_input_frame_count = 0 + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + #### 1. + # InputAudioRawFrames are noisy- probably don't want to log every instance + # keep a count and only log it when we see `UserStoppedSpeakingFrame` + if isinstance(frame, InputAudioRawFrame): + self._raw_audio_input_frame_count = self._raw_audio_input_frame_count + 1 + await self.push_frame(frame, direction) + + elif isinstance(frame, UserStoppedSpeakingFrame): + logger.info( + f"* * frame: {frame}; number of `InputAudioRawFrame` frames so far: {self._raw_audio_input_frame_count}" + ) + await self.push_frame(frame, direction) + + #### 2. + # everytime the LLM's response includes "the", replace it with "the pumpkin" + elif isinstance(frame, LLMTextFrame): + if "the" in frame.text: + text = re.sub(r" the\b", " the pumpkin", frame.text) + frame.text = text + await self.push_frame(frame, direction) + + #### 3. + # frames types to log + elif isinstance( + frame, + ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + InterruptionFrame, + StartFrame, + UserStartedSpeakingFrame, + VADUserStartedSpeakingFrame, + ), + ): + logger.info(f"* * frame: {frame}") + await self.push_frame(frame, direction) + + #### 4. + # ALWAYS push all other frames + else: + # SUPER IMPORTANT: always push every frame! + await self.push_frame(frame, direction) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + custom_frame_processor = CustomFrameProcessor() + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + custom_frame_processor, # filter and log frames + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Kick off the conversation. + messages.append( + { + "role": "system", + "content": "Please introduce yourself to the user and inform them that your responses illustrate use of a Custom Frame Processor.", + } + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() From de46631bac1eddc7a01a8ac29bb90c47bdddb3e6 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Mon, 20 Oct 2025 18:39:00 -0300 Subject: [PATCH 15/93] Fixed an issue where the RTVIProcessor was sending duplicate UserStartedSpeakingFrame and UserStoppedSpeakingFrame messages. --- CHANGELOG.md | 3 +++ src/pipecat/processors/frameworks/rtvi.py | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f13b3cece..4763aecd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue where the `RTVIProcessor` was sending duplicate + `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` messages. + - Fixed an issue in `RivaSegmentedSTTService` where a runtime error occurred due to a mismatch in the _handle_transcription method's signature. diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 393cb7115..08d127ef6 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -1018,6 +1018,7 @@ class RTVIObserver(BaseObserver): if ( isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)) + and (direction == FrameDirection.DOWNSTREAM) and self._params.user_speaking_enabled ): await self._handle_interruptions(frame) From 427efecf5bf4eb358d19eca4cbafbd04ca62c07a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Oct 2025 09:21:30 -0400 Subject: [PATCH 16/93] Organize the env.example file --- env.example | 200 +++++++++++++++++++++++++++++----------------------- 1 file changed, 112 insertions(+), 88 deletions(-) diff --git a/env.example b/env.example index f707f49c9..2865772ea 100644 --- a/env.example +++ b/env.example @@ -4,6 +4,9 @@ AICOUSTICS_LICENSE_KEY=... # Anthropic ANTHROPIC_API_KEY=... +# Assembly AI +ASSEMBLYAI_API_KEY=... + # Async ASYNCAI_API_KEY=... ASYNCAI_VOICE_ID=... @@ -21,12 +24,19 @@ AZURE_CHATGPT_API_KEY=... AZURE_CHATGPT_ENDPOINT=https://... AZURE_CHATGPT_MODEL=... +AZURE_REALTIME_API_KEY=... +AZURE_REALTIME_BASE_URL=... + AZURE_DALLE_API_KEY=... AZURE_DALLE_ENDPOINT=https://... AZURE_DALLE_MODEL=... # Cartesia CARTESIA_API_KEY=... +CARTESIA_VOICE_ID=... + +# Cerebras +CEREBRAS_API_KEY=... # Daily DAILY_API_KEY=... @@ -35,57 +45,48 @@ DAILY_SAMPLE_ROOM_URL=https://... # Deepgram DEEPGRAM_API_KEY=... +# DeepSeek +DEEPSEEK_API_KEY=... + # ElevenLabs ELEVENLABS_API_KEY=... ELEVENLABS_VOICE_ID=... -# Neuphonic -NEUPHONIC_API_KEY=... - # Fal FAL_KEY=... # Fireworks FIREWORKS_API_KEY=... +# Fish Audio +FISH_API_KEY=... + # Gladia GLADIA_API_KEY=... GLADIA_REGION=... # Google GOOGLE_API_KEY=... -GOOGLE_CLOUD_PROJECT_ID=... -GOOGLE_TEST_CREDENTIALS=... GOOGLE_VERTEX_TEST_CREDENTIALS=... +GOOGLE_CLOUD_PROJECT_ID=... +GOOGLE_CLOUD_LOCATION=... +GOOGLE_TEST_CREDENTIALS=... + +# Grok +GROK_API_KEY=... + +# Groq +GROQ_API_KEY=... + +# Heygen +HEYGEN_API_KEY=... # Hume HUME_API_KEY=... +HUME_VOICE_ID=... -# LMNT -LMNT_API_KEY=... -LMNT_VOICE_ID=... - -# Perplexity -PERPLEXITY_API_KEY=... - -# PlayHT -PLAYHT_USER_ID=... -PLAYHT_API_KEY=... - -# OpenAI -OPENAI_API_KEY=... - -# OpenPipe -OPENPIPE_API_KEY=... - -# Tavus -TAVUS_API_KEY=... -TAVUS_REPLICA_ID=... -TAVUS_PERSONA_ID=... - -# Simli -SIMLI_API_KEY=... -SIMLI_FACE_ID=... +# Inworld +INWORLD_API_KEY=... # Krisp KRISP_MODEL_PATH=... @@ -93,77 +94,100 @@ KRISP_MODEL_PATH=... # Krisp Viva KRISP_VIVA_MODEL_PATH=... -# DeepSeek -DEEPSEEK_API_KEY=... +# LiveKit +LIVEKIT_API_KEY=... +LIVEKIT_API_SECRET=... -# Groq -GROQ_API_KEY=... - -# Grok -GROK_API_KEY=... - -# Inworld -INWORLD_API_KEY=... - -# Together.ai -TOGETHER_API_KEY=... - -# Cerebras -CEREBRAS_API_KEY=... - -# Fish Audio -FISH_API_KEY=... - -# Assembly AI -ASSEMBLYAI_API_KEY=... - -# OpenRouter -OPENROUTER_API_KEY=... - -# Piper -PIPER_BASE_URL=... - -# Smart turn -LOCAL_SMART_TURN_MODEL_PATH=... -FAL_SMART_TURN_API_KEY=... - -# Twilio -TWILIO_ACCOUNT_SID=... -TWILIO_AUTH_TOKEN=... +# LMNT +LMNT_API_KEY=... +LMNT_VOICE_ID=... # MiniMax MINIMAX_API_KEY=... MINIMAX_GROUP_ID=... -# Sarvam AI -SARVAM_API_KEY=... - -# Soniox -SONIOX_API_KEY= - -# Speechmatics -SPEECHMATICS_API_KEY=... - -# SambaNova -SAMBANOVA_API_KEY=... - -# Sentry -SENTRY_DSN=... - -# Heygen -HEYGEN_API_KEY=... - # Mistral MISTRAL_API_KEY=... +# Neuphonic +NEUPHONIC_API_KEY=... + # NVIDIA NVIDIA_API_KEY=... +# OpenAI +OPENAI_API_KEY=... + +# OpenPipe +OPENPIPE_API_KEY=... + +# OpenRouter +OPENROUTER_API_KEY=... + +# Perplexity +PERPLEXITY_API_KEY=... + +# Picovoice Koala +KOALA_ACCESS_KEY=... + +# Piper +PIPER_BASE_URL=... + +# PlayHT +PLAYHT_USER_ID=... +PLAYHT_API_KEY=... + +# Plivo +PLIVO_AUTH_ID=... +PLIVO_AUTH_TOKEN=... + # Qwen QWEN_API_KEY=... +# Rime +RIME_API_KEY=... +RIME_VOICE_ID=... + +# SambaNova +SAMBANOVA_API_KEY=... + +# Sarvam AI +SARVAM_API_KEY=... + +# Sentry +SENTRY_DSN=... + +# Simli +SIMLI_API_KEY=... +SIMLI_FACE_ID=... + +# Smart turn +LOCAL_SMART_TURN_MODEL_PATH=... +FAL_SMART_TURN_API_KEY=... + +# Soniox +SONIOX_API_KEY=... + +# Speechmatics +SPEECHMATICS_API_KEY=... + +# Tavus +TAVUS_API_KEY=... +TAVUS_REPLICA_ID=... + +# Telnyx +TELNYX_API_KEY=... +TELNYX_ACCOUNT_SID=... + +# Together.ai +TOGETHER_API_KEY=... + +# Twilio +TWILIO_ACCOUNT_SID=... +TWILIO_AUTH_TOKEN=... + # WhatsApp -WHATSAPP_TOKEN= -WHATSAPP_WEBHOOK_VERIFICATION_TOKEN= -WHATSAPP_PHONE_NUMBER_ID= -WHATSAPP_APP_SECRET= \ No newline at end of file +WHATSAPP_TOKEN=... +WHATSAPP_WEBHOOK_VERIFICATION_TOKEN=... +WHATSAPP_PHONE_NUMBER_ID=... +WHATSAPP_APP_SECRET=... \ No newline at end of file From fbf274374cf5a61d42d2dbad9b36d43e32d5c0ef Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 09:56:31 -0400 Subject: [PATCH 17/93] Add back types that were removed, when they should only have been deprecated --- .../services/aws/nova_sonic/context.py | 365 +++++++++++++++++- .../services/aws_nova_sonic/context.py | 76 +--- 2 files changed, 367 insertions(+), 74 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/context.py b/src/pipecat/services/aws/nova_sonic/context.py index 864ae783e..d21a14608 100644 --- a/src/pipecat/services/aws/nova_sonic/context.py +++ b/src/pipecat/services/aws/nova_sonic/context.py @@ -10,7 +10,8 @@ This module provides specialized context aggregators and message handling for AW including conversation history management and role-specific message processing. .. deprecated:: 0.0.91 - AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. + AWS Nova Sonic no longer uses types from this module under the hood. + It now uses `LLMContext` and `LLMContextAggregatorPair`. Using the new patterns should allow you to not need types from this module. BEFORE: @@ -53,8 +54,10 @@ import warnings with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( - "Types in pipecat.services.aws.nova_sonic.context are deprecated. \n" - "AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. \n" + "Types in pipecat.services.aws.nova_sonic.context (or " + "pipecat.services.aws_nova_sonic.context) are deprecated. \n" + "AWS Nova Sonic no longer uses types from this module under the hood. \n" + "It now uses `LLMContext` and `LLMContextAggregatorPair`. \n" "Using the new patterns should allow you to not need types from this module.\n\n" "BEFORE:\n" "```\n" @@ -85,3 +88,359 @@ with warnings.catch_warnings(): DeprecationWarning, stacklevel=2, ) + +import copy +from dataclasses import dataclass, field +from enum import Enum + +from loguru import logger + +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + DataFrame, + Frame, + FunctionCallResultFrame, + InterruptionFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolChoiceFrame, + LLMSetToolsFrame, + TextFrame, + UserImageRawFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) + + +class Role(Enum): + """Roles supported in AWS Nova Sonic conversations. + + Parameters: + SYSTEM: System-level messages (not used in conversation history). + USER: Messages sent by the user. + ASSISTANT: Messages sent by the assistant. + TOOL: Messages sent by tools (not used in conversation history). + """ + + SYSTEM = "SYSTEM" + USER = "USER" + ASSISTANT = "ASSISTANT" + TOOL = "TOOL" + + +@dataclass +class AWSNovaSonicConversationHistoryMessage: + """A single message in AWS Nova Sonic conversation history. + + Parameters: + role: The role of the message sender (USER or ASSISTANT only). + text: The text content of the message. + """ + + role: Role # only USER and ASSISTANT + text: str + + +@dataclass +class AWSNovaSonicConversationHistory: + """Complete conversation history for AWS Nova Sonic initialization. + + Parameters: + system_instruction: System-level instruction for the conversation. + messages: List of conversation messages between user and assistant. + """ + + system_instruction: str = None + messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) + + +class AWSNovaSonicLLMContext(OpenAILLMContext): + """Specialized LLM context for AWS Nova Sonic service. + + Extends OpenAI context with Nova Sonic-specific message handling, + conversation history management, and text buffering capabilities. + """ + + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize AWS Nova Sonic LLM context. + + Args: + messages: Initial messages for the context. + tools: Available tools for the context. + **kwargs: Additional arguments passed to parent class. + """ + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self, system_instruction: str = ""): + self._assistant_text = "" + self._user_text = "" + self._system_instruction = system_instruction + + @staticmethod + def upgrade_to_nova_sonic( + obj: OpenAILLMContext, system_instruction: str + ) -> "AWSNovaSonicLLMContext": + """Upgrade an OpenAI context to AWS Nova Sonic context. + + Args: + obj: The OpenAI context to upgrade. + system_instruction: System instruction for the context. + + Returns: + The upgraded AWS Nova Sonic context. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): + obj.__class__ = AWSNovaSonicLLMContext + obj.__setup_local(system_instruction) + return obj + + # NOTE: this method has the side-effect of updating _system_instruction from messages + def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: + """Get conversation history for initializing AWS Nova Sonic session. + + Processes stored messages and extracts system instruction and conversation + history in the format expected by AWS Nova Sonic. + + Returns: + Formatted conversation history with system instruction and messages. + """ + history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) + + # Bail if there are no messages + if not self.messages: + return history + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into "instruction" + if messages[0].get("role") == "system": + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + history.system_instruction = content + elif isinstance(content, list): + history.system_instruction = content[0].get("text") + if history.system_instruction: + self._system_instruction = history.system_instruction + + # Process remaining messages to fill out conversation history. + # Nova Sonic supports "user" and "assistant" messages in history. + for message in messages: + history_message = self.from_standard_message(message) + if history_message: + history.messages.append(history_message) + + return history + + def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Returns: + List of messages including system instruction if present. + """ + messages = super().get_messages_for_persistent_storage() + # If we have a system instruction and messages doesn't already contain it, add it + if self._system_instruction and not (messages and messages[0].get("role") == "system"): + messages.insert(0, {"role": "system", "content": self._system_instruction}) + return messages + + def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + """Convert standard message format to Nova Sonic format. + + Args: + message: Standard message dictionary to convert. + + Returns: + Nova Sonic conversation history message, or None if not convertible. + """ + role = message.get("role") + if message.get("role") == "user" or message.get("role") == "assistant": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + # There won't be content if this is an assistant tool call entry. + # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation + # history + if content: + return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova + # Sonic conversation history + + def buffer_user_text(self, text): + """Buffer user text for later flushing to context. + + Args: + text: User text to buffer. + """ + self._user_text += f" {text}" if self._user_text else text + # logger.debug(f"User text buffered: {self._user_text}") + + def flush_aggregated_user_text(self) -> str: + """Flush buffered user text to context as a complete message. + + Returns: + The flushed user text, or empty string if no text was buffered. + """ + if not self._user_text: + return "" + user_text = self._user_text + message = { + "role": "user", + "content": [{"type": "text", "text": user_text}], + } + self._user_text = "" + self.add_message(message) + # logger.debug(f"Context updated (user): {self.get_messages_for_logging()}") + return user_text + + def buffer_assistant_text(self, text): + """Buffer assistant text for later flushing to context. + + Args: + text: Assistant text to buffer. + """ + self._assistant_text += text + # logger.debug(f"Assistant text buffered: {self._assistant_text}") + + def flush_aggregated_assistant_text(self): + """Flush buffered assistant text to context as a complete message.""" + if not self._assistant_text: + return + message = { + "role": "assistant", + "content": [{"type": "text", "text": self._assistant_text}], + } + self._assistant_text = "" + self.add_message(message) + # logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}") + + +@dataclass +class AWSNovaSonicMessagesUpdateFrame(DataFrame): + """Frame containing updated AWS Nova Sonic context. + + Parameters: + context: The updated AWS Nova Sonic LLM context. + """ + + context: AWSNovaSonicLLMContext + + +class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): + """Context aggregator for user messages in AWS Nova Sonic conversations. + + Extends the OpenAI user context aggregator to emit Nova Sonic-specific + context update frames. + """ + + async def process_frame( + self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM + ): + """Process frames and emit Nova Sonic-specific context updates. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + await super().process_frame(frame, direction) + + # Parent does not push LLMMessagesUpdateFrame + if isinstance(frame, LLMMessagesUpdateFrame): + await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) + + +class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Context aggregator for assistant messages in AWS Nova Sonic conversations. + + Provides specialized handling for assistant responses and function calls + in AWS Nova Sonic context, with custom frame processing logic. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Nova Sonic-specific logic. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + # HACK: For now, disable the context aggregator by making it just pass through all frames + # that the parent handles (except the function call stuff, which we still need). + # For an explanation of this hack, see + # AWSNovaSonicLLMService._report_assistant_response_text_added. + if isinstance( + frame, + ( + InterruptionFrame, + LLMFullResponseStartFrame, + LLMFullResponseEndFrame, + TextFrame, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMSetToolChoiceFrame, + UserImageRawFrame, + BotStoppedSpeakingFrame, + ), + ): + await self.push_frame(frame, direction) + else: + await super().process_frame(frame, direction) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call results for AWS Nova Sonic. + + Args: + frame: The function call result frame to handle. + """ + await super().handle_function_call_result(frame) + + # The standard function callback code path pushes the FunctionCallResultFrame from the LLM + # itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side + # context. Let's push a special frame to do that. + await self.push_frame( + AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) + + +@dataclass +class AWSNovaSonicContextAggregatorPair: + """Pair of user and assistant context aggregators for AWS Nova Sonic. + + Parameters: + _user: The user context aggregator. + _assistant: The assistant context aggregator. + """ + + _user: AWSNovaSonicUserContextAggregator + _assistant: AWSNovaSonicAssistantContextAggregator + + def user(self) -> AWSNovaSonicUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ + return self._user + + def assistant(self) -> AWSNovaSonicAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ + return self._assistant diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index b54f80185..5728e4ff0 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -10,78 +10,12 @@ This module provides specialized context aggregators and message handling for AW including conversation history management and role-specific message processing. .. deprecated:: 0.0.91 - AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. + AWS Nova Sonic no longer uses types from this module under the hood. + It now uses `LLMContext` and `LLMContextAggregatorPair`. Using the new patterns should allow you to not need types from this module. - BEFORE: - ``` - # Setup - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) - - # Context frame type - frame: OpenAILLMContextFrame - - # Context type - context: AWSNovaSonicLLMContext - # or - context: OpenAILLMContext - - # Reading messages from context - messages = context.messages - ``` - - AFTER: - ``` - # Setup - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - # Context frame type - frame: LLMContextFrame - - # Context type - context: LLMContext - - # Reading messages from context - messages = context.get_messages() - ``` + See deprecation warning in pipecat.services.aws.nova_sonic.context for more + details. """ -import warnings - -with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Types in pipecat.services.aws_nova_sonic.context are deprecated. \n" - "AWS Nova Sonic now supports `LLMContext` and `LLMContextAggregatorPair`. \n" - "Using the new patterns should allow you to not need types from this module.\n\n" - "BEFORE:\n" - "```\n" - "# Setup\n" - "context = OpenAILLMContext(messages, tools)\n" - "context_aggregator = llm.create_context_aggregator(context)\n\n" - "# Context frame type\n" - "frame: OpenAILLMContextFrame\n\n" - "# Context type\n" - "context: AWSNovaSonicLLMContext\n" - "# or\n" - "context: OpenAILLMContext\n\n" - "# Reading messages from context\n" - "messages = context.messages\n" - "```\n\n" - "AFTER:\n" - "```\n" - "# Setup\n" - "context = LLMContext(messages, tools)\n" - "context_aggregator = LLMContextAggregatorPair(context)\n\n" - "# Context frame type\n" - "frame: LLMContextFrame\n\n" - "# Context type\n" - "context: LLMContext\n\n" - "# Reading messages from context\n" - "messages = context.messages\n" - "```", - DeprecationWarning, - stacklevel=2, - ) +from pipecat.services.aws.nova_sonic.context import * From 2f92cb878114dcf33642bfc59720f1148b73a4fd Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 21 Oct 2025 11:41:52 -0300 Subject: [PATCH 18/93] Fixed an issue in the runner's proxy_request where a session that exists but has empty data was being treated as invalid. --- src/pipecat/runner/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index a90c96ac5..1e690145d 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -309,7 +309,7 @@ def _setup_webrtc_routes( ): """Mimic Pipecat Cloud's proxy.""" active_session = active_sessions.get(session_id) - if not active_session: + if active_session is None: return Response(content="Invalid or not-yet-ready session_id", status_code=404) if path.endswith("api/offer"): From 0502ec6c440c660cd1f92c27d01370110aa173d3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Oct 2025 11:57:02 -0400 Subject: [PATCH 19/93] Changelog entry for PR #2877 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4763aecd2..311b2b94f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added an ellipsis character (`…`) to the end of sentence detection in the + string utils. + - Expanded support for universal `LLMContext` to `AWSNovaSonicLLMService`. As a reminder, the context-setup pattern when using `LLMContext` is: @@ -110,7 +113,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` messages. - Fixed an issue in `RivaSegmentedSTTService` where a runtime error occurred due - to a mismatch in the _handle_transcription method's signature. + to a mismatch in the `_handle_transcription` method's signature. - Fixed multiple pipeline task cancellation issues. `asyncio.CancelledError` is now handled properly in `PipelineTask` making it possible to cancel an asyncio From 8b24bae9c5f60e1af56859a541c66bb9a4a0bbb9 Mon Sep 17 00:00:00 2001 From: vipyne Date: Tue, 21 Oct 2025 11:42:06 -0500 Subject: [PATCH 20/93] pr notes --- .../foundational/47-custom-frame-processor.py | 100 +++++------------- 1 file changed, 25 insertions(+), 75 deletions(-) diff --git a/examples/foundational/47-custom-frame-processor.py b/examples/foundational/47-custom-frame-processor.py index 7de01ab17..26fe86391 100644 --- a/examples/foundational/47-custom-frame-processor.py +++ b/examples/foundational/47-custom-frame-processor.py @@ -16,20 +16,9 @@ from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnal from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - CancelFrame, - EndFrame, Frame, - FunctionCallResultFrame, - InputAudioRawFrame, - InterruptionFrame, LLMRunFrame, - LLMTextFrame, - StartFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, - VADUserStartedSpeakingFrame, + MetricsFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -48,76 +37,42 @@ from pipecat.transports.daily.transport import DailyParams load_dotenv(override=True) -class CustomFrameProcessor(FrameProcessor): - """CustomFrameProcessor does 3 things: +def format_metrics(metrics, indent=0): + lines = [] + tab = "\t" * indent - 1. keeps count of `InputAudioRawFrame` frames and logs count - when a `UserStoppedSpeakingFrame` is emitted. + for metric in metrics: + lines.append(tab + type(metric).__name__) + for field, value in vars(metric).items(): + if hasattr(value, "__dict__") and not isinstance( + value, (str, int, float, bool, type(None)) + ): + lines.append(f"{tab}\t{field}={type(value).__name__}") + for k, v in vars(value).items(): + lines.append(f"{tab}\t\t{k}={repr(v)}") + else: + lines.append(f"{tab}\t{field}={repr(value)}") - 2. Filters `LLMTextFrame` frames and replaces "the" with "the pumpkin". + return "\n".join(lines) - 3. Logs the following frames: - BotStartedSpeakingFrame - BotStoppedSpeakingFrame - CancelFrame - EndFrame - InterruptionFrame - StartFrame - UserStartedSpeakingFrame - VADUserStartedSpeakingFrame - 4. Always pushes all frames +class MetricsFrameLogger(FrameProcessor): + """MetricsFrameLogger logs all MetericsFrames. + AND it Always pushes all frames. """ def __init__(self): super().__init__() - self._raw_audio_input_frame_count = 0 async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - #### 1. - # InputAudioRawFrames are noisy- probably don't want to log every instance - # keep a count and only log it when we see `UserStoppedSpeakingFrame` - if isinstance(frame, InputAudioRawFrame): - self._raw_audio_input_frame_count = self._raw_audio_input_frame_count + 1 + if isinstance(frame, MetricsFrame): + logger.info(f"{frame.name}\n {format_metrics(frame.data)}") await self.push_frame(frame, direction) - elif isinstance(frame, UserStoppedSpeakingFrame): - logger.info( - f"* * frame: {frame}; number of `InputAudioRawFrame` frames so far: {self._raw_audio_input_frame_count}" - ) - await self.push_frame(frame, direction) - - #### 2. - # everytime the LLM's response includes "the", replace it with "the pumpkin" - elif isinstance(frame, LLMTextFrame): - if "the" in frame.text: - text = re.sub(r" the\b", " the pumpkin", frame.text) - frame.text = text - await self.push_frame(frame, direction) - - #### 3. - # frames types to log - elif isinstance( - frame, - ( - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - CancelFrame, - EndFrame, - InterruptionFrame, - StartFrame, - UserStartedSpeakingFrame, - VADUserStartedSpeakingFrame, - ), - ): - logger.info(f"* * frame: {frame}") - await self.push_frame(frame, direction) - - #### 4. - # ALWAYS push all other frames + # ALWAYS push all frames else: # SUPER IMPORTANT: always push every frame! await self.push_frame(frame, direction) @@ -155,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - custom_frame_processor = CustomFrameProcessor() + metrics_frame_processor = MetricsFrameLogger() messages = [ { @@ -173,10 +128,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt, context_aggregator.user(), llm, - custom_frame_processor, # filter and log frames tts, transport.output(), context_aggregator.assistant(), + metrics_frame_processor, # pretty print metrics frames ] ) @@ -193,12 +148,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") # Kick off the conversation. - messages.append( - { - "role": "system", - "content": "Please introduce yourself to the user and inform them that your responses illustrate use of a Custom Frame Processor.", - } - ) + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") From 23385ca3d2d1645d0c6e4710916c35d11e7d8112 Mon Sep 17 00:00:00 2001 From: vipyne Date: Tue, 21 Oct 2025 11:43:47 -0500 Subject: [PATCH 21/93] replace foundational example 08-bots-arguing.py with 08-custom-frame-processor.py --- examples/foundational/08-bots-arguing.py | 147 ------------------ ...cessor.py => 08-custom-frame-processor.py} | 13 +- 2 files changed, 5 insertions(+), 155 deletions(-) delete mode 100644 examples/foundational/08-bots-arguing.py rename examples/foundational/{47-custom-frame-processor.py => 08-custom-frame-processor.py} (97%) diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py deleted file mode 100644 index b84e945c3..000000000 --- a/examples/foundational/08-bots-arguing.py +++ /dev/null @@ -1,147 +0,0 @@ -import asyncio -import logging -import os -from typing import Tuple - -import aiohttp -from dotenv import load_dotenv - -from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMContextFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.processors.aggregators import SentenceAggregator -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.daily import configure -from pipecat.services.azure import AzureLLMService, AzureTTSService -from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.fal import FalImageGenService -from pipecat.transports.daily.transport import DailyTransport - -load_dotenv(override=True) - -logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") -logger = logging.getLogger("pipecat") -logger.setLevel(logging.DEBUG) - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Respond bot", - duration_minutes=10, - mic_enabled=True, - mic_sample_rate=16000, - camera_enabled=True, - camera_width=1024, - camera_height=1024, - ) - - llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), - model=os.getenv("AZURE_CHATGPT_MODEL"), - ) - tts1 = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), - ) - tts2 = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id="jBpfuIE2acCO8z3wKNLl", - ) - dalle = FalImageGenService( - params=FalImageGenService.InputParams(image_size="1024x1024"), - aiohttp_session=session, - key=os.getenv("FAL_KEY"), - ) - - bot1_messages = [ - { - "role": "system", - "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long.", - }, - ] - bot2_messages = [ - { - "role": "system", - "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich.", - }, - ] - - async def get_text_and_audio(messages) -> Tuple[str, bytearray]: - """This function streams text from the LLM and uses the TTS service to convert - that text to speech as it's received. - """ - source_queue = asyncio.Queue() - sink_queue = asyncio.Queue() - sentence_aggregator = SentenceAggregator() - pipeline = Pipeline([llm, sentence_aggregator, tts1], source_queue, sink_queue) - - await source_queue.put(LLMContextFrame(LLMContext(messages))) - await source_queue.put(EndFrame()) - await pipeline.run_pipeline() - - message = "" - all_audio = bytearray() - while sink_queue.qsize(): - frame = sink_queue.get_nowait() - if isinstance(frame, TextFrame): - message += frame.text - elif isinstance(frame, AudioFrame): - all_audio.extend(frame.audio) - - return (message, all_audio) - - async def get_bot1_statement(): - message, audio = await get_text_and_audio(bot1_messages) - - bot1_messages.append({"role": "assistant", "content": message}) - bot2_messages.append({"role": "user", "content": message}) - - return audio - - async def get_bot2_statement(): - message, audio = await get_text_and_audio(bot2_messages) - - bot2_messages.append({"role": "assistant", "content": message}) - bot1_messages.append({"role": "user", "content": message}) - - return audio - - async def argue(): - for i in range(100): - print(f"In iteration {i}") - - bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed" - - (audio1, image_data1) = await asyncio.gather( - get_bot1_statement(), dalle.run_image_gen(bot1_description) - ) - await transport.send_queue.put( - [ - ImageFrame(image_data1[1], image_data1[2]), - AudioFrame(audio1), - ] - ) - - bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed" - - (audio2, image_data2) = await asyncio.gather( - get_bot2_statement(), dalle.run_image_gen(bot2_description) - ) - await transport.send_queue.put( - [ - ImageFrame(image_data2[1], image_data2[2]), - AudioFrame(audio2), - ] - ) - - await asyncio.gather(transport.run(), argue()) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/47-custom-frame-processor.py b/examples/foundational/08-custom-frame-processor.py similarity index 97% rename from examples/foundational/47-custom-frame-processor.py rename to examples/foundational/08-custom-frame-processor.py index 26fe86391..20da4f876 100644 --- a/examples/foundational/47-custom-frame-processor.py +++ b/examples/foundational/08-custom-frame-processor.py @@ -57,13 +57,10 @@ def format_metrics(metrics, indent=0): class MetricsFrameLogger(FrameProcessor): - """MetricsFrameLogger logs all MetericsFrames. + """MetricsFrameLogger formats and logs all MetericsFrames""" - AND it Always pushes all frames. - """ - - def __init__(self): - super().__init__() + def __init__(self, **kwargs): + super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -110,8 +107,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - metrics_frame_processor = MetricsFrameLogger() - messages = [ { "role": "system", @@ -122,6 +117,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context = LLMContext(messages) context_aggregator = LLMContextAggregatorPair(context) + metrics_frame_processor = MetricsFrameLogger() + pipeline = Pipeline( [ transport.input(), From 9f66b0ba41d0ee89f78404d25755dd3d8afb5585 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Oct 2025 08:24:00 -0400 Subject: [PATCH 22/93] Add Pipecat CLI to README's ecosystem section --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6b00832f3..ea25d492c 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,10 @@ Looking to build structured conversations? Check out [Pipecat Flows](https://git Want to build beautiful and engaging experiences? Checkout the [Voice UI Kit](https://github.com/pipecat-ai/voice-ui-kit), a collection of components, hooks and templates for building voice AI applications quickly. +### 🛠️ Create and deploy projects + +Create a new project in under a minute with the [Pipecat CLI](https://github.com/pipecat-ai/pipecat-cli). Then use the CLI to monitor and deploy your agent to production. + ### 🔍 Debugging Looking for help debugging your pipeline and processors? Check out [Whisker](https://github.com/pipecat-ai/whisker), a real-time Pipecat debugger. From 52ab0eccc0f285a3eccb6afa7fc3ce51fb1e3bc8 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Oct 2025 15:52:27 -0400 Subject: [PATCH 23/93] Quickstart to use Pipecat CLI --- CHANGELOG.md | 2 +- examples/quickstart/README.md | 22 ++++++++++++++++------ examples/quickstart/pcc-deploy.toml | 5 +++++ examples/quickstart/pyproject.toml | 7 ++++--- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4763aecd2..945b25667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,7 +110,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` messages. - Fixed an issue in `RivaSegmentedSTTService` where a runtime error occurred due - to a mismatch in the _handle_transcription method's signature. + to a mismatch in the `_handle_transcription` method's signature. - Fixed multiple pipeline task cancellation issues. `asyncio.CancelledError` is now handled properly in `PipelineTask` making it possible to cancel an asyncio diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md index cf7c2de1a..91a3fd888 100644 --- a/examples/quickstart/README.md +++ b/examples/quickstart/README.md @@ -73,13 +73,13 @@ Transform your local bot into a production-ready service. Pipecat Cloud handles 1. [Sign up for Pipecat Cloud](https://pipecat.daily.co/sign-up). -2. Install the Pipecat Cloud CLI: +2. Install the Pipecat CLI: ```bash - uv add pipecatcloud + uv tool install pipecat-ai-cli ``` -> 💡 Tip: You can run the `pipecatcloud` CLI using the `pcc` alias. +> 💡 Tip: You can run the `pipecat` CLI using the `pc` alias. 3. Set up Docker for building your bot image: @@ -113,12 +113,22 @@ secret_set = "quickstart-secrets" > 💡 Tip: [Set up `image_credentials`](https://docs.pipecat.ai/deployment/pipecat-cloud/fundamentals/secrets#image-pull-secrets) in your TOML file for authenticated image pulls +### Log in to Pipecat Cloud + +To start using the CLI, authenticate to Pipecat Cloud: + +```bash +pipecat cloud auth login +``` + +You'll be presented with a link that you can click to authenticate your client. + ### Configure secrets Upload your API keys to Pipecat Cloud's secure storage: ```bash -uv run pcc secrets set quickstart-secrets --file .env +pipecat cloud secrets set quickstart-secrets --file .env ``` This creates a secret set called `quickstart-secrets` (matching your TOML file) and uploads all your API keys from `.env`. @@ -128,13 +138,13 @@ This creates a secret set called `quickstart-secrets` (matching your TOML file) Build your Docker image and push to Docker Hub: ```bash -uv run pcc docker build-push +pipecat cloud docker build-push ``` Deploy to Pipecat Cloud: ```bash -uv run pcc deploy +pipecat cloud deploy ``` ### Connect to your agent diff --git a/examples/quickstart/pcc-deploy.toml b/examples/quickstart/pcc-deploy.toml index 28413327f..ff77c45dc 100644 --- a/examples/quickstart/pcc-deploy.toml +++ b/examples/quickstart/pcc-deploy.toml @@ -1,6 +1,11 @@ agent_name = "quickstart" image = "your_username/quickstart:0.1" secret_set = "quickstart-secrets" +agent_profile = "agent-1x" + +# RECOMMENDED: Set an image pull secret: +# https://docs.pipecat.ai/deployment/pipecat-cloud/fundamentals/secrets#image-pull-secrets +# image_credentials = "your_image_pull_secret" [scaling] min_agents = 1 diff --git a/examples/quickstart/pyproject.toml b/examples/quickstart/pyproject.toml index 5d9df3eb4..863e350d4 100644 --- a/examples/quickstart/pyproject.toml +++ b/examples/quickstart/pyproject.toml @@ -4,13 +4,14 @@ version = "0.1.0" description = "Quickstart example for building voice AI bots with Pipecat" requires-python = ">=3.10" dependencies = [ - "pipecat-ai[webrtc,daily,silero,deepgram,openai,cartesia,local-smart-turn-v3,runner]>=0.0.86", - "pipecatcloud>=0.2.4" + "pipecat-ai[webrtc,daily,silero,deepgram,openai,cartesia,local-smart-turn-v3,runner]", + "pipecat-ai-cli" ] [dependency-groups] dev = [ - "ruff~=0.12.1", + "pyright>=1.1.404,<2", + "ruff>=0.12.11,<1", ] [tool.ruff] From 776a3526f9d0192e373bb1d9de9e1c703cf36fc7 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 17:35:57 -0400 Subject: [PATCH 24/93] Add `messages` property to `LLMContext` for usage parity with `OpenAILLMContext`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This wasn't really an issue before, when folks were *knowingly* migrating from `OpenAILLMContext` to `LLMContext`. But in the latest AWS Nova Sonic change, we're swapping it out from under folks, so this kind of compatibility is more important. For context, the reason we *didn't* offer the `messages` property earlier was to aid in the development of `LLMContext`—we wanted to draw attention to all the places where messages were being read from context, so we could find the places where we might need to pass an argument to the read. --- CHANGELOG.md | 6 ------ .../processors/aggregators/llm_context.py | 16 +++++++++++++++- src/pipecat/services/aws/nova_sonic/context.py | 10 ---------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 311b2b94f..34cee203f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,9 +41,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # or context: OpenAILLMContext - # Reading messages from context - messages = context.messages - ## AFTER: # Context aggregator type @@ -54,9 +51,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # Context type context: LLMContext - - # Reading messages from context - messages = context.get_messages() ``` - Added support for `bulbul:v3` model in `SarvamTTSService` and diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index b60c029cb..913566909 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -106,6 +106,19 @@ class LLMContext: self._tools: ToolsSchema | NotGiven = LLMContext._normalize_and_validate_tools(tools) self._tool_choice: LLMContextToolChoice | NotGiven = tool_choice + @property + def messages(self) -> List[LLMContextMessage]: + """Get the current messages list. + + NOTE: This is equivalent to calling `get_messages()` with no filter. If + you want to filter out LLM-specific messages that don't pertain to your + LLM, use `get_messages()` directly. + + Returns: + List of conversation messages. + """ + return self.get_messages() + def get_messages(self, llm_specific_filter: Optional[str] = None) -> List[LLMContextMessage]: """Get the current messages list. @@ -113,7 +126,8 @@ class LLMContext: llm_specific_filter: Optional filter to return LLM-specific messages for the given LLM, in addition to the standard messages. If messages end up being filtered, an error will be - logged. + logged; this is intended to catch accidental use of + incompatible LLM-specific messages. Returns: List of conversation messages. diff --git a/src/pipecat/services/aws/nova_sonic/context.py b/src/pipecat/services/aws/nova_sonic/context.py index d21a14608..c9aab439f 100644 --- a/src/pipecat/services/aws/nova_sonic/context.py +++ b/src/pipecat/services/aws/nova_sonic/context.py @@ -27,9 +27,6 @@ including conversation history management and role-specific message processing. context: AWSNovaSonicLLMContext # or context: OpenAILLMContext - - # Reading messages from context - messages = context.messages ``` AFTER: @@ -43,9 +40,6 @@ including conversation history management and role-specific message processing. # Context type context: LLMContext - - # Reading messages from context - messages = context.get_messages() ``` """ @@ -70,8 +64,6 @@ with warnings.catch_warnings(): "context: AWSNovaSonicLLMContext\n" "# or\n" "context: OpenAILLMContext\n\n" - "# Reading messages from context\n" - "messages = context.messages\n" "```\n\n" "AFTER:\n" "```\n" @@ -82,8 +74,6 @@ with warnings.catch_warnings(): "frame: LLMContextFrame\n\n" "# Context type\n" "context: LLMContext\n\n" - "# Reading messages from context\n" - "messages = context.messages\n" "```", DeprecationWarning, stacklevel=2, From 1e8629bf96a245ecf9dc4d6866ff5eaa8dd5bac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 20 Oct 2025 17:06:13 -0700 Subject: [PATCH 25/93] runner: allow passing an api_key to configure --- src/pipecat/runner/daily.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/runner/daily.py b/src/pipecat/runner/daily.py index 5bff7d060..568d54381 100644 --- a/src/pipecat/runner/daily.py +++ b/src/pipecat/runner/daily.py @@ -76,6 +76,7 @@ class DailyRoomConfig(BaseModel): async def configure( aiohttp_session: aiohttp.ClientSession, *, + api_key: Optional[str] = None, room_exp_duration: Optional[float] = 2.0, token_exp_duration: Optional[float] = 2.0, sip_caller_phone: Optional[str] = None, @@ -92,6 +93,7 @@ async def configure( Args: aiohttp_session: HTTP session for making API requests. + api_key: Daily API key. room_exp_duration: Room expiration time in hours. token_exp_duration: Token expiration time in hours. sip_caller_phone: Phone number or identifier for SIP display name. @@ -129,7 +131,7 @@ async def configure( config = await configure(session, room_properties=custom_props) """ # Check for required API key - api_key = os.getenv("DAILY_API_KEY") + api_key = api_key or os.getenv("DAILY_API_KEY") if not api_key: raise Exception( "DAILY_API_KEY environment variable is required. " From 576bd67e85c1e7d909b15a59b8180f450468c56e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 20 Oct 2025 17:06:33 -0700 Subject: [PATCH 26/93] runner: add body field to RunnerArguments --- CHANGELOG.md | 3 +++ src/pipecat/runner/types.py | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 311b2b94f..4004bfd24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `RunnerArguments` now include the `body` field, so there's no need to add it + to subclasses. Also, all `RunnerArguments` fields are now keyword-only. + - `CartesiaSTTService` now inherits from `WebsocketSTTService`. - Package upgrades: diff --git a/src/pipecat/runner/types.py b/src/pipecat/runner/types.py index 89aecd84c..fab2404f3 100644 --- a/src/pipecat/runner/types.py +++ b/src/pipecat/runner/types.py @@ -20,9 +20,11 @@ from fastapi import WebSocket class RunnerArguments: """Base class for runner session arguments.""" - handle_sigint: bool = field(init=False) - handle_sigterm: bool = field(init=False) - pipeline_idle_timeout_secs: int = field(init=False) + # Use kw_only so subclasses don't need to worry about ordering. + handle_sigint: bool = field(init=False, kw_only=True) + handle_sigterm: bool = field(init=False, kw_only=True) + pipeline_idle_timeout_secs: int = field(init=False, kw_only=True) + body: Optional[Any] = field(default_factory=dict, kw_only=True) def __post_init__(self): self.handle_sigint = False @@ -42,7 +44,6 @@ class DailyRunnerArguments(RunnerArguments): room_url: str token: Optional[str] = None - body: Optional[Any] = field(default_factory=dict) @dataclass @@ -55,7 +56,6 @@ class WebSocketRunnerArguments(RunnerArguments): """ websocket: WebSocket - body: Optional[Any] = field(default_factory=dict) @dataclass From 459af58540e9c19ad4278e54105332ddd093e91b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 20 Oct 2025 17:06:53 -0700 Subject: [PATCH 27/93] runner: allow starting a bot from Daily's /start endpoint --- CHANGELOG.md | 4 +++ src/pipecat/runner/run.py | 68 ++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4004bfd24..cb0ecae43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- It is now possible to start a bot from the `/start` endpoint when using the + runner Daily's transport. This follows the Pipecat Cloud format with + `createDailyRoom` and `body` fields in the POST request body. + - Added an ellipsis character (`…`) to the end of sentence detection in the string utils. diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index 1e690145d..4c1d3135d 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -82,6 +82,7 @@ from loguru import logger from pipecat.runner.types import ( DailyRunnerArguments, + RunnerArguments, SmallWebRTCRunnerArguments, WebSocketRunnerArguments, ) @@ -529,9 +530,9 @@ def _setup_daily_routes(app: FastAPI): """Set up Daily-specific routes.""" @app.get("/") - async def start_agent(): + async def create_room_and_start_agent(): """Launch a Daily bot and redirect to room.""" - print("Starting bot with Daily transport") + print("Starting bot with Daily transport and redirecting to Daily room") import aiohttp @@ -546,11 +547,11 @@ def _setup_daily_routes(app: FastAPI): asyncio.create_task(bot_module.bot(runner_args)) return RedirectResponse(room_url) - async def _handle_rtvi_request(request: Request): - """Common handler for both /start and /connect endpoints. + @app.post("/start") + async def start_agent(request: Request): + """Handler for /start endpoints. Expects POST body like:: - { "createDailyRoom": true, "dailyRoomProperties": { "start_video_off": true }, @@ -567,45 +568,32 @@ def _setup_daily_routes(app: FastAPI): logger.error(f"Failed to parse request body: {e}") request_data = {} - # Extract the body data that should be passed to the bot - # This mimics Pipecat Cloud's behavior - bot_body = request_data.get("body", {}) + create_daily_room = request_data.get("createDailyRoom", False) + body = request_data.get("body", {}) - # Log the extracted body data for debugging - if bot_body: - logger.info(f"Extracted body data for bot: {bot_body}") + bot_module = _get_bot_module() + + result = None + if create_daily_room: + import aiohttp + + from pipecat.runner.daily import configure + + async with aiohttp.ClientSession() as session: + room_url, token = await configure(session) + runner_args = DailyRunnerArguments(room_url=room_url, token=token, body=body) + result = { + "dailyRoom": room_url, + "dailyToken": token, + "sessionId": str(uuid.uuid4()), + } else: - logger.debug("No body data provided in request") + runner_args = RunnerArguments(body=body) - from pipecat.runner.daily import configure + # Start the bot in the background + asyncio.create_task(bot_module.bot(runner_args)) - async with aiohttp.ClientSession() as session: - room_url, token = await configure(session) - - # Start the bot in the background with extracted body data - bot_module = _get_bot_module() - runner_args = DailyRunnerArguments(room_url=room_url, token=token, body=bot_body) - asyncio.create_task(bot_module.bot(runner_args)) - # Match PCC /start endpoint response format: - return {"dailyRoom": room_url, "dailyToken": token} - - @app.post("/start") - async def rtvi_start(request: Request): - """Launch a Daily bot and return connection info for RTVI clients.""" - return await _handle_rtvi_request(request) - - @app.post("/connect") - async def rtvi_connect(request: Request): - """Launch a Daily bot and return connection info for RTVI clients. - - .. deprecated:: 0.0.78 - Use /start instead. This endpoint will be removed in a future version. - """ - logger.warning( - "DEPRECATED: /connect endpoint is deprecated. Please use /start instead. " - "This endpoint will be removed in a future version." - ) - return await _handle_rtvi_request(request) + return result def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str): From 5849485bc603a19e09ddfc5d2e716457ca6f79b3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 20 Oct 2025 20:20:33 -0400 Subject: [PATCH 28/93] fix: AWSBedrockLLMService compatibility for newer Claude models --- CHANGELOG.md | 6 +++ .../foundational/07m-interruptible-aws.py | 4 +- .../foundational/14r-function-calling-aws.py | 4 +- src/pipecat/services/aws/llm.py | 49 +++++++++++++------ 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 311b2b94f..6ccaf1798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue where the `RTVIProcessor` was sending duplicate `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` messages. +- Fixed an issue in `AWSBedrockLLMService` where both `temperature` and `top_p` + were always sent together, causing conflicts with models like Claude Sonnet 4.5 + that don't allow both parameters simultaneously. The service now only includes + inference parameters that are explicitly set, and `InputParams` defaults have + been changed to `None` to rely on AWS Bedrock's built-in model defaults. + - Fixed an issue in `RivaSegmentedSTTService` where a runtime error occurred due to a mismatch in the `_handle_transcription` method's signature. diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index 9343797a9..2d3bb1dac 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -67,8 +67,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = AWSBedrockLLMService( aws_region="us-west-2", - model="us.anthropic.claude-3-5-haiku-20241022-v1:0", - params=AWSBedrockLLMService.InputParams(temperature=0.8, latency="optimized"), + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8), ) messages = [ diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index 03aa7bb96..15f7e37a0 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -79,8 +79,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = AWSBedrockLLMService( aws_region="us-west-2", - model="us.anthropic.claude-3-5-haiku-20241022-v1:0", - params=AWSBedrockLLMService.InputParams(temperature=0.8, latency="optimized"), + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8), ) # You can also register a function_name of None to get all functions diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 377848f76..716aee776 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -720,11 +720,11 @@ class AWSBedrockLLMService(LLMService): additional_model_request_fields: Additional model-specific parameters. """ - max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1) - temperature: Optional[float] = Field(default_factory=lambda: 0.7, ge=0.0, le=1.0) - top_p: Optional[float] = Field(default_factory=lambda: 0.999, ge=0.0, le=1.0) + max_tokens: Optional[int] = Field(default=None, ge=1) + temperature: Optional[float] = Field(default=None, ge=0.0, le=1.0) + top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) stop_sequences: Optional[List[str]] = Field(default_factory=lambda: []) - latency: Optional[str] = Field(default_factory=lambda: "standard") + latency: Optional[str] = Field(default=None) additional_model_request_fields: Optional[Dict[str, Any]] = Field(default_factory=dict) def __init__( @@ -801,6 +801,24 @@ class AWSBedrockLLMService(LLMService): """ return True + def _build_inference_config(self) -> Dict[str, Any]: + """Build inference config with only the parameters that are set. + + This prevents conflicts with models (e.g., Claude Sonnet 4.5) that don't + allow certain parameter combinations like temperature and top_p together. + + Returns: + Dictionary containing only the inference parameters that are not None. + """ + inference_config = {} + if self._settings["max_tokens"] is not None: + inference_config["maxTokens"] = self._settings["max_tokens"] + if self._settings["temperature"] is not None: + inference_config["temperature"] = self._settings["temperature"] + if self._settings["top_p"] is not None: + inference_config["topP"] = self._settings["top_p"] + return inference_config + async def run_inference(self, context: LLMContext | OpenAILLMContext) -> Optional[str]: """Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context. @@ -826,16 +844,16 @@ class AWSBedrockLLMService(LLMService): model_id = self.model_name # Prepare request parameters + inference_config = self._build_inference_config() + request_params = { "modelId": model_id, "messages": messages, - "inferenceConfig": { - "maxTokens": 8192, - "temperature": 0.7, - "topP": 0.9, - }, } + if inference_config: + request_params["inferenceConfig"] = inference_config + if system: request_params["system"] = system @@ -974,21 +992,20 @@ class AWSBedrockLLMService(LLMService): tools = params_from_context["tools"] tool_choice = params_from_context["tool_choice"] - # Set up inference config - inference_config = { - "maxTokens": self._settings["max_tokens"], - "temperature": self._settings["temperature"], - "topP": self._settings["top_p"], - } + # Set up inference config - only include parameters that are set + inference_config = self._build_inference_config() # Prepare request parameters request_params = { "modelId": self.model_name, "messages": messages, - "inferenceConfig": inference_config, "additionalModelRequestFields": self._settings["additional_model_request_fields"], } + # Only add inference config if it has parameters + if inference_config: + request_params["inferenceConfig"] = inference_config + # Add system message if system: request_params["system"] = system From cc4c96d0990a82b340a0c7291aea8de994a538c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 21 Oct 2025 19:00:51 -0700 Subject: [PATCH 29/93] update CHANGELOG for 0.0.91 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca9da580..06e7b998b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.91] - 2025-10-21 ### Added From 49239a23c668766d024b7624cf5ebb2959f536da Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 21 Oct 2025 23:27:38 -0400 Subject: [PATCH 30/93] Upgrade aws_sdk_bedrock_runtime to v0.1.1 --- CHANGELOG.md | 7 ++++++ pyproject.toml | 2 +- uv.lock | 66 +++++++++++++++++++++++++------------------------- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06e7b998b..6e727c42e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- Upgraded `aws_sdk_bedrock_runtime` to v0.1.1 to resolve potential CPU issues + when running `AWSNovaSonicLLMService`. + ## [0.0.91] - 2025-10-21 ### Added diff --git a/pyproject.toml b/pyproject.toml index c3bac258a..fd992fa86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] -aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.0; python_version>='3.12'" ] +aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] diff --git a/uv.lock b/uv.lock index 96761c015..8fae18c09 100644 --- a/uv.lock +++ b/uv.lock @@ -410,16 +410,16 @@ wheels = [ [[package]] name = "aws-sdk-bedrock-runtime" -version = "0.1.0" +version = "0.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/84/e1/39971b907c83a7525bab112c9b395e1bb6d4bc23bc1712d6d7a050662217/aws_sdk_bedrock_runtime-0.1.0.tar.gz", hash = "sha256:bd062de5a48404f64e1dfe6fb8841fbbf68e8f1798c357d14eb427274cb96a2b", size = 85419, upload-time = "2025-09-29T19:40:01.855Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/78/48574454b3cac869df67665e4a403ebfc3abfcfba2c2ff01ccfd67d55f8f/aws_sdk_bedrock_runtime-0.1.1.tar.gz", hash = "sha256:c896f99e675c3a1ab600633a07b785f3dc9fe8ab94f640b1f992b63da2dfc784", size = 82446, upload-time = "2025-10-21T20:25:25.845Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/e1/5b36bffe85010cdcd44730d1c2d5244653d57c002f440141d7fc3b9f1347/aws_sdk_bedrock_runtime-0.1.0-py3-none-any.whl", hash = "sha256:aac6ff47069d456ca5e23083d96a01e3e0cbc215414e6753c289d7d9efef3335", size = 78853, upload-time = "2025-09-29T19:40:00.341Z" }, + { url = "https://files.pythonhosted.org/packages/83/07/62c0b70223d178c138f29124ac2f7973a6ba803abc7735b6a01a85217f3d/aws_sdk_bedrock_runtime-0.1.1-py3-none-any.whl", hash = "sha256:c0336b377b2112cf88197d3d44302fbeb3efb1101989fa49ae55e78f49cfe345", size = 74954, upload-time = "2025-10-21T20:25:24.973Z" }, ] [[package]] @@ -433,31 +433,31 @@ wheels = [ [[package]] name = "awscrt" -version = "0.28.1" +version = "0.28.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a0/1c/5c9e6a7375c2a1355aadeb2d06c96c95934ec37ff29ebaab2919f59c3ff1/awscrt-0.28.1.tar.gz", hash = "sha256:70a28fd6ff3e0abb7854ea8a9133bc9e5de681a0d9bdbd8a599a23d13a448685", size = 37956730, upload-time = "2025-09-19T00:58:31.564Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/1b/a885a699217967c3ff0e1c49ac5b1e2a050d1a8b87d1e85e958a56e3d3f5/awscrt-0.28.2.tar.gz", hash = "sha256:9715a888f2042e710dc8aeb355963a29b77e7a4cc25a14659cebd21a5fa476c1", size = 37894849, upload-time = "2025-10-14T19:06:16.867Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/75/dd62276f2907a9ffcf9f8f780c08ce9938bd0550a15c887db198b47f24d3/awscrt-0.28.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:47f885104065918d311102e2b08b943966717c0f3b0c5de5908d2fd08de32198", size = 3376838, upload-time = "2025-09-19T00:57:32.988Z" }, - { url = "https://files.pythonhosted.org/packages/a7/93/562709cdf13a7606548426ecc31326ba3f6839f91e98a1e9230208308afb/awscrt-0.28.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3df2316e77ad88c456b7eb2c9928007d379ed892154c1969d35b98653617e576", size = 3821522, upload-time = "2025-09-19T00:57:35.456Z" }, - { url = "https://files.pythonhosted.org/packages/43/f0/6c6ff81f5a4c6d085eb450854149087bf9240c37c467c747521f47901b32/awscrt-0.28.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3a060d930939f142345f46a344e19ffc0dada657b04d02216b8adffba550c0a0", size = 4087344, upload-time = "2025-09-19T00:57:36.62Z" }, - { url = "https://files.pythonhosted.org/packages/37/0a/71c097505add4ceea4ac05153311715acb7489cd82ec69db4570130f4698/awscrt-0.28.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43f81ca6bfe85c38ad9765605aaaa646a1ed6fd7210dbedf67c113dd245f425e", size = 3745148, upload-time = "2025-09-19T00:57:38Z" }, - { url = "https://files.pythonhosted.org/packages/79/1b/2b02b705a47b64e6c4d401087ddd30d4ad9af70172812ae8c62fb2b7a70c/awscrt-0.28.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fc8e2307d9dbe76842015a14701ff7e9cf2619d674621b2d55b769414e17b3fc", size = 3972439, upload-time = "2025-09-19T00:57:39.74Z" }, - { url = "https://files.pythonhosted.org/packages/f1/19/429c81c7a0d81a5edce9cc6d9a878c8b65d8b5b69fa5a2725a6e0b1380c1/awscrt-0.28.1-cp310-cp310-win32.whl", hash = "sha256:6e7b094587e5332d428300340dcc18794a1fcfa76d636f216fc0f5c8405ba604", size = 3915231, upload-time = "2025-09-19T00:57:41.096Z" }, - { url = "https://files.pythonhosted.org/packages/83/81/769ad51fc6dcfd8bf9e0aa59c252013da0eb9e32c050ecbd1fc25f71689a/awscrt-0.28.1-cp310-cp310-win_amd64.whl", hash = "sha256:ac02f10f7384fdb68187f8d5d94743a271b16fa94be81481ce7684942f6a4b35", size = 4051668, upload-time = "2025-09-19T00:57:42.696Z" }, - { url = "https://files.pythonhosted.org/packages/9e/55/0ee537d146f24d6e76eaf02d462a83c572788233603bb9bda969fbf23307/awscrt-0.28.1-cp311-abi3-macosx_10_15_universal2.whl", hash = "sha256:cb36052f9aa34e77687a8037559bbea331fc9d5d77cd71ab0cf4e6d72af73f72", size = 3376673, upload-time = "2025-09-19T00:57:43.875Z" }, - { url = "https://files.pythonhosted.org/packages/f0/54/12700a4b9545680baa3e2d4d0e543bb4775a639df56ee51cbb29b71e0947/awscrt-0.28.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc59829152a5806eb2708aca5c5084c11dd18ecbe765e03eb314d5a360eeaa62", size = 3782870, upload-time = "2025-09-19T00:57:45.737Z" }, - { url = "https://files.pythonhosted.org/packages/1d/e7/7b189ace9e187b9b55ed4a6ec9a451579b2f16bd01d402f79a19cc8e1603/awscrt-0.28.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2f20bc774599b9d85ce66689415da529ddd1d2215da818e005deedc4688fe61", size = 4048789, upload-time = "2025-09-19T00:57:47.327Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e0/2e5472019906dfcc5fadcdba4bad9e69dabb95bbc0c110cfe555ee8461dc/awscrt-0.28.1-cp311-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:491b8b9c73a288cfd5e0cbdac16aabb5313d5cfc33bbe461763a5ddc26624f70", size = 3687832, upload-time = "2025-09-19T00:57:48.563Z" }, - { url = "https://files.pythonhosted.org/packages/71/f2/7e05d371bb888ee9f15e83d189287838f7b6ea40dfc91eacb3acd24b8529/awscrt-0.28.1-cp311-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:4c6c7125b7e9fcc999eb685d1cace8d4f2ffc63f8f3d8ef7f77e1a97d9552863", size = 3913378, upload-time = "2025-09-19T00:57:50.185Z" }, - { url = "https://files.pythonhosted.org/packages/79/6b/a542a65a22edb85d64742970c21721e66e0f9f67911a11c7a5c3626a1b17/awscrt-0.28.1-cp311-abi3-win32.whl", hash = "sha256:1dcb33d7cf8f69881ac6ef75a5b9b40816be58678b1bb07ccbe0230281bdbc81", size = 3912809, upload-time = "2025-09-19T00:57:51.797Z" }, - { url = "https://files.pythonhosted.org/packages/df/64/16cc8a0011e3ca5dda13605befa7e6db29bfb3073c67f6e8dad90be0a8ae/awscrt-0.28.1-cp311-abi3-win_amd64.whl", hash = "sha256:670caaf556876913bcfb9d8183d43d67a6c7b52998f2f398abd1c21632a006f8", size = 4048979, upload-time = "2025-09-19T00:57:53.061Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/debbd3a2f03c5953b56b1c3b321bab16293f857ea3005e3f7e5dded5e0b2/awscrt-0.28.1-cp313-abi3-macosx_10_15_universal2.whl", hash = "sha256:22311d25135b937ee5617e35a6554961727527dcfa3e06efdefe187a6abe65c4", size = 3375565, upload-time = "2025-09-19T00:57:54.598Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4f/9388917ad45c043acd7c4ab2c28b9e2b5ddf29e21a82bfc01a7626c18c04/awscrt-0.28.1-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e58740cf0e41552fdf7909e10814b312ab090ebe54741354a61507e0c6d4ebfd", size = 3775366, upload-time = "2025-09-19T00:57:56.238Z" }, - { url = "https://files.pythonhosted.org/packages/8a/e3/3ef301cdef76b22ce14b041e04c6cf65ba4491d00e9f5b400c0699f6c63e/awscrt-0.28.1-cp313-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e69f163a207a8b172abbfea1f51045301ed1ac8bbaf76958a6b5e81d72e5b89", size = 4043403, upload-time = "2025-09-19T00:57:57.4Z" }, - { url = "https://files.pythonhosted.org/packages/60/9c/4f89922333724c4da851752549ca97dd147420734ef6c4ece56d5dd65e09/awscrt-0.28.1-cp313-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:592f4b234ecafa6cde86e55e42c4fe84c4e1ffe9fb11b0a8b8f0ffb8c62fa2cc", size = 3678742, upload-time = "2025-09-19T00:57:59.055Z" }, - { url = "https://files.pythonhosted.org/packages/0e/d4/adb97ba5f888ed201aa1f9e9f8d6cfc0dbaf80f0e937b3acb7411febdaa8/awscrt-0.28.1-cp313-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b16321f1d2bf5b4991a213059c1b5dc07954edfc424d154b093824465ec94ce2", size = 3908438, upload-time = "2025-09-19T00:58:00.71Z" }, - { url = "https://files.pythonhosted.org/packages/41/ac/600ea0a6f4ba6543c50417c8e78b09f2cd73dd0f0d4c3e9e52220a8badbe/awscrt-0.28.1-cp313-abi3-win32.whl", hash = "sha256:3e0a23635aa75b4af163ff9bf5a0873928369b1ac32c8b1351741a95472ccf71", size = 3907625, upload-time = "2025-09-19T00:58:03.235Z" }, - { url = "https://files.pythonhosted.org/packages/9e/24/d22c7197b1e53c76b5eb71d640a4728b9b7621075d8dbcc054e16b5b98f0/awscrt-0.28.1-cp313-abi3-win_amd64.whl", hash = "sha256:9849c88ca0830396724acf988e2759895118fe7dd2a23dab21978c8600d01a11", size = 4043878, upload-time = "2025-09-19T00:58:04.595Z" }, + { url = "https://files.pythonhosted.org/packages/73/b4/1a566e493bdfa6e918ba78bcd2e45dda99a25407a4fd974db2666228d154/awscrt-0.28.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:bec19c0dd780293a26c809aabb9f7675b28cb3a1bf05b4a5bc9f28d5ced75a81", size = 3380735, upload-time = "2025-10-14T19:05:16.58Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/6602a87aead1d413c7bd77d059b301745146635cda99ee2a61ec0d23691e/awscrt-0.28.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01f33076759ba6285f25ccc6016355607df2e715d0bab3a1ef2416b87a6c3ade", size = 3827084, upload-time = "2025-10-14T19:05:19.335Z" }, + { url = "https://files.pythonhosted.org/packages/d8/62/61fe39ae5950ad00e10dcbf6e4f4f344dc93957757160c0000390331a11b/awscrt-0.28.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b5c807b9972795ce54c05aea6918c60983c51d879ebbff7a67adb8b0d28a121", size = 4092678, upload-time = "2025-10-14T19:05:20.8Z" }, + { url = "https://files.pythonhosted.org/packages/25/7d/e38f18cfb203e8f09842c0e3f422992887ce285ecc3bf18816d559a13c80/awscrt-0.28.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf4ff9c8c6a233246320c2d41d939b6e25cdae97728d827186e4771a9edda688", size = 3749978, upload-time = "2025-10-14T19:05:22.16Z" }, + { url = "https://files.pythonhosted.org/packages/16/6f/e8a3c0daed8f7b60c76fc2721bd4e83580ddecace24e0cb0ebb99564f699/awscrt-0.28.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c738b83b66d1a8b43089556247fbe4adf2b73d610c7938d3bae1718a0fe8b1d", size = 3977237, upload-time = "2025-10-14T19:05:23.368Z" }, + { url = "https://files.pythonhosted.org/packages/92/3d/8400203f02dd924bcc8255703179b0c26efd03c84f838db6f026fcef9ba6/awscrt-0.28.2-cp310-cp310-win32.whl", hash = "sha256:23c30004c736a2f826a32c9720f1ccf71e8e4deb8535da5915d6073604853098", size = 3919413, upload-time = "2025-10-14T19:05:24.477Z" }, + { url = "https://files.pythonhosted.org/packages/c0/5e/b5ccf377880a70425b100f1e5f5ba516ff75e291585b3dc129239fbd1ec3/awscrt-0.28.2-cp310-cp310-win_amd64.whl", hash = "sha256:859ae8a195d51f15b631147d6792953a563bfe0a1cc7a75b6750977634de54b8", size = 4056024, upload-time = "2025-10-14T19:05:25.956Z" }, + { url = "https://files.pythonhosted.org/packages/ed/79/94e9f0ee7c60ec6233c7ad6293589c56d5145172e49eb5328eda37d3fdd1/awscrt-0.28.2-cp311-abi3-macosx_10_15_universal2.whl", hash = "sha256:025eab99b58586d8c95f8fafe1f4695ad477eda20d1207240ee4f8ee79742059", size = 3381061, upload-time = "2025-10-14T19:05:27.187Z" }, + { url = "https://files.pythonhosted.org/packages/2d/b8/0da80dd58682ddf3ec204e877d5891198654647c085e65b6b8eacd214edb/awscrt-0.28.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e5c18d035d6cd92228e1db2f043517c1bcf9e0f6430c0af60cc34257dcca092c", size = 3788011, upload-time = "2025-10-14T19:05:28.768Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d2/f51cf4364364399fe90d557e2fed14c1f114720191a5825898b1242bd607/awscrt-0.28.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c75f077e90d0220a49b75a9bca914e5aa1a3c8f28af6bce4d0332be0b98dd3cb", size = 4055226, upload-time = "2025-10-14T19:05:30.054Z" }, + { url = "https://files.pythonhosted.org/packages/41/47/0fde8738a8c76de278ce431d8468ef18aeaca424329decca9ad5092df812/awscrt-0.28.2-cp311-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1432c5c59a7e36b33eb2746cfbf30058f19ed43f2c117863897681f70bc246ba", size = 3692839, upload-time = "2025-10-14T19:05:31.471Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/cb3762f6b47fe503eea7f337eca7cfd044ab28bcc2452fbf298c6492ec8b/awscrt-0.28.2-cp311-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f96703c30b22ba1e43e1bb2fe996ac7af513bea411c54dbf09a3a1af329b9a76", size = 3918023, upload-time = "2025-10-14T19:05:33.162Z" }, + { url = "https://files.pythonhosted.org/packages/95/0a/0b609acd45dbb83c04c7ecb8c7c789f5c15bbdd422129360bde093bc4a99/awscrt-0.28.2-cp311-abi3-win32.whl", hash = "sha256:3e94f63497b454d30892d7a7ce917a451c6f33590964d3a475d93f93b20083b6", size = 3917048, upload-time = "2025-10-14T19:05:34.745Z" }, + { url = "https://files.pythonhosted.org/packages/d1/38/bf33abd6d09c8572f8e09488db2b0a60124767d7f5d6d9a33cf8b051b7af/awscrt-0.28.2-cp311-abi3-win_amd64.whl", hash = "sha256:3e094772b1f6fd0f8c5f7cf37655d0984739f99493f66f534979a2a7bb7fc9f6", size = 4052877, upload-time = "2025-10-14T19:05:36.01Z" }, + { url = "https://files.pythonhosted.org/packages/10/71/4be198e472d95702434cee1f9dd889c56e22bea8554b466fad754148fd24/awscrt-0.28.2-cp313-abi3-macosx_10_15_universal2.whl", hash = "sha256:5fda9e7d0eb800491fadebe2b6c2560ac2f5742b60f4106440dca4b49da7fb03", size = 3379585, upload-time = "2025-10-14T19:05:37.225Z" }, + { url = "https://files.pythonhosted.org/packages/43/09/77084249d07dca71352341ad3fbcfa75deaccf25bd65f9fdbb36ce1f978b/awscrt-0.28.2-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:994a795bdc83344922a15891abb30155ec292093e856eef3929dd63dd6cadaca", size = 3779843, upload-time = "2025-10-14T19:05:38.774Z" }, + { url = "https://files.pythonhosted.org/packages/a6/bb/fcee9365e58e5860582398317571a9a5517da258cd81c3d987b9882f61d4/awscrt-0.28.2-cp313-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28537c4517168927ef74aa007a2e0c9f436921227934d82da31e9a1cec7e0c4a", size = 4049154, upload-time = "2025-10-14T19:05:40.301Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8e/ac92b2707dbe05e56d0dd5af73cb4e07a3da4aee66936071123966523759/awscrt-0.28.2-cp313-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:b9fc6be63832da3ff244d56c7d9a43326d89d79e68162419c35f33e6ad033be0", size = 3683672, upload-time = "2025-10-14T19:05:41.536Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d0/15308ec37e762691f5d1871b0f1a6e462da8e421c6c38d6724e3cf0994b2/awscrt-0.28.2-cp313-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:efb57103a368de1d33148cb70a382c4f82ac376c744de9484e0f621cef8313f3", size = 3912823, upload-time = "2025-10-14T19:05:43.781Z" }, + { url = "https://files.pythonhosted.org/packages/bc/cd/7693b1d72069908b7a3ee30e4ef2b5fc8f54948a96397729277cb0b0c7b4/awscrt-0.28.2-cp313-abi3-win32.whl", hash = "sha256:594dc61f4f0c1c9fb7292364d25c21810b3608cd67c0de78a032ad48f7bfd88c", size = 3911514, upload-time = "2025-10-14T19:05:45.019Z" }, + { url = "https://files.pythonhosted.org/packages/93/d6/5d8545c967690f03d55d44ed56ceff26d88363cd7d0435fd80a1c843ac2a/awscrt-0.28.2-cp313-abi3-win_amd64.whl", hash = "sha256:a17f0ab9dc5e5301da0fb00ccc4511a136d13abbd4a9564827547333fcd7ba16", size = 4047912, upload-time = "2025-10-14T19:05:46.302Z" }, ] [[package]] @@ -4633,7 +4633,7 @@ requires-dist = [ { name = "aiortc", marker = "extra == 'webrtc'", specifier = ">=1.13.0,<2" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = "~=0.49.0" }, { name = "audioop-lts", marker = "python_full_version >= '3.13'", specifier = "~=0.2.1" }, - { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.1.0" }, + { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.1.1" }, { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, @@ -6483,16 +6483,16 @@ wheels = [ [[package]] name = "smithy-aws-core" -version = "0.1.0" +version = "0.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aws-sdk-signers", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/e8/8cef48be92ed09a112c54747a4515313ba96e767e7e0118a769aeb147e07/smithy_aws_core-0.1.0.tar.gz", hash = "sha256:5f197b69ad1380e9118e1e3c9032e0e305525ef56fb4fc97dea6414281065526", size = 11135, upload-time = "2025-09-29T19:37:13.072Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/d3/f847e0fd36b95aa36ce3a4c9ce1a08e16b2aa9a56b71714045c9c924e282/smithy_aws_core-0.1.1.tar.gz", hash = "sha256:78dfd7040fc2bc72b6af293096642fc9a7bfd2db28ddbdf7c4110535eab9d662", size = 11196, upload-time = "2025-10-21T20:21:18.648Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/7e/6d05275646bc2cdf7b0749e9bd54958a4e808aafeee4d8ff2fdaa8233dc2/smithy_aws_core-0.1.0-py3-none-any.whl", hash = "sha256:a8cda4011562f45f1fc5957c3a981b6016d736178450e5f2a1586937632af487", size = 18959, upload-time = "2025-09-29T19:37:12.041Z" }, + { url = "https://files.pythonhosted.org/packages/d0/04/87cb06f0f6d664b5cffdef6d4042dd52c11c138436084d30ffdaa3543031/smithy_aws_core-0.1.1-py3-none-any.whl", hash = "sha256:0d1634f276c2999dc2a04fafef63b9d28309de50d939d1d49df952773a7063c4", size = 18963, upload-time = "2025-10-21T20:21:17.692Z" }, ] [package.optional-dependencies] @@ -6526,14 +6526,14 @@ wheels = [ [[package]] name = "smithy-http" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/62/5ba46c7432fbb0852acf8340402879ba53bb4c009b875e1b5b2e9df844ff/smithy_http-0.1.0.tar.gz", hash = "sha256:ed44552531f594e31101f7186c7b01b508ecd38a860b45390a1cce7da700df4b", size = 28269, upload-time = "2025-09-29T19:37:18.629Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/1c/44e99a7dfb8c39bf0c3d998accdf4573a7a3488863b90f21af260cec2d45/smithy_http-0.2.0.tar.gz", hash = "sha256:2382562fa9af326be455f14b18615f16ffe9db756e51b2a4ca0d23e1b881cff8", size = 26729, upload-time = "2025-10-21T20:21:06.146Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/23/d18076ea45b3000c5e9eb8ebd75a4ea1b65b5c59e5c2080a119e2679dfba/smithy_http-0.1.0-py3-none-any.whl", hash = "sha256:7657aaf4b9e025cb9d317406f417b49cf19fba9d1b2ab4f5e6d9dc5a2dd7cdba", size = 38995, upload-time = "2025-09-29T19:37:17.506Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e2/d475fad81ac74ec0e145cb6d72afe5ecde4e2358bd632c2fd5d3f4bc87dc/smithy_http-0.2.0-py3-none-any.whl", hash = "sha256:49ee2402d7737798d70f99f491fbfb2a5767283ae562e21b6f86e3fd14f3e3e0", size = 37328, upload-time = "2025-10-21T20:21:05.362Z" }, ] [package.optional-dependencies] From f1040100f4a1e105c25d404e96bcc2a95abfe0dd Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 22 Oct 2025 08:58:14 -0400 Subject: [PATCH 31/93] Update ServiceSwitcher and LLMSwitcher docstrings --- src/pipecat/pipeline/llm_switcher.py | 29 +++++++++++++-- src/pipecat/pipeline/service_switcher.py | 47 +++++++++++++++++++++--- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/pipecat/pipeline/llm_switcher.py b/src/pipecat/pipeline/llm_switcher.py index a15e0a3c2..50d919263 100644 --- a/src/pipecat/pipeline/llm_switcher.py +++ b/src/pipecat/pipeline/llm_switcher.py @@ -14,20 +14,41 @@ from pipecat.services.llm_service import LLMService class LLMSwitcher(ServiceSwitcher[StrategyType]): - """A pipeline that switches between different LLMs at runtime.""" + """A pipeline that switches between different LLMs at runtime. + + Example:: + + llm_switcher = LLMSwitcher( + llms=[openai_llm, anthropic_llm], + strategy_type=ServiceSwitcherStrategyManual + ) + """ def __init__(self, llms: List[LLMService], strategy_type: Type[StrategyType]): - """Initialize the service switcher with a list of LLMs and a switching strategy.""" + """Initialize the service switcher with a list of LLMs and a switching strategy. + + Args: + llms: List of LLM services to switch between. + strategy_type: The strategy class to use for switching between LLMs. + """ super().__init__(llms, strategy_type) @property def llms(self) -> List[LLMService]: - """Get the list of LLMs managed by this switcher.""" + """Get the list of LLMs managed by this switcher. + + Returns: + List of LLM services managed by this switcher. + """ return self.services @property def active_llm(self) -> Optional[LLMService]: - """Get the currently active LLM, if any.""" + """Get the currently active LLM. + + Returns: + The currently active LLM service, or None if no LLM is active. + """ return self.strategy.active_service async def run_inference(self, context: LLMContext) -> Optional[str]: diff --git a/src/pipecat/pipeline/service_switcher.py b/src/pipecat/pipeline/service_switcher.py index eea55e68d..095f211ea 100644 --- a/src/pipecat/pipeline/service_switcher.py +++ b/src/pipecat/pipeline/service_switcher.py @@ -21,10 +21,22 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class ServiceSwitcherStrategy: - """Base class for service switching strategies.""" + """Base class for service switching strategies. + + Note: + Strategy classes are instantiated internally by ServiceSwitcher. + Developers should pass the strategy class (not an instance) to ServiceSwitcher. + """ def __init__(self, services: List[FrameProcessor]): - """Initialize the service switcher strategy with a list of services.""" + """Initialize the service switcher strategy with a list of services. + + Note: + This is called internally by ServiceSwitcher. Do not instantiate directly. + + Args: + services: List of frame processors to switch between. + """ self.services = services self.active_service: Optional[FrameProcessor] = None @@ -46,10 +58,24 @@ class ServiceSwitcherStrategyManual(ServiceSwitcherStrategy): This strategy allows the user to manually select which service is active. The initial active service is the first one in the list. + + Example:: + + stt_switcher = ServiceSwitcher( + services=[stt_1, stt_2], + strategy_type=ServiceSwitcherStrategyManual + ) """ def __init__(self, services: List[FrameProcessor]): - """Initialize the manual service switcher strategy with a list of services.""" + """Initialize the manual service switcher strategy with a list of services. + + Note: + This is called internally by ServiceSwitcher. Do not instantiate directly. + + Args: + services: List of frame processors to switch between. + """ super().__init__(services) self.active_service = services[0] if services else None @@ -85,7 +111,12 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): """A pipeline that switches between different services at runtime.""" def __init__(self, services: List[FrameProcessor], strategy_type: Type[StrategyType]): - """Initialize the service switcher with a list of services and a switching strategy.""" + """Initialize the service switcher with a list of services and a switching strategy. + + Args: + services: List of frame processors to switch between. + strategy_type: The strategy class to use for switching between services. + """ strategy = strategy_type(services) super().__init__(*self._make_pipeline_definitions(services, strategy)) self.services = services @@ -100,7 +131,13 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): active_service: FrameProcessor, direction: FrameDirection, ): - """Initialize the service switcher filter with a strategy and direction.""" + """Initialize the service switcher filter with a strategy and direction. + + Args: + wrapped_service: The service that this filter wraps. + active_service: The currently active service. + direction: The direction of frame flow to filter. + """ async def filter(_: Frame) -> bool: return self._wrapped_service == self._active_service From 5b921fc054f9feb8a08cf38e9ee8c74ae9af0be3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 22 Oct 2025 10:48:48 -0400 Subject: [PATCH 32/93] fix: FunctionFilter adds block_system_frame arg --- CHANGELOG.md | 8 + examples/foundational/48-service-switcher.py | 138 ++++++++++++++++++ src/pipecat/pipeline/service_switcher.py | 6 +- .../processors/filters/function_filter.py | 21 ++- 4 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 examples/foundational/48-service-switcher.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e727c42e..a07832525 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `FunctionFilter` now has a `block_system_frames` arg, which controls whether + or not SystemFrames are filtered. + - Upgraded `aws_sdk_bedrock_runtime` to v0.1.1 to resolve potential CPU issues when running `AWSNovaSonicLLMService`. +### Fixed + +- Fixed an issue in `ServiceSwitcher` where the `STTService`s would result in + all STT services producing `TranscriptionFrame`s. + ## [0.0.91] - 2025-10-21 ### Added diff --git a/examples/foundational/48-service-switcher.py b/examples/foundational/48-service-switcher.py new file mode 100644 index 000000000..e003db7eb --- /dev/null +++ b/examples/foundational/48-service-switcher.py @@ -0,0 +1,138 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, ManuallySwitchServiceFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.service_switcher import ServiceSwitcher, ServiceSwitcherStrategyManual +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.stt import CartesiaSTTService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.stt_service import STTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt_cartesia = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt_deepgram = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt_switcher = ServiceSwitcher( + services=[stt_cartesia, stt_deepgram], strategy_type=ServiceSwitcherStrategyManual + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt_switcher, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + await asyncio.sleep(15) + print(f"Switching to {stt_deepgram}") + await task.queue_frames([ManuallySwitchServiceFrame(service=stt_deepgram)]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/src/pipecat/pipeline/service_switcher.py b/src/pipecat/pipeline/service_switcher.py index 095f211ea..3cfcd0fd8 100644 --- a/src/pipecat/pipeline/service_switcher.py +++ b/src/pipecat/pipeline/service_switcher.py @@ -138,13 +138,13 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): active_service: The currently active service. direction: The direction of frame flow to filter. """ + self._wrapped_service = wrapped_service + self._active_service = active_service async def filter(_: Frame) -> bool: return self._wrapped_service == self._active_service - super().__init__(filter, direction) - self._wrapped_service = wrapped_service - self._active_service = active_service + super().__init__(filter, direction, block_system_frames=True) async def process_frame(self, frame, direction): """Process a frame through the filter, handling special internal filter-updating frames.""" diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index e663b81f4..5bc6e1eb9 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -12,7 +12,7 @@ allowing for flexible frame filtering logic in processing pipelines. from typing import Awaitable, Callable -from pipecat.frames.frames import EndFrame, Frame, SystemFrame +from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -28,6 +28,7 @@ class FunctionFilter(FrameProcessor): self, filter: Callable[[Frame], Awaitable[bool]], direction: FrameDirection = FrameDirection.DOWNSTREAM, + block_system_frames: bool = False, ): """Initialize the function filter. @@ -36,10 +37,12 @@ class FunctionFilter(FrameProcessor): frame should pass through, False otherwise. direction: The direction to apply filtering. Only frames moving in this direction will be filtered. Defaults to DOWNSTREAM. + block_system_frames: Whether to block system frames. Defaults to False. """ super().__init__() self._filter = filter self._direction = direction + self._block_system_frames = block_system_frames # # Frame processor @@ -49,9 +52,19 @@ class FunctionFilter(FrameProcessor): # direction of this gate def _should_passthrough_frame(self, frame, direction): """Check if a frame should pass through without filtering.""" - # Ignore system frames, end frames and frames that are not following the - # direction of this gate - return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction + # Always passthrough frames in the wrong direction + if direction != self._direction: + return True + + # Always passthrough lifecycle frames + if isinstance(frame, (StartFrame, EndFrame, CancelFrame)): + return True + + # If not blocking system frames, passthrough all other system frames + if not self._block_system_frames and isinstance(frame, SystemFrame): + return True + + return False async def process_frame(self, frame: Frame, direction: FrameDirection): """Process a frame through the filter. From ec890a834fa63917ac277529ab9b5f7f3530acef Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 22 Oct 2025 10:59:43 -0400 Subject: [PATCH 33/93] Rename to filter_system_frames --- CHANGELOG.md | 2 +- examples/foundational/48-service-switcher.py | 27 ++++++++++++++----- src/pipecat/pipeline/service_switcher.py | 2 +- .../processors/filters/function_filter.py | 12 ++++----- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a07832525..3cbf4a57e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- `FunctionFilter` now has a `block_system_frames` arg, which controls whether +- `FunctionFilter` now has a `filter_system_frames` arg, which controls whether or not SystemFrames are filtered. - Upgraded `aws_sdk_bedrock_runtime` to v0.1.1 to resolve potential CPU issues diff --git a/examples/foundational/48-service-switcher.py b/examples/foundational/48-service-switcher.py index e003db7eb..d0e15d2d3 100644 --- a/examples/foundational/48-service-switcher.py +++ b/examples/foundational/48-service-switcher.py @@ -26,8 +26,9 @@ from pipecat.runner.utils import create_transport from pipecat.services.cartesia.stt import CartesiaSTTService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepgram.tts import DeepgramTTSService +from pipecat.services.google.llm import GoogleLLMService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.services.stt_service import STTService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -68,12 +69,20 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): services=[stt_cartesia, stt_deepgram], strategy_type=ServiceSwitcherStrategyManual ) - tts = CartesiaTTSService( + tts_cartesia = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", + ) + tts_deepgram = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts_switcher = ServiceSwitcher( + services=[tts_cartesia, tts_deepgram], strategy_type=ServiceSwitcherStrategyManual ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm_openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm_google = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + llm_switcher = ServiceSwitcher( + services=[llm_openai, llm_google], strategy_type=ServiceSwitcherStrategyManual + ) messages = [ { @@ -90,8 +99,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): transport.input(), # Transport user input stt_switcher, context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS + llm_switcher, # LLM + tts_switcher, # TTS transport.output(), # Transport bot output context_aggregator.assistant(), # Assistant spoken responses ] @@ -115,6 +124,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): await asyncio.sleep(15) print(f"Switching to {stt_deepgram}") await task.queue_frames([ManuallySwitchServiceFrame(service=stt_deepgram)]) + await asyncio.sleep(15) + print(f"Switching to {llm_google}") + await task.queue_frames([ManuallySwitchServiceFrame(service=llm_google)]) + await asyncio.sleep(15) + print(f"Switching to {tts_deepgram}") + await task.queue_frames([ManuallySwitchServiceFrame(service=tts_deepgram)]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/src/pipecat/pipeline/service_switcher.py b/src/pipecat/pipeline/service_switcher.py index 3cfcd0fd8..8895d663c 100644 --- a/src/pipecat/pipeline/service_switcher.py +++ b/src/pipecat/pipeline/service_switcher.py @@ -144,7 +144,7 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): async def filter(_: Frame) -> bool: return self._wrapped_service == self._active_service - super().__init__(filter, direction, block_system_frames=True) + super().__init__(filter, direction, filter_system_frames=True) async def process_frame(self, frame, direction): """Process a frame through the filter, handling special internal filter-updating frames.""" diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index 5bc6e1eb9..556f2bc87 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -28,7 +28,7 @@ class FunctionFilter(FrameProcessor): self, filter: Callable[[Frame], Awaitable[bool]], direction: FrameDirection = FrameDirection.DOWNSTREAM, - block_system_frames: bool = False, + filter_system_frames: bool = False, ): """Initialize the function filter. @@ -37,19 +37,17 @@ class FunctionFilter(FrameProcessor): frame should pass through, False otherwise. direction: The direction to apply filtering. Only frames moving in this direction will be filtered. Defaults to DOWNSTREAM. - block_system_frames: Whether to block system frames. Defaults to False. + filter_system_frames: Whether to filter system frames. Defaults to False. """ super().__init__() self._filter = filter self._direction = direction - self._block_system_frames = block_system_frames + self._filter_system_frames = filter_system_frames # # Frame processor # - # Ignore system frames, end frames and frames that are not following the - # direction of this gate def _should_passthrough_frame(self, frame, direction): """Check if a frame should pass through without filtering.""" # Always passthrough frames in the wrong direction @@ -60,8 +58,8 @@ class FunctionFilter(FrameProcessor): if isinstance(frame, (StartFrame, EndFrame, CancelFrame)): return True - # If not blocking system frames, passthrough all other system frames - if not self._block_system_frames and isinstance(frame, SystemFrame): + # If not filtering system frames, passthrough all other system frames + if not self._filter_system_frames and isinstance(frame, SystemFrame): return True return False From ea6e146f2d29dad4584ea5d959cc14e9ed71870f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 22 Oct 2025 11:14:27 -0400 Subject: [PATCH 34/93] Update `TestServiceSwitcher` to exercise targeting system frames only to the active service --- tests/test_service_switcher.py | 41 ++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/test_service_switcher.py b/tests/test_service_switcher.py index bf80d842e..83d2d226b 100644 --- a/tests/test_service_switcher.py +++ b/tests/test_service_switcher.py @@ -7,10 +7,12 @@ """Unit tests for ServiceSwitcher and related components.""" import unittest +from dataclasses import dataclass from pipecat.frames.frames import ( Frame, ManuallySwitchServiceFrame, + SystemFrame, TextFrame, ) from pipecat.pipeline.pipeline import Pipeline @@ -52,6 +54,13 @@ class MockFrameProcessor(FrameProcessor): self.frame_count = 0 +@dataclass +class DummySystemFrame(SystemFrame): + """A dummy system frame for testing purposes.""" + + text: str = "" + + class TestServiceSwitcherStrategyManual(unittest.IsolatedAsyncioTestCase): """Test cases for ServiceSwitcherStrategyManual.""" @@ -140,14 +149,22 @@ class TestServiceSwitcher(unittest.IsolatedAsyncioTestCase): # Send some test frames frames_to_send = [ TextFrame(text="Hello 1"), + DummySystemFrame(text="System Message 1"), TextFrame(text="Hello 2"), + DummySystemFrame(text="System Message 2"), TextFrame(text="Hello 3"), ] await run_test( switcher, frames_to_send=frames_to_send, - expected_down_frames=[TextFrame, TextFrame, TextFrame], + expected_down_frames=[ + DummySystemFrame, + DummySystemFrame, + TextFrame, + TextFrame, + TextFrame, + ], expected_up_frames=[], # Expect no error frames ) @@ -156,7 +173,13 @@ class TestServiceSwitcher(unittest.IsolatedAsyncioTestCase): text_frames = [f for f in self.service1.processed_frames if isinstance(f, TextFrame)] self.assertEqual(len(text_frames), 3) - # Check that other services don't receive text frames (they might get StartFrame/EndFrame) + # Only service1 should have processed the system frames + system_frames = [ + f for f in self.service1.processed_frames if isinstance(f, DummySystemFrame) + ] + self.assertEqual(len(system_frames), 2) + + # Check that other services don't receive text frames (they still get StartFrame/EndFrame) service2_text_frames = [ f for f in self.service2.processed_frames if isinstance(f, TextFrame) ] @@ -166,10 +189,24 @@ class TestServiceSwitcher(unittest.IsolatedAsyncioTestCase): self.assertEqual(len(service2_text_frames), 0) self.assertEqual(len(service3_text_frames), 0) + # Check that other services don't receive dummy system frames (they still get StartFrame/EndFrame) + service2_system_frames = [ + f for f in self.service2.processed_frames if isinstance(f, DummySystemFrame) + ] + service3_system_frames = [ + f for f in self.service3.processed_frames if isinstance(f, DummySystemFrame) + ] + self.assertEqual(len(service2_system_frames), 0) + self.assertEqual(len(service3_system_frames), 0) + # Verify the actual text frames processed for i, frame in enumerate(text_frames): self.assertEqual(frame.text, f"Hello {i + 1}") + # Verify the actual system frames processed + for i, frame in enumerate(system_frames): + self.assertEqual(frame.text, f"System Message {i + 1}") + async def test_service_switching(self): """Test that after service switching using ManuallySwitchServiceFrame, the new active service receives frames while others don't.""" switcher = ServiceSwitcher(self.services, ServiceSwitcherStrategyManual) From 60e9817f166182c0cf891b1c527c4bd21256b7b2 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 22 Oct 2025 16:48:30 -0400 Subject: [PATCH 35/93] fix: add support for DAILY_SAMPLE_ROOM_URL when calling /start for DailyTransport --- CHANGELOG.md | 5 +++++ src/pipecat/runner/run.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e727c42e..397d47123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Upgraded `aws_sdk_bedrock_runtime` to v0.1.1 to resolve potential CPU issues when running `AWSNovaSonicLLMService`. +### Fixed + +- Fixed an issue in the runner where starting a DailyTransport room via + `/start` didn't support using the `DAILY_SAMPLE_ROOM_URL` env var. + ## [0.0.91] - 2025-10-21 ### Added diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index 4c1d3135d..b5a805c1f 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -573,8 +573,14 @@ def _setup_daily_routes(app: FastAPI): bot_module = _get_bot_module() + existing_room_url = os.getenv("DAILY_SAMPLE_ROOM_URL") + result = None - if create_daily_room: + + # Configure room if: + # 1. Explicitly requested via createDailyRoom in payload + # 2. Using pre-configured room from DAILY_SAMPLE_ROOM_URL env var + if create_daily_room or existing_room_url: import aiohttp from pipecat.runner.daily import configure From 99d94fc62506b67a256826d8ce756ace30ad3114 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 24 Oct 2025 10:04:02 -0400 Subject: [PATCH 36/93] Update Gemini service to use "user" role for function responses, as shown in the Gemini docs --- src/pipecat/adapters/services/gemini_adapter.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index abe33a8ec..53e4e4ee0 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -242,7 +242,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Converts to Google Content with:: Content( - role="model", + role="user", parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))] ) """ @@ -273,7 +273,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): ) ) elif role == "tool": - role = "model" + role = "user" try: response = json.loads(message["content"]) if isinstance(response, dict): @@ -285,11 +285,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string. response_dict = {"value": message["content"]} parts.append( - Part( - function_response=FunctionResponse( - name="tool_call_result", # seems to work to hard-code the same name every time - response=response_dict, - ) + Part.from_function_response( + name="tool_call_result", # seems to work to hard-code the same name every time + response=response_dict, ) ) elif isinstance(content, str): From 4ad15f9a01e43cece28f11be08e683b9e482d48d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 24 Oct 2025 10:38:17 -0400 Subject: [PATCH 37/93] Update Gemini service to include function name when sending function responses in context --- .../adapters/services/anthropic_adapter.py | 2 +- .../adapters/services/bedrock_adapter.py | 2 +- .../adapters/services/gemini_adapter.py | 106 +++++++++++++----- 3 files changed, 81 insertions(+), 29 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index adfe81005..a106b4de4 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -110,7 +110,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): system = NOT_GIVEN messages = [] - # first, map messages using self._from_universal_context_message(m) + # First, map messages using self._from_universal_context_message(m) try: messages = [self._from_universal_context_message(m) for m in universal_context_messages] except Exception as e: diff --git a/src/pipecat/adapters/services/bedrock_adapter.py b/src/pipecat/adapters/services/bedrock_adapter.py index 681dfb3dc..852ea17a4 100644 --- a/src/pipecat/adapters/services/bedrock_adapter.py +++ b/src/pipecat/adapters/services/bedrock_adapter.py @@ -107,7 +107,7 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): system = None messages = [] - # first, map messages using self._from_universal_context_message(m) + # First, map messages using self._from_universal_context_message(m) try: messages = [self._from_universal_context_message(m) for m in universal_context_messages] except Exception as e: diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 53e4e4ee0..75577f160 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -8,8 +8,8 @@ import base64 import json -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, TypedDict +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, TypedDict from loguru import logger from openai import NotGiven @@ -133,6 +133,28 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): messages: List[Content] system_instruction: Optional[str] = None + @dataclass + class MessageConversionResult: + """Result of converting a single universal context message to Google format. + + Either content (a Google Content object) or a system instruction string + is guaranteed to be set. + + Also returns a tool call ID to name mapping for any tool calls + discovered in the message. + """ + + content: Optional[Content] = None + system_instruction: Optional[str] = None + tool_call_id_to_name_mapping: Dict[str, str] = field(default_factory=dict) + + @dataclass + class MessageConversionParams: + """Parameters for converting a single universal context message to Google format.""" + + already_have_system_instruction: bool + tool_call_id_to_name_mapping: Dict[str, str] + def _from_universal_context_messages( self, universal_context_messages: List[LLMContextMessage] ) -> ConvertedMessages: @@ -156,24 +178,26 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): """ system_instruction = None messages = [] + tool_call_id_to_name_mapping = {} # Process each message, preserving Google-formatted messages and converting others for message in universal_context_messages: - if isinstance(message, LLMSpecificMessage): - # Assume that LLMSpecificMessage wraps a message in Google format - messages.append(message.message) - continue - - # Convert standard format to Google format - converted = self._from_standard_message( - message, already_have_system_instruction=bool(system_instruction) + result = self._from_universal_context_message( + message, + params=self.MessageConversionParams( + already_have_system_instruction=bool(system_instruction), + tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, + ), ) - if isinstance(converted, Content): - # Regular (non-system) message - messages.append(converted) - else: - # System instruction - system_instruction = converted + # Each result is either a Content or a system instruction + if result.content: + messages.append(result.content) + elif result.system_instruction: + system_instruction = result.system_instruction + + # Merge tool call ID to name mapping + if result.tool_call_id_to_name_mapping: + tool_call_id_to_name_mapping.update(result.tool_call_id_to_name_mapping) # Check if we only have function-related messages (no regular text) has_regular_messages = any( @@ -193,9 +217,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): return self.ConvertedMessages(messages=messages, system_instruction=system_instruction) + def _from_universal_context_message( + self, message: LLMContextMessage, *, params: MessageConversionParams + ) -> MessageConversionResult: + if isinstance(message, LLMSpecificMessage): + return message.message + return self._from_standard_message(message, params=params) + def _from_standard_message( - self, message: LLMStandardMessage, already_have_system_instruction: bool - ) -> Content | str: + self, message: LLMStandardMessage, *, params: MessageConversionParams + ) -> MessageConversionResult: """Convert standard universal context message to Google Content object. Handles conversion of text, images, and function calls to Google's @@ -205,10 +236,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Args: message: Message in standard universal context format. already_have_system_instruction: Whether we already have a system instruction + params: Parameters for conversion. Returns: - Content object with role and parts, or a plain string for system - messages. + MessageConversionResult containing either a Content object or a + system instruction string. Examples: Standard text message:: @@ -248,26 +280,36 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): """ role = message["role"] content = message.get("content", []) + if role == "system": - if already_have_system_instruction: + if params.already_have_system_instruction: role = "user" # Convert system message to user role if we already have a system instruction else: - # System instructions are returned as plain text + system_instruction: str = None if isinstance(content, str): - return content + system_instruction = content elif isinstance(content, list): # If content is a list, we assume it's a list of text parts, per the standard - return " ".join(part["text"] for part in content if part.get("type") == "text") + system_instruction = " ".join( + part["text"] for part in content if part.get("type") == "text" + ) + if system_instruction: + return self.MessageConversionResult(system_instruction=system_instruction) elif role == "assistant": role = "model" parts = [] + tool_call_id_to_name_mapping = {} + if message.get("tool_calls"): for tc in message["tool_calls"]: + id = tc["id"] + name = tc["function"]["name"] + tool_call_id_to_name_mapping[id] = name parts.append( Part( function_call=FunctionCall( - name=tc["function"]["name"], + name=name, args=json.loads(tc["function"]["arguments"]), ) ) @@ -284,9 +326,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # Response might not be JSON-deserializable. # This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string. response_dict = {"value": message["content"]} + + # Get function name from mapping using tool_call_id, or fallback + tool_call_id = message.get("tool_call_id") + function_name = "tool_call_result" # Default fallback + if tool_call_id and tool_call_id in params.tool_call_id_to_name_mapping: + function_name = params.tool_call_id_to_name_mapping[tool_call_id] + parts.append( Part.from_function_response( - name="tool_call_result", # seems to work to hard-code the same name every time + name=function_name, response=response_dict, ) ) @@ -310,4 +359,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): audio_bytes = base64.b64decode(input_audio["data"]) parts.append(Part(inline_data=Blob(mime_type="audio/wav", data=audio_bytes))) - return Content(role=role, parts=parts) + return self.MessageConversionResult( + content=Content(role=role, parts=parts), + tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, + ) From 949b8070230e496db1509c7b3a31939977de94e2 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 24 Oct 2025 11:36:25 -0400 Subject: [PATCH 38/93] Close genai client more gracefully to avoid printed warnings. We're now following the genai library guidance: https://github.com/googleapis/python-genai?tab=readme-ov-file#close-a-client --- src/pipecat/services/google/llm.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index b45c276d0..c59cb41ae 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -1034,6 +1034,23 @@ class GoogleLLMService(LLMService): if context: await self._process_context(context) + async def stop(self, frame): + """Override stop to gracefully close the client.""" + await super().stop(frame) + await self._close_client() + + async def cancel(self, frame): + """Override cancel to gracefully close the client.""" + await super().cancel(frame) + await self._close_client() + + async def _close_client(self): + try: + await self._client.aio.aclose() + except Exception: + # Do nothing - we're shutting down anyway + pass + def create_context_aggregator( self, context: OpenAILLMContext, From 71841f71ef243aea2643cc0cd69b4cc273c31072 Mon Sep 17 00:00:00 2001 From: Pyae Sone Myo <66440980+Rickaym@users.noreply.github.com> Date: Sat, 25 Oct 2025 00:47:46 +0630 Subject: [PATCH 39/93] fix: use correct property names --- src/pipecat/pipeline/task_observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index 7bf83480d..98ff7c91e 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -189,7 +189,7 @@ class TaskObserver(BaseObserver): if isinstance(data, FramePushed): if on_push_frame_deprecated: await observer.on_push_frame( - data.src, data.dst, data.frame, data.direction, data.timestamp + data.source, data.destination, data.frame, data.direction, data.timestamp ) else: await observer.on_push_frame(data) From 6feaf9178920f9fddef5961ab2b431101e7d41de Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Oct 2025 09:42:24 -0400 Subject: [PATCH 40/93] Fix a bug in `GeminiLLMAdapter`'s handling of Gemini-specific context messages --- src/pipecat/adapters/services/gemini_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 75577f160..26f037127 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -221,7 +221,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): self, message: LLMContextMessage, *, params: MessageConversionParams ) -> MessageConversionResult: if isinstance(message, LLMSpecificMessage): - return message.message + return self.MessageConversionResult(content=message.message) return self._from_standard_message(message, params=params) def _from_standard_message( From 8096e62b347a28463e146880ae53601563651b75 Mon Sep 17 00:00:00 2001 From: vipyne Date: Mon, 27 Oct 2025 11:27:30 -0500 Subject: [PATCH 41/93] cleanup logger message --- src/pipecat/services/azure/realtime/llm.py | 2 +- src/pipecat/services/openai_realtime_beta/azure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/azure/realtime/llm.py b/src/pipecat/services/azure/realtime/llm.py index fb420e10b..1193b82d4 100644 --- a/src/pipecat/services/azure/realtime/llm.py +++ b/src/pipecat/services/azure/realtime/llm.py @@ -52,7 +52,7 @@ class AzureRealtimeLLMService(OpenAIRealtimeLLMService): # handle disconnections in the send/recv code paths. return - logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") + logger.info(f"Connecting to {self.base_url}") self._websocket = await websocket_connect( uri=self.base_url, additional_headers={ diff --git a/src/pipecat/services/openai_realtime_beta/azure.py b/src/pipecat/services/openai_realtime_beta/azure.py index 784438e81..2ec556ee3 100644 --- a/src/pipecat/services/openai_realtime_beta/azure.py +++ b/src/pipecat/services/openai_realtime_beta/azure.py @@ -70,7 +70,7 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): # handle disconnections in the send/recv code paths. return - logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") + logger.info(f"Connecting to {self.base_url}") self._websocket = await websocket_connect( uri=self.base_url, additional_headers={ From f2cfbee3c3bdc861e1e4c79a71e53fc16dd2496d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 27 Oct 2025 16:18:31 -0400 Subject: [PATCH 42/93] Remove development runner requirement for proxy --- src/pipecat/runner/run.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index b5a805c1f..28ca81bb9 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -794,10 +794,6 @@ def main(): logger.error("For ESP32, you need to specify `--host IP` so we can do SDP munging.") return - if args.transport in TELEPHONY_TRANSPORTS and not args.proxy: - logger.error(f"For telephony transports, you need to specify `--proxy PROXY`.") - return - # Log level logger.remove() logger.add(sys.stderr, level="TRACE" if args.verbose else "DEBUG") From f3c4bf08ddae22b3f46d666f357f4a737002aa8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Oct 2025 17:52:19 -0700 Subject: [PATCH 43/93] DailyTransport: don't timeout prematurely on join --- CHANGELOG.md | 2 ++ src/pipecat/transports/daily/transport.py | 43 ++++++++++------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd1091e87..a5813a5b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue where `DailyTransport` would timeout prematurely on join. + - Fixed an issue in the runner where starting a DailyTransport room via `/start` didn't support using the `DAILY_SAMPLE_ROOM_URL` env var. diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 7f6b21ee2..675db37e5 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -744,32 +744,27 @@ class DailyTransportClient(EventHandler): self._client.set_user_name(self._bot_name) - try: - (data, error) = await self._join() + (data, error) = await self._join() - if not error: - self._joined = True - self._joining = False - # Increment leave counter if we successfully joined. - self._leave_counter += 1 - - logger.info(f"Joined {self._room_url}") - - if self._params.transcription_enabled: - await self.start_transcription(self._params.transcription_settings) - - await self._callbacks.on_joined(data) - - self._joined_event.set() - else: - error_msg = f"Error joining {self._room_url}: {error}" - logger.error(error_msg) - await self._callbacks.on_error(error_msg) - except asyncio.TimeoutError: - error_msg = f"Time out joining {self._room_url}" - logger.error(error_msg) + if not error: + self._joined = True self._joining = False + # Increment leave counter if we successfully joined. + self._leave_counter += 1 + + logger.info(f"Joined {self._room_url}") + + if self._params.transcription_enabled: + await self.start_transcription(self._params.transcription_settings) + + await self._callbacks.on_joined(data) + + self._joined_event.set() + else: + error_msg = f"Error joining {self._room_url}: {error}" + logger.error(error_msg) await self._callbacks.on_error(error_msg) + self._joining = False async def _join(self): """Execute the actual room join operation.""" @@ -828,7 +823,7 @@ class DailyTransportClient(EventHandler): }, ) - return await asyncio.wait_for(future, timeout=10) + return await future async def leave(self): """Leave the Daily room and cleanup resources.""" From 9ef60bd468f6e7b55c1b45ba0d5c5373be378402 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 28 Oct 2025 11:49:54 -0400 Subject: [PATCH 44/93] Update Cartesia's default model to sonic-3 --- CHANGELOG.md | 3 +++ src/pipecat/services/cartesia/tts.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd1091e87..a8d076302 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated the default model to `sonic-3` for `CartesiaTTSService` and + `CartesiaHttpTTSService`. + - `FunctionFilter` now has a `filter_system_frames` arg, which controls whether or not SystemFrames are filtered. diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 90f0ac3b4..3c0fe279c 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -119,7 +119,7 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_id: str, cartesia_version: str = "2025-04-16", url: str = "wss://api.cartesia.ai/tts/websocket", - model: str = "sonic-2", + model: str = "sonic-3", sample_rate: Optional[int] = None, encoding: str = "pcm_s16le", container: str = "raw", @@ -135,7 +135,7 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_id: ID of the voice to use for synthesis. cartesia_version: API version string for Cartesia service. url: WebSocket URL for Cartesia TTS API. - model: TTS model to use (e.g., "sonic-2"). + model: TTS model to use (e.g., "sonic-3"). sample_rate: Audio sample rate. If None, uses default. encoding: Audio encoding format. container: Audio container format. @@ -498,7 +498,7 @@ class CartesiaHttpTTSService(TTSService): *, api_key: str, voice_id: str, - model: str = "sonic-2", + model: str = "sonic-3", base_url: str = "https://api.cartesia.ai", cartesia_version: str = "2024-11-13", sample_rate: Optional[int] = None, @@ -512,7 +512,7 @@ class CartesiaHttpTTSService(TTSService): Args: api_key: Cartesia API key for authentication. voice_id: ID of the voice to use for synthesis. - model: TTS model to use (e.g., "sonic-2"). + model: TTS model to use (e.g., "sonic-3"). base_url: Base URL for Cartesia HTTP API. cartesia_version: API version string for Cartesia service. sample_rate: Audio sample rate. If None, uses default. From d82d855c206ca93136ad4772fa4db31b220d9303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Oct 2025 18:24:19 -0700 Subject: [PATCH 45/93] DailyTransport: don't timeout prematurely on leave --- CHANGELOG.md | 3 ++- src/pipecat/transports/daily/transport.py | 19 +++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5813a5b0..3e0486d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed an issue where `DailyTransport` would timeout prematurely on join. +- Fixed an issue where `DailyTransport` would timeout prematurely on join and on + leave. - Fixed an issue in the runner where starting a DailyTransport room via `/start` didn't support using the `DAILY_SAMPLE_ROOM_URL` env var. diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 675db37e5..2e48c4d2e 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -849,17 +849,12 @@ class DailyTransportClient(EventHandler): for track_name, _ in self._custom_audio_tracks.items(): await self.remove_custom_audio_track(track_name) - try: - error = await self._leave() - if not error: - logger.info(f"Left {self._room_url}") - await self._callbacks.on_left() - else: - error_msg = f"Error leaving {self._room_url}: {error}" - logger.error(error_msg) - await self._callbacks.on_error(error_msg) - except asyncio.TimeoutError: - error_msg = f"Time out leaving {self._room_url}" + error = await self._leave() + if not error: + logger.info(f"Left {self._room_url}") + await self._callbacks.on_left() + else: + error_msg = f"Error leaving {self._room_url}: {error}" logger.error(error_msg) await self._callbacks.on_error(error_msg) @@ -870,7 +865,7 @@ class DailyTransportClient(EventHandler): future = self._get_event_loop().create_future() self._client.leave(completion=completion_callback(future)) - return await asyncio.wait_for(future, timeout=10) + return await future def _cleanup(self): """Cleanup the Daily client instance.""" From a9118eb2cde012630f9fdf0ed02db0dbedbfb3fe Mon Sep 17 00:00:00 2001 From: ivaaan Date: Tue, 28 Oct 2025 20:36:34 +0100 Subject: [PATCH 46/93] use Octave 1 if description provided --- src/pipecat/services/hume/tts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 0629a8909..c995f426d 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -184,11 +184,15 @@ class HumeTTSService(TTSService): # Hume emits mono PCM at 48 kHz; downstream can resample if needed. # We buffer audio bytes before sending to prevent glitches. self._audio_bytes = b"" + + # Use version "2" by default if no description is provided + # Version "1" is needed when description is used + version = "1" if self._params.description is not None else "2" async for chunk in self._client.tts.synthesize_json_streaming( utterances=[utterance], format=pcm_fmt, instant_mode=True, - version="2", + version=version, ): audio_b64 = getattr(chunk, "audio", None) if not audio_b64: From 533372ed3740fbf747c6920f23b31ee3fb98ec34 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 28 Oct 2025 15:39:14 -0400 Subject: [PATCH 47/93] Remove aiohttp_session arg from SarvamTTSService --- CHANGELOG.md | 5 +++++ src/pipecat/services/sarvam/tts.py | 15 --------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce1952c3f..c76d57d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Upgraded `aws_sdk_bedrock_runtime` to v0.1.1 to resolve potential CPU issues when running `AWSNovaSonicLLMService`. +### Removed + +- Removed the `aiohttp_session` arg from `SarvamTTSService` as it's no longer + used. + ### Fixed - Fixed an issue where `DailyTransport` would timeout prematurely on join and on diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 762776d50..7096683eb 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -374,7 +374,6 @@ class SarvamTTSService(InterruptibleTTSService): model: str = "bulbul:v2", voice_id: str = "anushka", url: str = "wss://api.sarvam.ai/text-to-speech/ws", - aiohttp_session: Optional[aiohttp.ClientSession] = None, aggregate_sentences: Optional[bool] = True, sample_rate: Optional[int] = None, params: Optional[InputParams] = None, @@ -388,11 +387,6 @@ class SarvamTTSService(InterruptibleTTSService): Supports "bulbul:v2", "bulbul:v3-beta" and "bulbul:v3". voice_id: Voice identifier for synthesis (default "anushka"). url: WebSocket URL for connecting to the TTS backend (default production URL). - aiohttp_session: Optional shared aiohttp session. To maintain backward compatibility. - - .. deprecated:: 0.0.81 - aiohttp_session is no longer used. This parameter will be removed in a future version. - aggregate_sentences: Whether to merge multiple sentences into one audio chunk (default True). sample_rate: Desired sample rate for the output audio in Hz (overrides default if set). params: Optional input parameters to override global configuration. @@ -413,16 +407,7 @@ class SarvamTTSService(InterruptibleTTSService): **kwargs, ) params = params or SarvamTTSService.InputParams() - if aiohttp_session is not None: - import warnings - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "The 'aiohttp_session' parameter is deprecated and will be removed in a future version. ", - DeprecationWarning, - stacklevel=2, - ) # WebSocket endpoint URL self._websocket_url = f"{url}?model={model}" self._api_key = api_key From f974c66e12070dcc3df5eabe7fe8ec5d393497e6 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Oct 2025 16:36:50 -0400 Subject: [PATCH 48/93] Update `GeminiLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` --- CHANGELOG.md | 53 +++++ .../26a-gemini-live-transcription.py | 13 +- .../26b-gemini-live-function-calling.py | 13 +- .../foundational/26c-gemini-live-video.py | 13 +- examples/foundational/26d-gemini-live-text.py | 7 +- .../26e-gemini-live-google-search.py | 13 +- .../foundational/26f-gemini-live-files-api.py | 15 +- .../26g-gemini-live-groundingMetadata.py | 13 +- ...26h-gemini-live-vertex-function-calling.py | 17 +- .../26i-gemini-live-graceful-end.py | 13 +- examples/foundational/46-video-processing.py | 13 +- .../adapters/services/gemini_adapter.py | 28 ++- .../services/google/gemini_live/llm.py | 204 ++++++++++++++---- 13 files changed, 328 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd1091e87..518791bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,59 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Expanded support for universal `LLMContext` to `GeminiLiveLLMService`. + As a reminder, the context-setup pattern when using `LLMContext` is: + + ```python + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair( + context, + # This part is `GeminiLiveLLMService`-specific. + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default). + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) + ``` + + (Note that even though `GeminiLiveLLMService` now supports the universal + `LLMContext`, it is not meant to be swapped out for another LLM service at + runtime.) + + Worth noting: whether or not you use the new context-setup pattern with + `GeminiLiveLLMService`, some types have changed under the hood: + + ```python + ## BEFORE: + + # Context aggregator type + context_aggregator: GeminiLiveContextAggregatorPair + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: GeminiLiveLLMContext + # or + context: OpenAILLMContext + + ## AFTER: + + # Context aggregator type + context_aggregator: LLMContextAggregatorPair + + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + ``` + + Also note that `LLMTextFrame`s are no longer pushed from `GeminiLiveLLMService` + when it's configured with `modalities=GeminiModalities.AUDIO`. If you need + to process its output, listen for `TTSTextFrame`s instead. + ### Changed - `FunctionFilter` now has a `filter_system_frames` arg, which controls whether diff --git a/examples/foundational/26a-gemini-live-transcription.py b/examples/foundational/26a-gemini-live-transcription.py index de277156b..9ac22a814 100644 --- a/examples/foundational/26a-gemini-live-transcription.py +++ b/examples/foundational/26a-gemini-live-transcription.py @@ -16,7 +16,9 @@ from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -72,7 +74,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # inference_on_context_initialization=False, ) - context = OpenAILLMContext( + context = LLMContext( [ { "role": "user", @@ -90,7 +92,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # }, ], ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) transcript = TranscriptProcessor() diff --git a/examples/foundational/26b-gemini-live-function-calling.py b/examples/foundational/26b-gemini-live-function-calling.py index 65d159bb0..fd2b36ab1 100644 --- a/examples/foundational/26b-gemini-live-function-calling.py +++ b/examples/foundational/26b-gemini-live-function-calling.py @@ -19,7 +19,9 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService @@ -139,10 +141,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_current_weather", fetch_weather_from_api) llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) - context = OpenAILLMContext( + context = LLMContext( [{"role": "user", "content": "Say hello."}], ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/examples/foundational/26c-gemini-live-video.py b/examples/foundational/26c-gemini-live-video.py index 7b765075e..be036a557 100644 --- a/examples/foundational/26c-gemini-live-video.py +++ b/examples/foundational/26c-gemini-live-video.py @@ -17,7 +17,9 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -65,7 +67,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # inference_on_context_initialization=False, ) - context = OpenAILLMContext( + context = LLMContext( [ { "role": "user", @@ -73,7 +75,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ], ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/examples/foundational/26d-gemini-live-text.py b/examples/foundational/26d-gemini-live-text.py index 062c0231b..fc9f68bcb 100644 --- a/examples/foundational/26d-gemini-live-text.py +++ b/examples/foundational/26d-gemini-live-text.py @@ -16,7 +16,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -109,8 +110,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set up conversation context and management # The context_aggregator will automatically collect conversation context - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26e-gemini-live-google-search.py b/examples/foundational/26e-gemini-live-google-search.py index 178fdd282..e80ed4536 100644 --- a/examples/foundational/26e-gemini-live-google-search.py +++ b/examples/foundational/26e-gemini-live-google-search.py @@ -16,7 +16,9 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService @@ -90,7 +92,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tools=tools, ) - context = OpenAILLMContext( + context = LLMContext( [ { "role": "user", @@ -98,7 +100,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): } ], ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/examples/foundational/26f-gemini-live-files-api.py b/examples/foundational/26f-gemini-live-files-api.py index eeda16f52..0091c01e6 100644 --- a/examples/foundational/26f-gemini-live-files-api.py +++ b/examples/foundational/26f-gemini-live-files-api.py @@ -16,7 +16,9 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService @@ -129,7 +131,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): mime_type = "text/plain" # Create context with file reference - context = OpenAILLMContext( + context = LLMContext( [ { "role": "user", @@ -152,7 +154,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): except Exception as e: logger.error(f"Error uploading file: {e}") # Continue with a basic context if file upload fails - context = OpenAILLMContext( + context = LLMContext( [ { "role": "user", @@ -162,7 +164,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) # Create context aggregator - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) # Build the pipeline pipeline = Pipeline( diff --git a/examples/foundational/26g-gemini-live-groundingMetadata.py b/examples/foundational/26g-gemini-live-groundingMetadata.py index bea1756b2..df86094da 100644 --- a/examples/foundational/26g-gemini-live-groundingMetadata.py +++ b/examples/foundational/26g-gemini-live-groundingMetadata.py @@ -10,7 +10,9 @@ from pipecat.frames.frames import Frame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -124,8 +126,13 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ] # Set up conversation context and management - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/examples/foundational/26h-gemini-live-vertex-function-calling.py b/examples/foundational/26h-gemini-live-vertex-function-calling.py index c0344a052..126d85ad7 100644 --- a/examples/foundational/26h-gemini-live-vertex-function-calling.py +++ b/examples/foundational/26h-gemini-live-vertex-function-calling.py @@ -9,21 +9,21 @@ import os from datetime import datetime from dotenv import load_dotenv -from google.genai.types import HttpOptions from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -139,10 +139,13 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_current_weather", fetch_weather_from_api) llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) - context = OpenAILLMContext( - [{"role": "user", "content": "Say hello."}], + context = LLMContext([{"role": "user", "content": "Say hello."}]) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), ) - context_aggregator = llm.create_context_aggregator(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26i-gemini-live-graceful-end.py b/examples/foundational/26i-gemini-live-graceful-end.py index e51bbb032..4e5933e8d 100644 --- a/examples/foundational/26i-gemini-live-graceful-end.py +++ b/examples/foundational/26i-gemini-live-graceful-end.py @@ -18,7 +18,9 @@ from pipecat.frames.frames import EndTaskFrame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -152,10 +154,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) llm.register_function("end_conversation", end_conversation) - context = OpenAILLMContext( + context = LLMContext( [{"role": "user", "content": "Say hello."}], ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py index 62ea5debe..5f92139bc 100644 --- a/examples/foundational/46-video-processing.py +++ b/examples/foundational/46-video-processing.py @@ -15,7 +15,9 @@ from pipecat.frames.frames import Frame, InputImageRawFrame, LLMRunFrame, Output from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments @@ -108,8 +110,13 @@ async def run_bot(pipecat_transport): } ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when Gemini Live used with AUDIO + # modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) # RTVI events for Pipecat client UI rtvi = RTVIProcessor() diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 26f037127..1fa8d9e6f 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -24,13 +24,7 @@ from pipecat.processors.aggregators.llm_context import ( ) try: - from google.genai.types import ( - Blob, - Content, - FunctionCall, - FunctionResponse, - Part, - ) + from google.genai.types import Blob, Content, FileData, FunctionCall, FunctionResponse, Part except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") @@ -309,6 +303,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): parts.append( Part( function_call=FunctionCall( + id=id, name=name, args=json.loads(tc["function"]["arguments"]), ) @@ -334,9 +329,12 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): function_name = params.tool_call_id_to_name_mapping[tool_call_id] parts.append( - Part.from_function_response( - name=function_name, - response=response_dict, + Part( + function_response=FunctionResponse( + id=tool_call_id, + name=function_name, + response=response_dict, + ) ) ) elif isinstance(content, str): @@ -358,6 +356,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): input_audio = c["input_audio"] audio_bytes = base64.b64decode(input_audio["data"]) parts.append(Part(inline_data=Blob(mime_type="audio/wav", data=audio_bytes))) + elif c["type"] == "file_data": + file_data = c["file_data"] + parts.append( + Part( + file_data=FileData( + mime_type=file_data.get("mime_type"), + file_uri=file_data.get("file_uri"), + ) + ) + ) return self.MessageConversionResult( content=Content(role=role, parts=parts), diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 4b5f05209..cf340c017 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -17,6 +17,7 @@ import json import random import time import uuid +import warnings from dataclasses import dataclass from enum import Enum from typing import Any, Dict, List, Optional, Union @@ -56,10 +57,12 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.metrics.metrics import LLMTokenUsage +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, ) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, @@ -219,6 +222,10 @@ class GeminiLiveContext(OpenAILLMContext): Provides Gemini-specific context management including system instruction extraction and message format conversion for the Live API. + + .. deprecated:: 0.0.93 + Gemini Live no longer uses `GeminiLiveContext` under the hood. + It now uses `LLMContext`. """ @staticmethod @@ -231,6 +238,22 @@ class GeminiLiveContext(OpenAILLMContext): Returns: The upgraded Gemini context instance. """ + # This warning is here rather than `__init__` since `upgrade()` was the + # "main" way that GeminiLiveContext instances were created. + # Almost no users should be seeing this message anyway, as + # GeminiLiveContext instances were typically created under the hood: + # the user would pass an OpenAILLMContext instance, which would be + # upgraded without them necessarily knowing. + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "GeminiLiveContext is deprecated. " + "Gemini Live no longer uses GeminiLiveContext under the hood. " + "It now uses LLMContext.", + DeprecationWarning, + stacklevel=2, + ) + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiLiveContext): logger.debug(f"Upgrading to Gemini Live Context: {obj}") obj.__class__ = GeminiLiveContext @@ -328,8 +351,28 @@ class GeminiLiveUserContextAggregator(OpenAIUserContextAggregator): Extends OpenAI user aggregator to handle Gemini-specific message passing while maintaining compatibility with the standard aggregation pipeline. + + .. deprecated:: 0.0.93 + Gemini Live no longer expects a `GeminiLiveUserContextAggregator`. + It now expects a `LLMUserAggregator`. """ + def __init__(self, *args, **kwargs): + """Initialize Gemini Live user context aggregator.""" + # Almost no users should be seeing this message, as + # `GeminiLiveUserContextAggregator`` instances were typically created + # under the hood, as part of `llm.create_context_aggregator()`. + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "GeminiLiveUserContextAggregator is deprecated. " + "Gemini Live no longer expects a GeminiLiveUserContextAggregator. " + "It now expects a LLMUserAggregator.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) + async def process_frame(self, frame, direction): """Process incoming frames for user context aggregation. @@ -349,8 +392,28 @@ class GeminiLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): Handles assistant response aggregation while filtering out LLMTextFrames to prevent duplicate context entries, as Gemini Live pushes both LLMTextFrames and TTSTextFrames. + + .. deprecated:: 0.0.93 + Gemini Live no longer uses `GeminiLiveAssistantContextAggregator` under the hood. + It now uses `LLMAssistantAggregator`. """ + def __init__(self, *args, **kwargs): + """Initialize Gemini Live assistant context aggregator.""" + # Almost no users should be seeing this message, as + # `GeminiLiveAssistantContextAggregator` instances were typically + # created under the hood, as part of `llm.create_context_aggregator()`. + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "GeminiLiveAssistantContextAggregator is deprecated. " + "Gemini Live no longer uses GeminiLiveAssistantContextAggregator under the hood. " + "It now uses LLMAssistantAggregator.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames for assistant context aggregation. @@ -380,6 +443,10 @@ class GeminiLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): class GeminiLiveContextAggregatorPair: """Pair of user and assistant context aggregators for Gemini Live. + .. deprecated:: 0.0.93 + `GeminiLiveContextAggregatorPair` is deprecated. + Use `LLMContextAggregatorPair` instead. + Parameters: _user: The user context aggregator instance. _assistant: The assistant context aggregator instance. @@ -388,6 +455,19 @@ class GeminiLiveContextAggregatorPair: _user: GeminiLiveUserContextAggregator _assistant: GeminiLiveAssistantContextAggregator + def __post_init__(self): + # Almost no users should be seeing this message, as + # `GeminiLiveContextAggregatorPair` instances were typically created + # under the hood, with `llm.create_context_aggregator()`. + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "GeminiLiveContextAggregatorPair is deprecated. " + "Use LLMContextAggregatorPair instead.", + DeprecationWarning, + stacklevel=2, + ) + def user(self) -> GeminiLiveUserContextAggregator: """Get the user context aggregator. @@ -665,6 +745,9 @@ class GeminiLiveLLMService(LLMService): # Initialize the API client. Subclasses can override this if needed. self.create_client() + # Bookkeeping for tool calls + self._completed_tool_calls = set() + def create_client(self): """Create the Gemini API client instance. Subclasses can override this.""" self._client = Client(api_key=self._api_key, http_options=self._http_options) @@ -787,9 +870,10 @@ class GeminiLiveLLMService(LLMService): # async def _handle_interruption(self): - await self._set_bot_is_speaking(False) - await self.push_frame(TTSStoppedFrame()) - await self.push_frame(LLMFullResponseEndFrame()) + if self._bot_is_speaking: + await self._set_bot_is_speaking(False) + await self.push_frame(TTSStoppedFrame()) + await self.push_frame(LLMFullResponseEndFrame()) async def _handle_user_started_speaking(self, frame): self._user_is_speaking = True @@ -807,7 +891,6 @@ class GeminiLiveLLMService(LLMService): # # frame processing - # # StartFrame, StopFrame, CancelFrame implemented in base class # @@ -829,22 +912,13 @@ class GeminiLiveLLMService(LLMService): if isinstance(frame, TranscriptionFrame): await self.push_frame(frame, direction) - elif isinstance(frame, OpenAILLMContextFrame): - context: GeminiLiveContext = GeminiLiveContext.upgrade(frame.context) - # For now, we'll only trigger inference here when either: - # 1. We have not seen a context frame before - # 2. The last message is a tool call result - if not self._context: - self._context = context - if frame.context.tools: - self._tools = frame.context.tools - await self._create_initial_response() - elif context.messages and context.messages[-1].get("role") == "tool": - # Support just one tool call per context frame for now - tool_result_message = context.messages[-1] - await self._tool_result(tool_result_message) - elif isinstance(frame, LLMContextFrame): - raise NotImplementedError("Universal LLMContext is not yet supported for Gemini Live.") + elif isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + context = ( + frame.context + if isinstance(frame, LLMContextFrame) + else LLMContext.from_openai_context(frame.context) + ) + await self._handle_context(context) elif isinstance(frame, InputTextRawFrame): await self._send_user_text(frame.text) await self.push_frame(frame, direction) @@ -883,6 +957,40 @@ class GeminiLiveLLMService(LLMService): else: await self.push_frame(frame, direction) + async def _handle_context(self, context: LLMContext): + if not self._context: + # We got our initial context + self._context = context + if context.tools: + self._tools = context.tools + # Initialize our bookkeeping of already-completed tool calls in + # the context + await self._process_completed_function_calls(send_new_results=False) + await self._create_initial_response() + else: + # We got an updated context. + # This may contain a new user message or tool call result. + self._context = context + # Send results for newly-completed function calls, if any. + await self._process_completed_function_calls(send_new_results=True) + + async def _process_completed_function_calls(self, send_new_results: bool): + # Check for set of completed function calls in the context + adapter: GeminiLLMAdapter = self.get_llm_adapter() + messages = adapter.get_llm_invocation_params(self._context).get("messages", []) + for message in messages: + if message.parts: + for part in message.parts: + if part.function_response: + # Found a newly-completed function call - send the result to the service + tool_call_id = part.function_response.id + tool_name = part.function_response.name + if send_new_results: + await self._tool_result( + tool_call_id, tool_name, part.function_response.response + ) + self._completed_tool_calls.add(tool_call_id) + async def _set_bot_is_speaking(self, speaking: bool): if self._bot_is_speaking == speaking: return @@ -1116,6 +1224,7 @@ class GeminiLiveLLMService(LLMService): if self._session: await self._session.close() self._session = None + self._completed_tool_calls = set() self._disconnecting = False except Exception as e: logger.error(f"{self} error disconnecting: {e}") @@ -1195,7 +1304,8 @@ class GeminiLiveLLMService(LLMService): self._run_llm_when_session_ready = True return - messages = self._context.get_messages_for_initializing_history() + adapter: GeminiLLMAdapter = self.get_llm_adapter() + messages = adapter.get_llm_invocation_params(self._context).get("messages", []) if not messages: return @@ -1223,8 +1333,9 @@ class GeminiLiveLLMService(LLMService): # Create a throwaway context just for the purpose of getting messages # in the right format - context = GeminiLiveContext.upgrade(OpenAILLMContext(messages=messages_list)) - messages = context.get_messages_for_initializing_history() + context = LLMContext(messages=messages_list) + adapter: GeminiLLMAdapter = self.get_llm_adapter() + messages = adapter.get_llm_invocation_params(context).get("messages", []) if not messages: return @@ -1239,17 +1350,16 @@ class GeminiLiveLLMService(LLMService): await self._handle_send_error(e) @traced_gemini_live(operation="llm_tool_result") - async def _tool_result(self, tool_result_message): + async def _tool_result( + self, tool_call_id: str, tool_name: str, tool_result_message: Dict[str, Any] + ): """Send tool result back to the API.""" if self._disconnecting or not self._session: return # For now we're shoving the name into the tool_call_id field, so this # will work until we revisit that. - id = tool_result_message.get("tool_call_id") - name = tool_result_message.get("tool_call_name") - result = json.loads(tool_result_message.get("content") or "") - response = FunctionResponse(name=name, id=id, response=result) + response = FunctionResponse(name=tool_name, id=tool_call_id, response=tool_result_message) try: await self._session.send_tool_response(function_responses=response) @@ -1442,8 +1552,8 @@ class GeminiLiveLLMService(LLMService): return # This is the output transcription text when modalities is set to AUDIO. - # In this case, we push LLMTextFrame and TTSTextFrame to be handled by the - # downstream assistant context aggregator. + # In this case, we push TTSTextFrame to be handled by the downstream + # assistant context aggregator. text = message.server_content.output_transcription.text if not text: @@ -1458,7 +1568,17 @@ class GeminiLiveLLMService(LLMService): # Collect text for tracing self._llm_output_buffer += text - await self.push_frame(LLMTextFrame(text=text)) + # NOTE: Shoot. When using Vertex AI, output transcription messages + # arrive *before* the model_turn messages with audio, so we need to + # handle sending TTSStartedFrame and LLMFullResponseStartFrame here as + # well. These messages also contain much *more* text (it looks further + # ahead). That means that on an interruption our recorded context will + # contain some text that was actually never spoken. + if not self._bot_is_speaking: + await self._set_bot_is_speaking(True) + await self.push_frame(TTSStartedFrame()) + await self.push_frame(LLMFullResponseStartFrame()) + await self.push_frame(TTSTextFrame(text=text)) async def _handle_msg_grounding_metadata(self, message: LiveServerMessage): @@ -1557,26 +1677,26 @@ class GeminiLiveLLMService(LLMService): *, user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), - ) -> GeminiLiveContextAggregatorPair: + ) -> LLMContextAggregatorPair: """Create an instance of GeminiLiveContextAggregatorPair from an OpenAILLMContext. Constructor keyword arguments for both the user and assistant aggregators can be provided. + NOTE: this method exists only for backward compatibility. New code + should instead do: + context = LLMContext(...) + context_aggregator = LLMContextAggregatorPair(context) + Args: context: The LLM context to use. user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams(). assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams(). Returns: - GeminiLiveContextAggregatorPair: A pair of context - aggregators, one for the user and one for the assistant, - encapsulated in an GeminiLiveContextAggregatorPair. + A pair of user and assistant context aggregators. """ - context.set_llm_adapter(self.get_llm_adapter()) - - GeminiLiveContext.upgrade(context) - user = GeminiLiveUserContextAggregator(context, params=user_params) - + context = LLMContext.from_openai_context(context) assistant_params.expect_stripped_words = False - assistant = GeminiLiveAssistantContextAggregator(context, params=assistant_params) - return GeminiLiveContextAggregatorPair(_user=user, _assistant=assistant) + return LLMContextAggregatorPair( + context, user_params=user_params, assistant_params=assistant_params + ) From df8aa3e4b0621461b808f873c6f6590eced75bae Mon Sep 17 00:00:00 2001 From: Roshan Date: Tue, 28 Oct 2025 15:10:46 -0700 Subject: [PATCH 49/93] feat: add generation_config support for Cartesia Sonic-3 Add GenerationConfig dataclass with volume, speed, and emotion parameters for Cartesia Sonic-3 TTS models. This enables fine-grained control over speech generation including volume (0.5-2.0), speed (0.6-1.5), and emotion (60+ options). Changes: - Add GenerationConfig dataclass with proper Google-style docstrings - Update CartesiaTTSService.InputParams to include generation_config - Update CartesiaHttpTTSService.InputParams to include generation_config - Modify _build_msg() to include generation_config in WebSocket messages - Modify run_tts() to include generation_config in HTTP requests - Maintain backward compatibility with existing speed and emotion parameters The legacy speed (literal strings) and emotion (list) parameters remain available for non-Sonic-3 models. --- src/pipecat/services/cartesia/tts.py | 60 ++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 3c0fe279c..9b5d7b0ac 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -10,6 +10,7 @@ import base64 import json import uuid import warnings +from dataclasses import dataclass from typing import AsyncGenerator, List, Literal, Optional, Union from loguru import logger @@ -48,6 +49,27 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +@dataclass +class GenerationConfig: + """Configuration for Cartesia Sonic-3 generation parameters. + + Sonic-3 interprets these parameters as guidance to ensure natural speech. + Test against your content for best results. + + Parameters: + volume: Volume multiplier for generated speech. Valid range: [0.5, 2.0]. Default is 1.0. + speed: Speed multiplier for generated speech. Valid range: [0.6, 1.5]. Default is 1.0. + emotion: Single emotion string to guide the emotional tone. Examples include neutral, + angry, excited, content, sad, scared. Over 60 emotions are supported. For best + results, use with recommended voices: Leo, Jace, Kyle, Gavin, Maya, Tessa, Dana, + and Marian. + """ + + volume: Optional[float] = None + speed: Optional[float] = None + emotion: Optional[str] = None + + def language_to_cartesia_language(language: Language) -> Optional[str]: """Convert a Language enum to Cartesia language code. @@ -101,16 +123,19 @@ class CartesiaTTSService(AudioContextWordTTSService): Parameters: language: Language to use for synthesis. - speed: Voice speed control. - emotion: List of emotion controls. + speed: Voice speed control for non-Sonic-3 models (literal values). + emotion: List of emotion controls for non-Sonic-3 models. .. deprecated:: 0.0.68 The `emotion` parameter is deprecated and will be removed in a future version. + generation_config: Generation configuration for Sonic-3 models. Includes volume, + speed (numeric), and emotion (string) parameters. """ language: Optional[Language] = Language.EN speed: Optional[Literal["slow", "normal", "fast"]] = None emotion: Optional[List[str]] = [] + generation_config: Optional[GenerationConfig] = None def __init__( self, @@ -179,6 +204,7 @@ class CartesiaTTSService(AudioContextWordTTSService): else "en", "speed": params.speed, "emotion": params.emotion, + "generation_config": params.generation_config, } self.set_model_name(model) self.set_voice(voice_id) @@ -297,6 +323,17 @@ class CartesiaTTSService(AudioContextWordTTSService): if self._settings["speed"]: msg["speed"] = self._settings["speed"] + if self._settings["generation_config"]: + gen_config = {} + if self._settings["generation_config"].volume is not None: + gen_config["volume"] = self._settings["generation_config"].volume + if self._settings["generation_config"].speed is not None: + gen_config["speed"] = self._settings["generation_config"].speed + if self._settings["generation_config"].emotion is not None: + gen_config["emotion"] = self._settings["generation_config"].emotion + if gen_config: + msg["generation_config"] = gen_config + return json.dumps(msg) async def start(self, frame: StartFrame): @@ -482,16 +519,19 @@ class CartesiaHttpTTSService(TTSService): Parameters: language: Language to use for synthesis. - speed: Voice speed control. - emotion: List of emotion controls. + speed: Voice speed control for non-Sonic-3 models (literal values). + emotion: List of emotion controls for non-Sonic-3 models. .. deprecated:: 0.0.68 The `emotion` parameter is deprecated and will be removed in a future version. + generation_config: Generation configuration for Sonic-3 models. Includes volume, + speed (numeric), and emotion (string) parameters. """ language: Optional[Language] = Language.EN speed: Optional[Literal["slow", "normal", "fast"]] = None emotion: Optional[List[str]] = Field(default_factory=list) + generation_config: Optional[GenerationConfig] = None def __init__( self, @@ -539,6 +579,7 @@ class CartesiaHttpTTSService(TTSService): else "en", "speed": params.speed, "emotion": params.emotion, + "generation_config": params.generation_config, } self.set_voice(voice_id) self.set_model_name(model) @@ -632,6 +673,17 @@ class CartesiaHttpTTSService(TTSService): if self._settings["speed"]: payload["speed"] = self._settings["speed"] + if self._settings["generation_config"]: + gen_config = {} + if self._settings["generation_config"].volume is not None: + gen_config["volume"] = self._settings["generation_config"].volume + if self._settings["generation_config"].speed is not None: + gen_config["speed"] = self._settings["generation_config"].speed + if self._settings["generation_config"].emotion is not None: + gen_config["emotion"] = self._settings["generation_config"].emotion + if gen_config: + payload["generation_config"] = gen_config + yield TTSStartedFrame() session = await self._client._get_session() From 408264a0fd0002c585a56a989e43560b3a4ff870 Mon Sep 17 00:00:00 2001 From: Roshan Date: Tue, 28 Oct 2025 15:16:49 -0700 Subject: [PATCH 50/93] docs: update CHANGELOG.md for generation_config feature --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c76d57d13..7c6928867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added `generation_config` parameter support to `CartesiaTTSService` and + `CartesiaHttpTTSService` for Cartesia Sonic-3 models. Includes a new + `GenerationConfig` dataclass with `volume` (0.5-2.0), `speed` (0.6-1.5), + and `emotion` (60+ options) parameters for fine-grained speech generation + control. + ### Changed - Updated the default model to `sonic-3` for `CartesiaTTSService` and From aaebcae2e8f1b13f105a9f3ed80548fe94706989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 17:01:49 -0700 Subject: [PATCH 51/93] pyproject: update daily-python to 0.21.0 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- uv.lock | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c76d57d13..d0ad0f9b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `daily-python` to 0.21.0. + - Updated the default model to `sonic-3` for `CartesiaTTSService` and `CartesiaHttpTTSService`. diff --git a/pyproject.toml b/pyproject.toml index fd992fa86..5760361b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.20.0" ] +daily = [ "daily-python~=0.21.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] diff --git a/uv.lock b/uv.lock index 8fae18c09..129491a61 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,13 +1282,13 @@ wheels = [ [[package]] name = "daily-python" -version = "0.20.0" +version = "0.21.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/02/ce81ebf11a04cd133a5539e08f85060574711fff05a1d6ad29705f0755c1/daily_python-0.20.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:7da3f1df8cd9ef7f7fcc96ce688348dc903f62d82b6dd155a53bc64b7a74f3a7", size = 13259887, upload-time = "2025-10-16T22:14:12.262Z" }, - { url = "https://files.pythonhosted.org/packages/4a/1e/51f06f3486c978e1184af2271e800ce6a6e8a8f95d61ee6624bae88ae9cd/daily_python-0.20.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:d02fd7b8c8079ceaa550ef23db052cdf70a8ffaf8ab6a8bc1a1e97bf0b939464", size = 11642453, upload-time = "2025-10-16T22:14:14.477Z" }, - { url = "https://files.pythonhosted.org/packages/71/c9/f767f0b479abd39330569ad61fb9db4661aae56cd74bb27c6f3483595463/daily_python-0.20.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a5c8718982c221dc18b41fb0692c9f8435f115f72e74994c94d3b9c6dad7c534", size = 13634216, upload-time = "2025-10-16T22:14:16.235Z" }, - { url = "https://files.pythonhosted.org/packages/e8/10/5c6d7b000bee36c2a0587a092a34c7486d2de831fc8e44ed42b16a6bd99f/daily_python-0.20.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ca9132aef1bdb5be663d1894b440dab1f998ebb3f45dfc31d44effabded4bc08", size = 14282189, upload-time = "2025-10-16T22:14:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/ff/11/99590f8b7aad077f3f9b5b59d39b010aee0bd01b14dece8ae1e93d8080e7/daily_python-0.21.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:bdec96417825181559769bb2258ae688d1215949a1878336194e36fb452274a8", size = 13277066, upload-time = "2025-10-29T00:20:49.523Z" }, + { url = "https://files.pythonhosted.org/packages/e5/db/8c57f1a1b713ba3393584ac2be32d8074d3022a2c2c17c28eb4cd2aa3629/daily_python-0.21.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:18677fa1415a0dc48b891cdf2fb8fe9dabc70e1b019d5aaa3d0699ccc8d187c9", size = 11644908, upload-time = "2025-10-29T00:20:52.106Z" }, + { url = "https://files.pythonhosted.org/packages/64/b6/b03f2f58a367d6ef4bb728715471542fdfa68afa8a177670139c3a2aadb7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:97eb97352fe74227061b678e330b8befcfa4c694feb6eb2b09fe6eacec00ad6d", size = 13652356, upload-time = "2025-10-29T00:20:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/bde65f6f8d4c1679dc6c185fa37dae9223f6ddb4b7ced728ef46504956f7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:68c3e36f609fc2fce79e4d17ecf1021eadd836506db6c5125f95c682bcf3612a", size = 14304643, upload-time = "2025-10-29T00:20:57.194Z" }, ] [[package]] @@ -4637,7 +4637,7 @@ requires-dist = [ { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, - { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.20.0" }, + { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.21.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = "~=4.7.0" }, { name = "docstring-parser", specifier = "~=0.16" }, { name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" }, From b0f5fc02c406c477185c0defc250fe50c109170f Mon Sep 17 00:00:00 2001 From: Roshan Date: Tue, 28 Oct 2025 18:41:58 -0700 Subject: [PATCH 52/93] refactor: use Pydantic BaseModel for GenerationConfig and simplify model_dump() - Change GenerationConfig from dataclass to Pydantic BaseModel for consistency - Simplify _build_msg() to use model_dump(exclude_none=True) instead of manual field extraction - Simplify HTTP run_tts() to use model_dump(exclude_none=True) instead of manual field extraction This addresses feedback from code review and reduces code duplication. --- src/pipecat/services/cartesia/tts.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 9b5d7b0ac..4e785a374 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -10,7 +10,6 @@ import base64 import json import uuid import warnings -from dataclasses import dataclass from typing import AsyncGenerator, List, Literal, Optional, Union from loguru import logger @@ -49,8 +48,7 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -@dataclass -class GenerationConfig: +class GenerationConfig(BaseModel): """Configuration for Cartesia Sonic-3 generation parameters. Sonic-3 interprets these parameters as guidance to ensure natural speech. @@ -324,15 +322,7 @@ class CartesiaTTSService(AudioContextWordTTSService): msg["speed"] = self._settings["speed"] if self._settings["generation_config"]: - gen_config = {} - if self._settings["generation_config"].volume is not None: - gen_config["volume"] = self._settings["generation_config"].volume - if self._settings["generation_config"].speed is not None: - gen_config["speed"] = self._settings["generation_config"].speed - if self._settings["generation_config"].emotion is not None: - gen_config["emotion"] = self._settings["generation_config"].emotion - if gen_config: - msg["generation_config"] = gen_config + msg["generation_config"] = self._settings["generation_config"].model_dump(exclude_none=True) return json.dumps(msg) @@ -674,15 +664,7 @@ class CartesiaHttpTTSService(TTSService): payload["speed"] = self._settings["speed"] if self._settings["generation_config"]: - gen_config = {} - if self._settings["generation_config"].volume is not None: - gen_config["volume"] = self._settings["generation_config"].volume - if self._settings["generation_config"].speed is not None: - gen_config["speed"] = self._settings["generation_config"].speed - if self._settings["generation_config"].emotion is not None: - gen_config["emotion"] = self._settings["generation_config"].emotion - if gen_config: - payload["generation_config"] = gen_config + payload["generation_config"] = self._settings["generation_config"].model_dump(exclude_none=True) yield TTSStartedFrame() From ede6c321495a83d0fc5a05a3fd697459643c598a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 28 Oct 2025 14:41:29 -0400 Subject: [PATCH 53/93] Update Simli to align with Pipecat constructor norms --- CHANGELOG.md | 9 +++ examples/foundational/27-simli-layer.py | 6 +- src/pipecat/services/simli/video.py | 94 ++++++++++++++++++++++--- 3 files changed, 97 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ad0f9b6..26ee846bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `daily-python` to 0.21.0. +- `SimliVideoService` now accepts `api_key` and `face_id` parameters directly, + with optional `params` for `max_session_length` and `max_idle_time` + configuration, aligning with other Pipecat service patterns. + - Updated the default model to `sonic-3` for `CartesiaTTSService` and `CartesiaHttpTTSService`. @@ -20,6 +24,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Upgraded `aws_sdk_bedrock_runtime` to v0.1.1 to resolve potential CPU issues when running `AWSNovaSonicLLMService`. +### Deprecated + +- `SimliVideoService` `simli_config` parameter is deprecated. Use `api_key` and + `face_id` parameters instead. + ### Removed - Removed the `aiohttp_session` arg from `SarvamTTSService` as it's no longer diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 9479632b5..348cf117b 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -9,7 +9,6 @@ import os from dotenv import load_dotenv from loguru import logger -from simli import SimliConfig from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 @@ -66,11 +65,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="a167e0f3-df7e-4d52-a9c3-f949145efdab", + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", ) simli_ai = SimliVideoService( - SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID")), + api_key=os.getenv("SIMLI_API_KEY"), + face_id="cace3ef7-a4c4-425d-a8cf-a5358eb0c427", ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index d48a744e0..383a8a3cb 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -7,9 +7,12 @@ """Simli video service for real-time avatar generation.""" import asyncio +import warnings +from typing import Optional import numpy as np from loguru import logger +from pydantic import BaseModel from pipecat.frames.frames import ( CancelFrame, @@ -41,30 +44,103 @@ class SimliVideoService(FrameProcessor): audio resampling, video frame processing, and connection management. """ + class InputParams(BaseModel): + """Input parameters for Simli video configuration. + + Parameters: + max_session_length: Absolute maximum session duration in seconds. + Avatar will disconnect after this time even if it's speaking. + max_idle_time: Maximum duration in seconds the avatar is not speaking + before the avatar disconnects. + """ + + max_session_length: Optional[int] = None + max_idle_time: Optional[int] = None + def __init__( self, - simli_config: SimliConfig, + *, + api_key: Optional[str] = None, + face_id: Optional[str] = None, + simli_config: Optional[SimliConfig] = None, use_turn_server: bool = False, latency_interval: int = 0, simli_url: str = "https://api.simli.ai", is_trinity_avatar: bool = False, + params: Optional[InputParams] = None, + **kwargs, ): """Initialize the Simli video service. Args: + api_key: Simli API key for authentication. + face_id: Simli Face ID. For Trinity avatars, specify "faceId/emotionId" + to use a different emotion than the default. simli_config: Configuration object for Simli client settings. - use_turn_server: Whether to use TURN server for connection. Defaults to False. - latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. - simli_url: URL of the simli servers. Can be changed for custom deployments of enterprise users. - is_trinity_avatar: boolean to tell simli client that this is a Trinity avatar which reduces latency when using Trinity. + Use api_key and face_id instead. + .. deprecated:: 0.0.92 + The 'simli_config' parameter is deprecated and will be removed in a future version. + Please use 'api_key' and 'face_id' parameters instead. + + use_turn_server: Whether to use TURN server for connection. Defaults to False. + latency_interval: Latency interval setting for sending health checks to check + the latency to Simli Servers. Defaults to 0. + simli_url: URL of the simli servers. Can be changed for custom deployments + of enterprise users. + is_trinity_avatar: Boolean to tell simli client that this is a Trinity avatar + which reduces latency when using Trinity. + params: Additional input parameters for session configuration. + **kwargs: Additional arguments passed to the parent FrameProcessor. """ - super().__init__() + super().__init__(**kwargs) + + params = params or SimliVideoService.InputParams() + + # Handle deprecated simli_config parameter + if simli_config is not None: + if api_key is not None or face_id is not None: + raise ValueError( + "Cannot specify both simli_config and api_key/face_id. " + "Please use api_key and face_id (simli_config is deprecated)." + ) + + warnings.warn( + "The 'simli_config' parameter is deprecated and will be removed in a future version. " + "Please use 'api_key' and 'face_id' parameters instead, with optional 'params' for " + "max_session_length and max_idle_time configuration.", + DeprecationWarning, + stacklevel=2, + ) + + # Use the provided simli_config + config = simli_config + else: + # Validate new parameters + if api_key is None: + raise ValueError("api_key is required") + if face_id is None: + raise ValueError("face_id is required") + + # Build SimliConfig from new parameters + # Only pass optional parameters if explicitly provided to use SimliConfig defaults + config_kwargs = { + "apiKey": api_key, + "faceId": face_id, + } + if params.max_session_length is not None: + config_kwargs["maxSessionLength"] = params.max_session_length + if params.max_idle_time is not None: + config_kwargs["maxIdleTime"] = params.max_idle_time + + config = SimliConfig(**config_kwargs) + self._initialized = False - simli_config.maxIdleTime += 5 - simli_config.maxSessionLength += 5 + # Add buffer time to session limits + config.maxIdleTime += 5 + config.maxSessionLength += 5 self._simli_client = SimliClient( - simli_config, + config, use_turn_server, latency_interval, simliURL=simli_url, From 9307079af2ab297b5db94354866ca3b8c1745389 Mon Sep 17 00:00:00 2001 From: ivaaan Date: Wed, 29 Oct 2025 17:05:41 +0100 Subject: [PATCH 54/93] upd changelog --- CHANGELOG.md | 309 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 295 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae7e3dcb1..13324c317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,18 +7,299 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Updated `daily-python` to 0.21.0. + +- `SimliVideoService` now accepts `api_key` and `face_id` parameters directly, + with optional `params` for `max_session_length` and `max_idle_time` + configuration, aligning with other Pipecat service patterns. + +- Updated the default model to `sonic-3` for `CartesiaTTSService` and + `CartesiaHttpTTSService`. + +- `FunctionFilter` now has a `filter_system_frames` arg, which controls whether + or not SystemFrames are filtered. + +- Upgraded `aws_sdk_bedrock_runtime` to v0.1.1 to resolve potential CPU issues + when running `AWSNovaSonicLLMService`. + +### Deprecated + +- `SimliVideoService` `simli_config` parameter is deprecated. Use `api_key` and + `face_id` parameters instead. + +### Removed + +- Removed the `aiohttp_session` arg from `SarvamTTSService` as it's no longer + used. + +### Fixed + +- Fixed an issue where `DailyTransport` would timeout prematurely on join and on + leave. + +- Fixed an issue in the runner where starting a DailyTransport room via + `/start` didn't support using the `DAILY_SAMPLE_ROOM_URL` env var. + +- Fixed an issue in `ServiceSwitcher` where the `STTService`s would result in + all STT services producing `TranscriptionFrame`s. + +- Fixed an issue in `HumeTTSService` that was only using Octave 2, which does not support the `description` field. Now, if a description is provided, it switches to Octave 1. + +## [0.0.91] - 2025-10-21 + ### Added +- It is now possible to start a bot from the `/start` endpoint when using the + runner Daily's transport. This follows the Pipecat Cloud format with + `createDailyRoom` and `body` fields in the POST request body. + +- Added an ellipsis character (`…`) to the end of sentence detection in the + string utils. + +- Expanded support for universal `LLMContext` to `AWSNovaSonicLLMService`. + As a reminder, the context-setup pattern when using `LLMContext` is: + + ```python + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + ``` + + (Note that even though `AWSNovaSonicLLMService` now supports the universal + `LLMContext`, it is not meant to be swapped out for another LLM service at + runtime.) + + Worth noting: whether or not you use the new context-setup pattern with + `AWSNovaSonicLLMService`, some types have changed under the hood: + + ```python + ## BEFORE: + + # Context aggregator type + context_aggregator: AWSNovaSonicContextAggregatorPair + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: AWSNovaSonicLLMContext + # or + context: OpenAILLMContext + + ## AFTER: + + # Context aggregator type + context_aggregator: LLMContextAggregatorPair + + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + ``` + +- Added support for `bulbul:v3` model in `SarvamTTSService` and + `SarvamHttpTTSService`. + +- Added `keyterms_prompt` parameter to `AssemblyAIConnectionParams`. + +- Added `speech_model` parameter to `AssemblyAIConnectionParams` to access the + multilingual model. + +- Added support for trickle ICE to the `SmallWebRTCTransport`. + +- Added support for updating `OpenAITTSService` settings (`instructions` and + `speed`) at runtime via `TTSUpdateSettingsFrame`. + +- Added `--whatsapp` flag to runner to better surface WhatsApp transport logs. + +- Added `on_connected` and `on_disconnected` events to TTS and STT + websocket-based services. + +- Added an `aggregate_sentences` arg in `ElevenLabsHttpTTSService`, where the + default value is True. + +- Added a `room_properties` arg to the Daily runner's `configure()` method, + allowing `DailyRoomProperties` to be provided. + +- The runner `--folder` argument now supports downloading files from + subdirectories. + +### Changed + +- `RunnerArguments` now include the `body` field, so there's no need to add it + to subclasses. Also, all `RunnerArguments` fields are now keyword-only. + +- `CartesiaSTTService` now inherits from `WebsocketSTTService`. + +- Package upgrades: + + - `daily-python` upgraded to 0.20.0. + - `openai` upgraded to support up to 2.x.x. + - `openpipe` upgraded to support up to 5.x.x. + +- `SpeechmaticsSTTService` updated dependencies for `speechmatics-rt>=0.5.0`. + +### Deprecated + +- The `send_transcription_frames` argument to `AWSNovaSonicLLMService` is + deprecated. Transcription frames are now always sent. They go upstream, to be + handled by the user context aggregator. See "Added" section for details. + +- Types in `pipecat.services.aws.nova_sonic.context` have been deprecated due + to changes to support `LLMContext`. See "Changed" section for details. + +### Fixed + +- Fixed an issue where the `RTVIProcessor` was sending duplicate + `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` messages. + +- Fixed an issue in `AWSBedrockLLMService` where both `temperature` and `top_p` + were always sent together, causing conflicts with models like Claude Sonnet 4.5 + that don't allow both parameters simultaneously. The service now only includes + inference parameters that are explicitly set, and `InputParams` defaults have + been changed to `None` to rely on AWS Bedrock's built-in model defaults. + +- Fixed an issue in `RivaSegmentedSTTService` where a runtime error occurred due + to a mismatch in the `_handle_transcription` method's signature. + +- Fixed multiple pipeline task cancellation issues. `asyncio.CancelledError` is + now handled properly in `PipelineTask` making it possible to cancel an asyncio + task that it's executing a `PipelineRunner` cleanly. Also, + `PipelineTask.cancel()` does not block anymore waiting for the `CancelFrame` + to reach the end of the pipeline (going back to the behavior in < 0.0.83). + +- Fixed an issue in `ElevenLabsTTSService` and `ElevenLabsHttpTTSService` where + the Flash models would split words, resulting in a space being inserted + between words. + +- Fixed an issue where audio filters' `stop()` would not be called when using + `CancelFrame`. + +- Fixed an issue in `ElevenLabsHttpTTSService`, where + `apply_text_normalization` was incorrectly set as a query parameter. It's now + being added as a request parameter. + +- Fixed an issue where `RimeHttpTTSService` and `PiperTTSService` could generate + incorrectly 16-bit aligned audio frames, potentially leading to internal + errors or static audio. + +- Fixed an issue in `SpeechmaticsSTTService` where `AdditionalVocabEntry` items + needed to have `sounds_like` for the session to start. + +### Other + +- Added foundational example `47-sentry-metrics.py`, demonstrating how to use the + `SentryMetrics` processor. + +- Added foundational example `14x-function-calling-openpipe.py`. + +## [0.0.90] - 2025-10-10 + +### Added + +- Added audio filter `KrispVivaFilter` using the Krisp VIVA SDK. + +- Added `--folder` argument to the runner, allowing files saved in that folder + to be downloaded from `http://HOST:PORT/file/FILE`. + +- Added `GeminiLiveVertexLLMService`, for accessing Gemini Live via Google + Vertex AI. + +- Added some new configuration options to `GeminiLiveLLMService`: + + - `thinking` + - `enable_affective_dialog` + - `proactivity` + + Note that these new configuration options require using a newer model than + the default, like "gemini-2.5-flash-native-audio-preview-09-2025". The last + two require specifying `http_options=HttpOptions(api_version="v1alpha")`. + +- Added `on_pipeline_error` event to `PipelineTask`. This event will get fired + when an `ErrorFrame` is pushed (use `FrameProcessor.push_error()`). + + ```python + @task.event_handler("on_pipeline_error") + async def on_pipeline_error(task: PipelineTask, frame: ErrorFrame): + ... + ``` + +- Added a `service_tier` `InputParam` to the `BaseOpenAILLMService`. This + parameter can influence the latency of the response. For example `"priority"` + will result in faster completions, but in exchange for a higher price. + +### Changed + +- Updated `GeminiLiveLLMService` to use the `google-genai` library rather than + use WebSockets directly. + +### Deprecated + +- `LivekitFrameSerializer` is now deprecated. Use `LiveKitTransport` instead. + +- `pipecat.service.openai_realtime` is now deprecated, use + `pipecat.services.openai.realtime` instead or + `pipecat.services.azure.realtime` for Azure Realtime. + +- `pipecat.service.aws_nova_sonic` is now deprecated, use + `pipecat.services.aws.nova_sonic` instead. + +- `GeminiMultimodalLiveLLMService` is now deprecated, use + `GeminiLiveLLMService`. + +### Fixed + +- Fixed a `GoogleVertexLLMService` issue that would generate an error if no + token information was returned. + +- `GeminiLiveLLMService` will now end gracefully (i.e. after the bot has + finished) upon receiving an `EndFrame`. + +- `GeminiLiveLLMService` will try to seamlessly reconnect when it loses its + connection. + +## [0.0.89] - 2025-10-07 + +### Fixed + +- Reverted a change introduced in 0.0.88 that was causing pipelines to be frozen + when using interruption strategies and processors that block interruption + frames (e.g. `STTMuteFilter`). + +## [0.0.88] - 2025-10-07 + +### Added + +- Added support for Nano Banana models to `GoogleLLMService`. For example, you + can now use the `gemini-2.5-flash-image` model to generate images. + +- Added `HumeTTSService` for text-to-speech synthesis using Hume AI's expressive + voice models. Provides high-quality, emotionally expressive speech synthesis + with support for various voice models. Includes example in + `examples/foundational/07ad-interruptible-hume.py`. Use with: + `uv pip install pipecat-ai[hume]`. + +### Changed + +- Updated default `GoogleLLMService` model to `gemini-2.5-flash`. + +### Deprecated + +- PlayHT is shutting down their API on December 31st, 2025. As a result, + `PlayHTTTSService` and `PlayHTHttpTTSService` are deprecated and will be + removed in a future version. + +### Fixed + +- Fixed an issue with `AWSNovaSonicLLMService` where the client wouldn't + connect due to a breaking change in the AWS dependency chain. + - `PermissionError` is now caught if NLTK's `punkt_tab` can't be downloaded. -- Added `HumeTTSService` for text-to-speech synthesis using Hume AI's - expressive voice models. Provides high-quality, emotionally expressive speech - synthesis with support for various voice models. Includes example in - `examples/foundational/07ad-interruptible-hume.py`. - -- Added `hume` optional dependency group for Hume AI TTS integration. - -### Fixed +- Fixed an issue that would cause wrong user/assistant context ordering when + using interruption strategies. - Fixed RTVI incoming message handling, broken in 0.0.87. @@ -1396,7 +1677,7 @@ quality and critical bugs impacting `ParallelPipelines` functionality.** - Added `session_token` parameter to `AWSNovaSonicLLMService`. - Added Gemini Multimodal Live File API for uploading, fetching, listing, and - deleting files. See `26f-gemini-multimodal-live-files-api.py` for example usage. + deleting files. See `26f-gemini-live-files-api.py` for example usage. ### Changed @@ -3402,7 +3683,7 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Added the new modalities option and helper function to set Gemini output modalities. -- Added `examples/foundational/26d-gemini-multimodal-live-text.py` which is +- Added `examples/foundational/26d-gemini-live-text.py` which is using Gemini as TEXT modality and using another TTS provider for TTS process. ### Changed @@ -3589,9 +3870,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Added new foundational examples for `GeminiMultimodalLiveLLMService`: - `26-gemini-multimodal-live.py` - - `26a-gemini-multimodal-live-transcription.py` - - `26b-gemini-multimodal-live-video.py` - - `26c-gemini-multimodal-live-video.py` + - `26a-gemini-live-transcription.py` + - `26b-gemini-live-video.py` + - `26c-gemini-live-video.py` - Added `SimliVideoService`. This is an integration for Simli AI avatars. (see https://www.simli.com) @@ -5041,4 +5322,4 @@ a bit. ## [0.0.2] - 2024-03-12 -Initial public release. +Initial public release. \ No newline at end of file From 615aae5b954da57ee576da63f884fd3597364270 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 12:21:13 -0400 Subject: [PATCH 55/93] Fix `GeminiLiveLLMService`'s sending of `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame` so that they properly bookend responses. Properly bookended responses now work with: - AUDIO modality (validated with 26b example) - TEXT modality (validated with 26d example) - AUDIO modality with Vertex AI (validated with 26h example) It doesn't seem that TEXT modality is supported with Vertex AI, hence the missing "quadrant" of validation. --- .../services/google/gemini_live/llm.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index cf340c017..b7b48c3dc 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -873,7 +873,9 @@ class GeminiLiveLLMService(LLMService): if self._bot_is_speaking: await self._set_bot_is_speaking(False) await self.push_frame(TTSStoppedFrame()) - await self.push_frame(LLMFullResponseEndFrame()) + # Do not send LLMFullResponseEndFrame here - an interruption + # already tells the assistant context aggregator that the response + # is over. async def _handle_user_started_speaking(self, frame): self._user_is_speaking = True @@ -1388,6 +1390,7 @@ class GeminiLiveLLMService(LLMService): text = part.text if text: if not self._bot_text_buffer: + # TEXT modality case: send service start frame await self.push_frame(LLMFullResponseStartFrame()) self._bot_text_buffer += text @@ -1423,6 +1426,7 @@ class GeminiLiveLLMService(LLMService): if not audio: return + # AUDIO modality case: update bot speaking state and send service start frames if not self._bot_is_speaking: await self._set_bot_is_speaking(True) await self.push_frame(TTSStartedFrame()) @@ -1464,7 +1468,6 @@ class GeminiLiveLLMService(LLMService): @traced_gemini_live(operation="llm_response") async def _handle_msg_turn_complete(self, message: LiveServerMessage): """Handle the turn complete message.""" - await self._set_bot_is_speaking(False) text = self._bot_text_buffer # Trace the complete LLM response (this will be handled by the decorator) @@ -1483,13 +1486,15 @@ class GeminiLiveLLMService(LLMService): self._search_result_buffer = "" self._accumulated_grounding_metadata = None - # Only push the TTSStoppedFrame if the bot is outputting audio - # when text is found, modalities is set to TEXT and no audio - # is produced. if not text: - await self.push_frame(TTSStoppedFrame()) - - await self.push_frame(LLMFullResponseEndFrame()) + # AUDIO modality case + if self._bot_is_speaking: + await self._set_bot_is_speaking(False) + await self.push_frame(TTSStoppedFrame()) + await self.push_frame(LLMFullResponseEndFrame()) + else: + # TEXT modality case + await self.push_frame(LLMFullResponseEndFrame()) @traced_stt async def _handle_user_transcription( From 65c17a698eef76b203bbb6d0f65d7ed63b8770fa Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 12:44:00 -0400 Subject: [PATCH 56/93] Whoops - fix a bug in `GeminiLiveLLMService` where we weren't checking if a tool call result was already handled before reporting it to the LLM --- src/pipecat/services/google/gemini_live/llm.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index b7b48c3dc..f16d5c2aa 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -984,14 +984,15 @@ class GeminiLiveLLMService(LLMService): if message.parts: for part in message.parts: if part.function_response: - # Found a newly-completed function call - send the result to the service tool_call_id = part.function_response.id tool_name = part.function_response.name - if send_new_results: - await self._tool_result( - tool_call_id, tool_name, part.function_response.response - ) - self._completed_tool_calls.add(tool_call_id) + if tool_call_id and tool_call_id not in self._completed_tool_calls: + # Found a newly-completed function call - send the result to the service + if send_new_results: + await self._tool_result( + tool_call_id, tool_name, part.function_response.response + ) + self._completed_tool_calls.add(tool_call_id) async def _set_bot_is_speaking(self, speaking: bool): if self._bot_is_speaking == speaking: From 82d494d3d4d69bc18db8bde8ec57958746327bff Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 14:31:32 -0400 Subject: [PATCH 57/93] =?UTF-8?q?Fix=20a=20bug=20in=20`GeminiLiveLLMServic?= =?UTF-8?q?e`=20related=20to=20ending=20gracefully=E2=80=94i.e.=20waiting?= =?UTF-8?q?=20for=20the=20bot=20to=20stop=20responding=20before=20ending?= =?UTF-8?q?=20the=20pipeline=E2=80=94when=20the=20service=20is=20configure?= =?UTF-8?q?d=20with=20the=20TEXT=20modality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../26i-gemini-live-graceful-end.py | 2 +- .../services/google/gemini_live/llm.py | 52 +++++++++++-------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/examples/foundational/26i-gemini-live-graceful-end.py b/examples/foundational/26i-gemini-live-graceful-end.py index 4e5933e8d..2865dbed4 100644 --- a/examples/foundational/26i-gemini-live-graceful-end.py +++ b/examples/foundational/26i-gemini-live-graceful-end.py @@ -64,7 +64,7 @@ You have three tools available to you: After you've responded to the user three times, do two things, in order: 1. Politely let them know that that's all the time you have today and say goodbye. -2. Call the end_conversation tool to gracefully end the conversation. +2. *WITHOUT WAITING FOR THE USER TO RESPOND*, call the end_conversation tool to gracefully end the conversation. """ diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index f16d5c2aa..9a6c7db81 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -689,7 +689,7 @@ class GeminiLiveLLMService(LLMService): self._run_llm_when_session_ready = False self._user_is_speaking = False - self._bot_is_speaking = False + self._bot_is_responding = False self._user_audio_buffer = bytearray() self._user_transcription_buffer = "" self._last_transcription_sent = "" @@ -870,9 +870,10 @@ class GeminiLiveLLMService(LLMService): # async def _handle_interruption(self): - if self._bot_is_speaking: - await self._set_bot_is_speaking(False) - await self.push_frame(TTSStoppedFrame()) + if self._bot_is_responding: + await self._set_bot_is_responding(False) + if self._settings.get("modalities") == GeminiModalities.AUDIO: + await self.push_frame(TTSStoppedFrame()) # Do not send LLMFullResponseEndFrame here - an interruption # already tells the assistant context aggregator that the response # is over. @@ -905,7 +906,7 @@ class GeminiLiveLLMService(LLMService): """ # Defer EndFrame handling until after the bot turn is finished if isinstance(frame, EndFrame): - if self._bot_is_speaking: + if self._bot_is_responding: logger.debug("Deferring handling EndFrame until bot turn is finished") self._end_frame_pending_bot_turn_finished = frame return @@ -994,13 +995,13 @@ class GeminiLiveLLMService(LLMService): ) self._completed_tool_calls.add(tool_call_id) - async def _set_bot_is_speaking(self, speaking: bool): - if self._bot_is_speaking == speaking: + async def _set_bot_is_responding(self, responding: bool): + if self._bot_is_responding == responding: return - self._bot_is_speaking = speaking + self._bot_is_responding = responding - if not self._bot_is_speaking and self._end_frame_pending_bot_turn_finished: + if not self._bot_is_responding and self._end_frame_pending_bot_turn_finished: await self.queue_frame(self._end_frame_pending_bot_turn_finished) self._end_frame_pending_bot_turn_finished = None @@ -1390,8 +1391,10 @@ class GeminiLiveLLMService(LLMService): # part.text is added when `modalities` is set to TEXT; otherwise, it's None text = part.text if text: - if not self._bot_text_buffer: - # TEXT modality case: send service start frame + if not self._bot_is_responding: + # Update bot responding state and send service start frame + # (AUDIO modality case) + await self._set_bot_is_responding(True) await self.push_frame(LLMFullResponseStartFrame()) self._bot_text_buffer += text @@ -1402,6 +1405,8 @@ class GeminiLiveLLMService(LLMService): if msg.server_content and msg.server_content.grounding_metadata: self._accumulated_grounding_metadata = msg.server_content.grounding_metadata + # If we have no audio, stop here. + # All logic below this point pertains to the AUDIO modality. inline_data = part.inline_data if not inline_data: return @@ -1427,9 +1432,10 @@ class GeminiLiveLLMService(LLMService): if not audio: return - # AUDIO modality case: update bot speaking state and send service start frames - if not self._bot_is_speaking: - await self._set_bot_is_speaking(True) + # Update bot responding state and send service start frames + # (AUDIO modality case) + if not self._bot_is_responding: + await self._set_bot_is_responding(True) await self.push_frame(TTSStartedFrame()) await self.push_frame(LLMFullResponseStartFrame()) @@ -1487,15 +1493,15 @@ class GeminiLiveLLMService(LLMService): self._search_result_buffer = "" self._accumulated_grounding_metadata = None - if not text: - # AUDIO modality case - if self._bot_is_speaking: - await self._set_bot_is_speaking(False) + if self._bot_is_responding: + await self._set_bot_is_responding(False) + if not text: + # AUDIO modality case await self.push_frame(TTSStoppedFrame()) await self.push_frame(LLMFullResponseEndFrame()) - else: - # TEXT modality case - await self.push_frame(LLMFullResponseEndFrame()) + else: + # TEXT modality case + await self.push_frame(LLMFullResponseEndFrame()) @traced_stt async def _handle_user_transcription( @@ -1580,8 +1586,8 @@ class GeminiLiveLLMService(LLMService): # well. These messages also contain much *more* text (it looks further # ahead). That means that on an interruption our recorded context will # contain some text that was actually never spoken. - if not self._bot_is_speaking: - await self._set_bot_is_speaking(True) + if not self._bot_is_responding: + await self._set_bot_is_responding(True) await self.push_frame(TTSStartedFrame()) await self.push_frame(LLMFullResponseStartFrame()) From 9dafb715c4c4e21c2d4d845c7c9bd3cef07cd1e4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 15:30:43 -0400 Subject: [PATCH 58/93] Update some deprecation versions --- src/pipecat/services/google/gemini_live/llm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 9a6c7db81..e42913771 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -223,7 +223,7 @@ class GeminiLiveContext(OpenAILLMContext): Provides Gemini-specific context management including system instruction extraction and message format conversion for the Live API. - .. deprecated:: 0.0.93 + .. deprecated:: 0.0.92 Gemini Live no longer uses `GeminiLiveContext` under the hood. It now uses `LLMContext`. """ @@ -352,7 +352,7 @@ class GeminiLiveUserContextAggregator(OpenAIUserContextAggregator): Extends OpenAI user aggregator to handle Gemini-specific message passing while maintaining compatibility with the standard aggregation pipeline. - .. deprecated:: 0.0.93 + .. deprecated:: 0.0.92 Gemini Live no longer expects a `GeminiLiveUserContextAggregator`. It now expects a `LLMUserAggregator`. """ @@ -393,7 +393,7 @@ class GeminiLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): to prevent duplicate context entries, as Gemini Live pushes both LLMTextFrames and TTSTextFrames. - .. deprecated:: 0.0.93 + .. deprecated:: 0.0.92 Gemini Live no longer uses `GeminiLiveAssistantContextAggregator` under the hood. It now uses `LLMAssistantAggregator`. """ @@ -443,7 +443,7 @@ class GeminiLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): class GeminiLiveContextAggregatorPair: """Pair of user and assistant context aggregators for Gemini Live. - .. deprecated:: 0.0.93 + .. deprecated:: 0.0.92 `GeminiLiveContextAggregatorPair` is deprecated. Use `LLMContextAggregatorPair` instead. From 3ea1e357f28c5b0ab35153e941c986a72cbaab15 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 16 Oct 2025 15:59:50 -0400 Subject: [PATCH 59/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (initial part of work) --- examples/foundational/19-openai-realtime.py | 6 +- .../services/open_ai_realtime_adapter.py | 125 +++++++++++++- .../services/openai/realtime/context.py | 154 ------------------ src/pipecat/services/openai/realtime/llm.py | 76 ++++++--- .../services/openai_realtime/context.py | 21 --- 5 files changed, 173 insertions(+), 209 deletions(-) delete mode 100644 src/pipecat/services/openai_realtime/context.py diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index f182d7c8c..685070d5e 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -19,6 +19,8 @@ from pipecat.observers.loggers.transcription_log_observer import TranscriptionLo from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments @@ -163,12 +165,12 @@ Remember, your responses should be short. Just one or two sentences, usually. Re # Create a standard OpenAI LLM context object using the normal messages format. The # OpenAIRealtimeLLMService will convert this internally to messages that the # openai WebSocket API can understand. - context = OpenAILLMContext( + context = LLMContext( [{"role": "user", "content": "Say hello!"}], tools, ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 2ff629e2e..67bdfb6ae 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -6,12 +6,18 @@ """OpenAI Realtime LLM adapter for Pipecat.""" -from typing import Any, Dict, List, TypedDict +import copy +import json +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, TypedDict + +from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage +from pipecat.services.openai.realtime import events class OpenAIRealtimeLLMInvocationParams(TypedDict): @@ -20,7 +26,9 @@ class OpenAIRealtimeLLMInvocationParams(TypedDict): This is a placeholder until support for universal LLMContext machinery is added for OpenAI Realtime. """ - pass + system_instruction: Optional[str] + messages: List[events.ConversationItem] + tools: List[Dict[str, Any]] class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): @@ -33,7 +41,7 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): @property def id_for_llm_specific_messages(self) -> str: """Get the identifier used in LLMSpecificMessage instances for OpenAI Realtime.""" - raise NotImplementedError("Universal LLMContext is not yet supported for OpenAI Realtime.") + return "openai-realtime" def get_llm_invocation_params(self, context: LLMContext) -> OpenAIRealtimeLLMInvocationParams: """Get OpenAI Realtime-specific LLM invocation parameters from a universal LLM context. @@ -46,7 +54,13 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): Returns: Dictionary of parameters for invoking OpenAI Realtime's API. """ - raise NotImplementedError("Universal LLMContext is not yet supported for OpenAI Realtime.") + messages = self._from_universal_context_messages(self.get_messages(context)) + return { + "system_instruction": messages.system_instruction, + "messages": messages.messages, + # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) + "tools": self.from_standard_tools(context.tools) or [], + } def get_messages_for_logging(self, context) -> List[Dict[str, Any]]: """Get messages from a universal LLM context in a format ready for logging about OpenAI Realtime. @@ -61,7 +75,106 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): Returns: List of messages in a format ready for logging about OpenAI Realtime. """ - raise NotImplementedError("Universal LLMContext is not yet supported for OpenAI Realtime.") + return self._from_universal_context_messages(self.get_messages(context)).messages + + @dataclass + class ConvertedMessages: + """Container for OpenAI-formatted messages converted from universal context.""" + + messages: List[events.ConversationItem] + system_instruction: Optional[str] = None + + def _from_universal_context_messages( + self, universal_context_messages: List[LLMContextMessage] + ) -> ConvertedMessages: + # We can't load a long conversation history into the openai realtime api yet. (The API/model + # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So + # our general strategy until this is fixed is just to put everything into a first "user" + # message as a single input. + + if not universal_context_messages: + return self.ConvertedMessages() + + messages = copy.deepcopy(universal_context_messages) + system_instruction = None + + # If we have a "system" message as our first message, let's pull that out into session + # "instructions" + if messages[0].get("role") == "system": + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + system_instruction = content + elif isinstance(content, list): + system_instruction = content[0].get("text") + if not messages: + return self.ConvertedMessages(messages=[], system_instruction=system_instruction) + + # If we have just a single "user" item, we can just send it normally + if len(messages) == 1 and messages[0].get("role") == "user": + return self.ConvertedMessages( + messages=[self._from_universal_context_message(messages[0])], + system_instruction=system_instruction, + ) + + # Otherwise, let's pack everything into a single "user" message with a bit of + # explanation for the LLM + intro_text = """ + This is a previously saved conversation. Please treat this conversation history as a + starting point for the current conversation.""" + + trailing_text = """ + This is the end of the previously saved conversation. Please continue the conversation + from here. If the last message is a user instruction or question, act on that instruction + or answer the question. If the last message is an assistant response, simple say that you + are ready to continue the conversation.""" + + self.ConvertedMessages( + messages=[ + { + "role": "user", + "type": "message", + "content": [ + { + "type": "input_text", + "text": "\n\n".join( + [intro_text, json.dumps(messages, indent=2), trailing_text] + ), + } + ], + } + ], + system_instruction=system_instruction, + ) + + def _from_universal_context_message( + self, message: LLMContextMessage + ) -> events.ConversationItem: + if message.get("role") == "user": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + return events.ConversationItem( + role="user", + type="message", + content=[events.ItemContent(type="input_text", text=content)], + ) + if message.get("role") == "assistant" and message.get("tool_calls"): + tc = message.get("tool_calls")[0] + return events.ConversationItem( + type="function_call", + call_id=tc["id"], + name=tc["function"]["name"], + arguments=tc["function"]["arguments"], + ) + logger.error(f"Unhandled message type in _from_universal_context_message: {message}") @staticmethod def _to_openai_realtime_function_format(function: FunctionSchema) -> Dict[str, Any]: diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py index cb1c0a9f5..da08f78ad 100644 --- a/src/pipecat/services/openai/realtime/context.py +++ b/src/pipecat/services/openai/realtime/context.py @@ -31,160 +31,6 @@ from . import events from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame -class OpenAIRealtimeLLMContext(OpenAILLMContext): - """OpenAI Realtime LLM context with session management and message conversion. - - Extends the standard OpenAI LLM context to support real-time session properties, - instruction management, and conversion between standard message formats and - realtime conversation items. - """ - - def __init__(self, messages=None, tools=None, **kwargs): - """Initialize the OpenAIRealtimeLLMContext. - - Args: - messages: Initial conversation messages. Defaults to None. - tools: Available function tools. Defaults to None. - **kwargs: Additional arguments passed to parent OpenAILLMContext. - """ - super().__init__(messages=messages, tools=tools, **kwargs) - self.__setup_local() - - def __setup_local(self): - self.llm_needs_settings_update = True - self.llm_needs_initial_messages = True - self._session_instructions = "" - - return - - @staticmethod - def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": - """Upgrade a standard OpenAI LLM context to a realtime context. - - Args: - obj: The OpenAILLMContext instance to upgrade. - - Returns: - The upgraded OpenAIRealtimeLLMContext instance. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): - obj.__class__ = OpenAIRealtimeLLMContext - obj.__setup_local() - return obj - - # todo - # - finish implementing all frames - - def from_standard_message(self, message): - """Convert a standard message format to a realtime conversation item. - - Args: - message: The standard message dictionary to convert. - - Returns: - A ConversationItem instance for the realtime API. - """ - if message.get("role") == "user": - content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): - if c.get("type") == "text": - content += " " + c.get("text") - else: - logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" - ) - return events.ConversationItem( - role="user", - type="message", - content=[events.ItemContent(type="input_text", text=content)], - ) - if message.get("role") == "assistant" and message.get("tool_calls"): - tc = message.get("tool_calls")[0] - return events.ConversationItem( - type="function_call", - call_id=tc["id"], - name=tc["function"]["name"], - arguments=tc["function"]["arguments"], - ) - logger.error(f"Unhandled message type in from_standard_message: {message}") - - def get_messages_for_initializing_history(self): - """Get conversation items for initializing the realtime session history. - - Converts the context's messages to a format suitable for the realtime API, - handling system instructions and conversation history packaging. - - Returns: - List of conversation items for session initialization. - """ - # We can't load a long conversation history into the openai realtime api yet. (The API/model - # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So - # our general strategy until this is fixed is just to put everything into a first "user" - # message as a single input. - if not self.messages: - return [] - - messages = copy.deepcopy(self.messages) - - # If we have a "system" message as our first message, let's pull that out into session - # "instructions" - if messages[0].get("role") == "system": - self.llm_needs_settings_update = True - system = messages.pop(0) - content = system.get("content") - if isinstance(content, str): - self._session_instructions = content - elif isinstance(content, list): - self._session_instructions = content[0].get("text") - if not messages: - return [] - - # If we have just a single "user" item, we can just send it normally - if len(messages) == 1 and messages[0].get("role") == "user": - return [self.from_standard_message(messages[0])] - - # Otherwise, let's pack everything into a single "user" message with a bit of - # explanation for the LLM - intro_text = """ - This is a previously saved conversation. Please treat this conversation history as a - starting point for the current conversation.""" - - trailing_text = """ - This is the end of the previously saved conversation. Please continue the conversation - from here. If the last message is a user instruction or question, act on that instruction - or answer the question. If the last message is an assistant response, simple say that you - are ready to continue the conversation.""" - - return [ - { - "role": "user", - "type": "message", - "content": [ - { - "type": "input_text", - "text": "\n\n".join( - [intro_text, json.dumps(messages, indent=2), trailing_text] - ), - } - ], - } - ] - - def add_user_content_item_as_message(self, item): - """Add a user content item as a standard message to the context. - - Args: - item: The conversation item to add as a user message. - """ - message = { - "role": "user", - "content": [{"type": "text", "text": item.content[0].transcript}], - } - self.add_message(message) - - class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): """User context aggregator for OpenAI Realtime API. diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 8b3d500eb..f33b664f7 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -14,7 +14,9 @@ from typing import Optional from loguru import logger -from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter +from pipecat.adapters.services.open_ai_realtime_adapter import ( + OpenAIRealtimeLLMAdapter, +) from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, @@ -41,6 +43,7 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.metrics.metrics import LLMTokenUsage +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, @@ -59,7 +62,6 @@ from pipecat.utils.tracing.service_decorators import traced_openai_realtime, tra from . import events from .context import ( OpenAIRealtimeAssistantContextAggregator, - OpenAIRealtimeLLMContext, OpenAIRealtimeUserContextAggregator, ) from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame @@ -138,7 +140,17 @@ class OpenAIRealtimeLLMService(LLMService): self._send_transcription_frames = send_transcription_frames self._websocket = None self._receive_task = None - self._context = None + # "Last received context" is only needed while we still support + # OpenAILLMContextFrame. The "last received context" is the context received + # in the most recent OpenAILLMContextFrame or LLMContextFrame, *before* + # it's converted to an LLMContext if needed. Storing the "last received + # context" lets us determine whether the context has changed. (We can't + # compare contexts after conversion because conversion creates a new + # object.) + self._context: LLMContext = None + self._last_received_context: OpenAILLMContext | LLMContext = None + + self._llm_needs_conversation_setup = True self._disconnecting = False self._api_session_ready = False @@ -347,22 +359,22 @@ class OpenAIRealtimeLLMService(LLMService): if isinstance(frame, TranscriptionFrame): pass - elif isinstance(frame, OpenAILLMContextFrame): - context: OpenAIRealtimeLLMContext = OpenAIRealtimeLLMContext.upgrade_to_realtime( + elif isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + context = ( frame.context + if isinstance(frame, LLMContextFrame) + else LLMContext.from_openai_context(frame.context) ) if not self._context: + self._last_received_context = frame.context self._context = context - elif frame.context is not self._context: + elif frame.context is not self._last_received_context: # If the context has changed, reset the conversation + self._last_received_context = frame.context self._context = context await self.reset_conversation() # Run the LLM at next opportunity await self._create_response() - elif isinstance(frame, LLMContextFrame): - raise NotImplementedError( - "Universal LLMContext is not yet supported for OpenAI Realtime." - ) elif isinstance(frame, InputAudioRawFrame): if not self._audio_input_paused: await self._send_user_audio(frame) @@ -377,6 +389,7 @@ class OpenAIRealtimeLLMService(LLMService): elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) elif isinstance(frame, RealtimeMessagesUpdateFrame): + # TODO: we don't need RealtimeMessagesUpdateFrame, I think...? self._context = frame.context elif isinstance(frame, LLMUpdateSettingsFrame): self._session_properties = events.SessionProperties(**frame.settings) @@ -459,13 +472,20 @@ class OpenAIRealtimeLLMService(LLMService): async def _update_settings(self): settings = self._session_properties - # tools given in the context override the tools in the session properties - if self._context and self._context.tools: - settings.tools = self._context.tools - # instructions in the context come from an initial "system" message in the - # messages list, and override instructions in the session properties - if self._context and self._context._session_instructions: - settings.instructions = self._context._session_instructions + + if self._context: + adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() + llm_invocation_params = adapter.get_llm_invocation_params(self._context) + + # tools given in the context override the tools in the session properties + if llm_invocation_params["tools"]: + settings.tools = llm_invocation_params["tools"] + + # instructions in the context come from an initial "system" message in the + # messages list, and override instructions in the session properties + if llm_invocation_params["system_instruction"]: + settings.instructions = llm_invocation_params["system_instruction"] + await self.send_client_event(events.SessionUpdateEvent(session=settings)) # @@ -760,9 +780,7 @@ class OpenAIRealtimeLLMService(LLMService): """ logger.debug("Resetting conversation") await self._disconnect() - if self._context: - self._context.llm_needs_settings_update = True - self._context.llm_needs_initial_messages = True + self._llm_needs_conversation_setup = True await self._connect() @traced_openai_realtime(operation="llm_request") @@ -771,19 +789,25 @@ class OpenAIRealtimeLLMService(LLMService): self._run_llm_when_api_session_ready = True return - if self._context.llm_needs_initial_messages: - messages = self._context.get_messages_for_initializing_history() + adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() + + # Configure the LLM for this session if needed + if self._llm_needs_conversation_setup: + # Send initial messages + llm_invocation_params = adapter.get_llm_invocation_params(self._context) + messages = llm_invocation_params["messages"] for item in messages: evt = events.ConversationItemCreateEvent(item=item) self._messages_added_manually[evt.item.id] = True await self.send_client_event(evt) - self._context.llm_needs_initial_messages = False - if self._context.llm_needs_settings_update: + # Send new settings if needed await self._update_settings() - self._context.llm_needs_settings_update = False - logger.debug(f"Creating response: {self._context.get_messages_for_logging()}") + # We're done configuring the LLM for this session + self._llm_needs_conversation_setup = False + + logger.debug(f"Creating response: {adapter.get_messages_for_logging(self._context)}") await self.push_frame(LLMFullResponseStartFrame()) await self.start_processing_metrics() diff --git a/src/pipecat/services/openai_realtime/context.py b/src/pipecat/services/openai_realtime/context.py deleted file mode 100644 index 58f1cfe75..000000000 --- a/src/pipecat/services/openai_realtime/context.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -"""OpenAI Realtime LLM context and aggregator implementations.""" - -import warnings - -from pipecat.services.openai.realtime.context import * - -with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Types in pipecat.services.openai_realtime.context are deprecated. " - "Please use the equivalent types from " - "pipecat.services.openai.realtime.context instead.", - DeprecationWarning, - stacklevel=2, - ) From 29fd17b9ff57b50442544c3908e21cb461d8ca15 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 13:29:27 -0400 Subject: [PATCH 60/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Avoid pushing `LLMTextFrame` when `OpenAIRealtimeLLMService` is configured to output audio. This avoids duplicate text in assistant messages in context. Conceptually, a speech-to-speech service encapsulates TTS behavior; in a "traditional" pipeline, `LLMTextFrames` are swallowed by the TTS service, so they should similarly not be pushed by a speech-to-speech service. Only. `TTSTextFrame`s should be pushed. --- src/pipecat/services/openai/realtime/llm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index f33b664f7..ace8be655 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -687,12 +687,15 @@ class OpenAIRealtimeLLMService(LLMService): logger.debug(f"Handling standalone response: {evt.response.id}") async def _handle_evt_text_delta(self, evt): + # We receive text deltas (as opposed to audio transcript deltas) when + # the output modality is "text" if evt.delta: await self.push_frame(LLMTextFrame(evt.delta)) async def _handle_evt_audio_transcript_delta(self, evt): + # We receive audio transcript deltas (as opposed to text deltas) when + # the output modality is "audio" (the default) if evt.delta: - await self.push_frame(LLMTextFrame(evt.delta)) await self.push_frame(TTSTextFrame(evt.delta)) async def _handle_evt_function_call_arguments_done(self, evt): From ec42f8c24e034316f605f03106d7decbe4a1a47f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 14:54:15 -0400 Subject: [PATCH 61/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Push `TranscriptionFrame`s upstream, to be handled by the user context aggregator. This will require at least a couple of other changes: - Updating examples to put transcript processors upstream from `OpenAIRealtimeLLMService` - Maybe figuring out a way to preserve backward compatibility with existing pipelines that put transcript processors downstream from `OpenAIRealtimeLLMService` - Updating `OpenAIRealtimeLLMService` to ignore new received context frames, since the upstream user context aggregator will generate those after each newly-added user message; hopefully nobody was reliant on the old behavior of resetting the conversation upon receiving a new context! --- examples/foundational/19-openai-realtime.py | 2 +- src/pipecat/services/openai/realtime/llm.py | 22 ++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 685070d5e..3bf12f916 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -176,8 +176,8 @@ Remember, your responses should be short. Just one or two sentences, usually. Re [ transport.input(), # Transport user input context_aggregator.user(), + transcript.user(), # LLM pushes TranscriptionFrames upstream llm, # LLM - transcript.user(), # Placed after the LLM, as LLM pushes TranscriptionFrames downstream transport.output(), # Transport bot output transcript.assistant(), # After the transcript output, to time with the audio output context_aggregator.assistant(), diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index ace8be655..7e65cc2f7 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -607,11 +607,11 @@ class OpenAIRealtimeLLMService(LLMService): # For now, no additional logic needed beyond the event handler call async def _handle_evt_input_audio_transcription_delta(self, evt): - if self._send_transcription_frames: - await self.push_frame( - # no way to get a language code? - InterimTranscriptionFrame(evt.delta, "", time_now_iso8601(), result=evt) - ) + await self.push_frame( + # no way to get a language code? + InterimTranscriptionFrame(evt.delta, "", time_now_iso8601(), result=evt), + direction=FrameDirection.UPSTREAM, + ) @traced_stt async def _handle_user_transcription( @@ -628,12 +628,12 @@ class OpenAIRealtimeLLMService(LLMService): """ await self._call_event_handler("on_conversation_item_updated", evt.item_id, None) - if self._send_transcription_frames: - await self.push_frame( - # no way to get a language code? - TranscriptionFrame(evt.transcript, "", time_now_iso8601(), result=evt) - ) - await self._handle_user_transcription(evt.transcript, True, Language.EN) + await self.push_frame( + # no way to get a language code? + TranscriptionFrame(evt.transcript, "", time_now_iso8601(), result=evt), + FrameDirection.UPSTREAM, + ) + await self._handle_user_transcription(evt.transcript, True, Language.EN) pair = self._user_and_response_message_tuple if pair: user, assistant = pair From 8a151235c36fbf4a6d8e3f2be86cd64520962e5b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 15:47:46 -0400 Subject: [PATCH 62/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deprecate `send_transcription_frames`—transcription frames are now always sent. --- src/pipecat/services/openai/realtime/llm.py | 22 ++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 7e65cc2f7..f5f54f8dd 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -110,7 +110,7 @@ class OpenAIRealtimeLLMService(LLMService): base_url: str = "wss://api.openai.com/v1/realtime", session_properties: Optional[events.SessionProperties] = None, start_audio_paused: bool = False, - send_transcription_frames: bool = True, + send_transcription_frames: Optional[bool] = None, **kwargs, ): """Initialize the OpenAI Realtime LLM service. @@ -123,9 +123,26 @@ class OpenAIRealtimeLLMService(LLMService): session_properties: Configuration properties for the realtime session. If None, uses default SessionProperties. start_audio_paused: Whether to start with audio input paused. Defaults to False. - send_transcription_frames: Whether to emit transcription frames. Defaults to True. + send_transcription_frames: Whether to emit transcription frames. + + .. deprecated:: 0.0.92 + This parameter is deprecated and will be removed in a future version. + Transcription frames are always sent. + **kwargs: Additional arguments passed to parent LLMService. """ + if send_transcription_frames is not None: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "`send_transcription_frames` is deprecated and will be removed in a future version. " + "Transcription frames are always sent.", + DeprecationWarning, + stacklevel=2, + ) + full_url = f"{base_url}?model={model}" super().__init__(base_url=full_url, **kwargs) @@ -137,7 +154,6 @@ class OpenAIRealtimeLLMService(LLMService): session_properties or events.SessionProperties() ) self._audio_input_paused = start_audio_paused - self._send_transcription_frames = send_transcription_frames self._websocket = None self._receive_task = None # "Last received context" is only needed while we still support From 5fa56df01409ae47ff3a34d5b63ad0d4da0dc305 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 15:51:00 -0400 Subject: [PATCH 63/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Update 19b example with new pattern. --- examples/foundational/19b-openai-realtime-text.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py index bb63a4814..fab029bb6 100644 --- a/examples/foundational/19b-openai-realtime-text.py +++ b/examples/foundational/19b-openai-realtime-text.py @@ -18,6 +18,8 @@ from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments @@ -169,20 +171,20 @@ Remember, your responses should be short. Just one or two sentences, usually. Re # Create a standard OpenAI LLM context object using the normal messages format. The # OpenAIRealtimeLLMService will convert this internally to messages that the # openai WebSocket API can understand. - context = OpenAILLMContext( + context = LLMContext( [{"role": "user", "content": "Say hello!"}], tools, ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ transport.input(), # Transport user input context_aggregator.user(), + transcript.user(), # LLM pushes TranscriptionFrames upstream llm, # LLM tts, # TTS - transcript.user(), # Placed after the LLM, as LLM pushes TranscriptionFrames downstream transport.output(), # Transport bot output transcript.assistant(), # After the transcript output, to time with the audio output context_aggregator.assistant(), From 47756319beda42cb4ae107ff45916ddba769cc19 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 16:04:09 -0400 Subject: [PATCH 64/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Receiving a new context (via a context frame) no longer serves as a signal to reset the conversation. That’s because we’re now receiving new contexts from the user aggregator every time new messages are added, and from the assistant aggregator when function call results come in. The code pattern we're heading towards, of “diffing” each new context with the previous on, sets us up for doing more sophisticated things in the future, like sending specific messages to OpenAI to edit its internally-tracked context. Also, remove code that was directly modifying context. --- src/pipecat/services/openai/realtime/llm.py | 54 ++++----------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index f5f54f8dd..a9d890bae 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -156,15 +156,7 @@ class OpenAIRealtimeLLMService(LLMService): self._audio_input_paused = start_audio_paused self._websocket = None self._receive_task = None - # "Last received context" is only needed while we still support - # OpenAILLMContextFrame. The "last received context" is the context received - # in the most recent OpenAILLMContextFrame or LLMContextFrame, *before* - # it's converted to an LLMContext if needed. Storing the "last received - # context" lets us determine whether the context has changed. (We can't - # compare contexts after conversion because conversion creates a new - # object.) self._context: LLMContext = None - self._last_received_context: OpenAILLMContext | LLMContext = None self._llm_needs_conversation_setup = True @@ -176,7 +168,6 @@ class OpenAIRealtimeLLMService(LLMService): self._current_audio_response = None self._messages_added_manually = {} - self._user_and_response_message_tuple = None self._pending_function_calls = {} # Track function calls by call_id self._register_event_handler("on_conversation_item_created") @@ -382,15 +373,15 @@ class OpenAIRealtimeLLMService(LLMService): else LLMContext.from_openai_context(frame.context) ) if not self._context: - self._last_received_context = frame.context + # We got our initial context + # Run the LLM at next opportunity self._context = context - elif frame.context is not self._last_received_context: - # If the context has changed, reset the conversation - self._last_received_context = frame.context - self._context = context - await self.reset_conversation() - # Run the LLM at next opportunity - await self._create_response() + await self._create_response() + else: + # We got an updated context + # Send results for any newly-completed function calls + # TODO: to implement + pass elif isinstance(frame, InputAudioRawFrame): if not self._audio_input_paused: await self._send_user_audio(frame) @@ -607,12 +598,7 @@ class OpenAIRealtimeLLMService(LLMService): del self._messages_added_manually[evt.item.id] return - if evt.item.role == "user": - # We need to wait for completion of both user message and response message. Then we'll - # add both to the context. User message is complete when we have a "transcript" field - # that is not None. Response message is complete when we get a "response.done" event. - self._user_and_response_message_tuple = (evt.item, {"done": False, "output": []}) - elif evt.item.role == "assistant": + if evt.item.role == "assistant": self._current_assistant_response = evt.item await self.push_frame(LLMFullResponseStartFrame()) @@ -650,16 +636,6 @@ class OpenAIRealtimeLLMService(LLMService): FrameDirection.UPSTREAM, ) await self._handle_user_transcription(evt.transcript, True, Language.EN) - pair = self._user_and_response_message_tuple - if pair: - user, assistant = pair - user.content[0].transcript = evt.transcript - if assistant["done"]: - self._user_and_response_message_tuple = None - self._context.add_user_content_item_as_message(user) - else: - # User message without preceding conversation.item.created. Bug? - logger.warning(f"Transcript for unknown user message: {evt}") async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved): futures = self._retrieve_conversation_item_futures.pop(evt.item.id, None) @@ -689,18 +665,6 @@ class OpenAIRealtimeLLMService(LLMService): # response content for item in evt.response.output: await self._call_event_handler("on_conversation_item_updated", item.id, item) - pair = self._user_and_response_message_tuple - if pair: - user, assistant = pair - assistant["done"] = True - assistant["output"] = evt.response.output - if user.content[0].transcript is not None: - self._user_and_response_message_tuple = None - self._context.add_user_content_item_as_message(user) - else: - # Response message without preceding user message (standalone response) - # Function calls in this response were already processed immediately when arguments were complete - logger.debug(f"Handling standalone response: {evt.response.id}") async def _handle_evt_text_delta(self, evt): # We receive text deltas (as opposed to audio transcript deltas) when From 61944d22ef148b16b5864c79bece7c2a98451910 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 16:35:04 -0400 Subject: [PATCH 65/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Implement sending tool call results to the OpenAI server based on reading context updates. This lets us use the normal assistant context aggregator and not a special OpenAI Realtime subclass that pushes up a special frame for function call results. --- .../services/open_ai_realtime_adapter.py | 2 +- src/pipecat/services/openai/realtime/llm.py | 67 ++++++++++++------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 67bdfb6ae..58cf284b1 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -129,7 +129,7 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): or answer the question. If the last message is an assistant response, simple say that you are ready to continue the conversation.""" - self.ConvertedMessages( + return self.ConvertedMessages( messages=[ { "role": "user", diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index a9d890bae..a3421cb75 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -169,6 +169,7 @@ class OpenAIRealtimeLLMService(LLMService): self._messages_added_manually = {} self._pending_function_calls = {} # Track function calls by call_id + self._completed_tool_calls = set() self._register_event_handler("on_conversation_item_created") self._register_event_handler("on_conversation_item_updated") @@ -372,16 +373,7 @@ class OpenAIRealtimeLLMService(LLMService): if isinstance(frame, LLMContextFrame) else LLMContext.from_openai_context(frame.context) ) - if not self._context: - # We got our initial context - # Run the LLM at next opportunity - self._context = context - await self._create_response() - else: - # We got an updated context - # Send results for any newly-completed function calls - # TODO: to implement - pass + await self._handle_context(context) elif isinstance(frame, InputAudioRawFrame): if not self._audio_input_paused: await self._send_user_audio(frame) @@ -395,30 +387,32 @@ class OpenAIRealtimeLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, LLMMessagesAppendFrame): await self._handle_messages_append(frame) - elif isinstance(frame, RealtimeMessagesUpdateFrame): - # TODO: we don't need RealtimeMessagesUpdateFrame, I think...? - self._context = frame.context elif isinstance(frame, LLMUpdateSettingsFrame): self._session_properties = events.SessionProperties(**frame.settings) await self._update_settings() elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() - elif isinstance(frame, RealtimeFunctionCallResultFrame): - await self._handle_function_call_result(frame.result_frame) await self.push_frame(frame, direction) + async def _handle_context(self, context: LLMContext): + if not self._context: + # We got our initial context + self._context = context + # Initialize our bookkeeping of already-completed tool calls in + # the context + await self._process_completed_function_calls(send_new_results=False) + # Run the LLM at next opportunity + await self._create_response() + else: + # We got an updated context + self._context = context + # Send results for any newly-completed function calls + await self._process_completed_function_calls(send_new_results=True) + async def _handle_messages_append(self, frame): logger.error("!!! NEED TO IMPLEMENT MESSAGES APPEND") - async def _handle_function_call_result(self, frame): - item = events.ConversationItem( - type="function_call_output", - call_id=frame.tool_call_id, - output=json.dumps(frame.result), - ) - await self.send_client_event(events.ConversationItemCreateEvent(item=item)) - # # websocket communication # @@ -459,6 +453,7 @@ class OpenAIRealtimeLLMService(LLMService): if self._receive_task: await self.cancel_task(self._receive_task, timeout=1.0) self._receive_task = None + self._completed_tool_calls = set() self._disconnecting = False except Exception as e: logger.error(f"{self} error disconnecting: {e}") @@ -801,10 +796,36 @@ class OpenAIRealtimeLLMService(LLMService): ) ) + async def _process_completed_function_calls(self, send_new_results: bool): + # Check for set of completed function calls in the context + sent_new_result = False + for message in self._context.get_messages(): + if message.get("role") and message.get("content") != "IN_PROGRESS": + tool_call_id = message.get("tool_call_id") + if tool_call_id and tool_call_id not in self._completed_tool_calls: + # Found a newly-completed function call - send the result to the service + if send_new_results: + sent_new_result = True + await self._send_tool_result(tool_call_id, message.get("content")) + self._completed_tool_calls.add(tool_call_id) + + # If we sent any new tool call results to the service, trigger another + # response + if sent_new_result: + await self._create_response() + async def _send_user_audio(self, frame): payload = base64.b64encode(frame.audio).decode("utf-8") await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) + async def _send_tool_result(self, tool_call_id: str, result: str): + item = events.ConversationItem( + type="function_call_output", + call_id=tool_call_id, + output=json.dumps(result), + ) + await self.send_client_event(events.ConversationItemCreateEvent(item=item)) + def create_context_aggregator( self, context: OpenAILLMContext, From bab0aaf585a4eb4ad089f36d7f9529661aacc95c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 16:58:44 -0400 Subject: [PATCH 66/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Update `create_context_aggregator()` (which we're keeping around for backward compatibility) to create a `LLMContextAggregatorPair` rather than OpenAI-Realtime-specific aggregators. --- .../services/openai/realtime/context.py | 243 +++++++++++++++++- .../services/openai/realtime/frames.py | 23 +- src/pipecat/services/openai/realtime/llm.py | 25 +- 3 files changed, 275 insertions(+), 16 deletions(-) diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py index da08f78ad..57979406c 100644 --- a/src/pipecat/services/openai/realtime/context.py +++ b/src/pipecat/services/openai/realtime/context.py @@ -4,7 +4,94 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""OpenAI Realtime LLM context and aggregator implementations.""" +"""OpenAI Realtime LLM context and aggregator implementations. + +.. deprecated:: 0.0.92 + OpenAI Realtime no longer uses types from this module under the hood. + It now uses `LLMContext` and `LLMContextAggregatorPair`. + Using the new patterns should allow you to not need types from this module. + + BEFORE: + ``` + # Setup + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + # Context aggregator type + context_aggregator: OpenAIContextAggregatorPair + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: OpenAIRealtimeLLMContext + # or + context: OpenAILLMContext + + # Reading messages from context + messages = context.messages + ``` + + AFTER: + ``` + # Setup + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + # Context aggregator type + context_aggregator: LLMContextAggregatorPair + + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + + # Reading messages from context + messages = context.get_messages() + ``` +""" + +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai.realtime.llm are deprecated. \n" + "OpenAI Realtime no longer uses types from this module under the hood. \n" + "It now uses `LLMContext` and `LLMContextAggregatorPair`. \n" + "Using the new patterns should allow you to not need types from this module.\n\n" + "BEFORE:\n" + "```\n" + "# Setup\n" + "context = OpenAILLMContext(messages, tools)\n" + "context_aggregator = llm.create_context_aggregator(context)\n\n" + "# Context aggregator type\n" + "context_aggregator: OpenAIContextAggregatorPair\n\n" + "# Context frame type\n" + "frame: OpenAILLMContextFrame\n\n" + "# Context type\n" + "context: OpenAIRealtimeLLMContext\n" + "# or\n" + "context: OpenAILLMContext\n\n" + "# Reading messages from context\n" + "messages = context.messages\n" + "```\n\n" + "AFTER:\n" + "```\n" + "# Setup\n" + "context = LLMContext(messages, tools)\n" + "context_aggregator = LLMContextAggregatorPair(context)\n\n" + "# Context aggregator type\n" + "context_aggregator: LLMContextAggregatorPair\n\n" + "# Context frame type\n" + "frame: LLMContextFrame\n\n" + "# Context type\n" + "context: LLMContext\n\n" + "# Reading messages from context\n" + "messages = context.get_messages()\n" + "```\n", + ) import copy import json @@ -31,6 +118,160 @@ from . import events from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame +class OpenAIRealtimeLLMContext(OpenAILLMContext): + """OpenAI Realtime LLM context with session management and message conversion. + + Extends the standard OpenAI LLM context to support real-time session properties, + instruction management, and conversion between standard message formats and + realtime conversation items. + """ + + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize the OpenAIRealtimeLLMContext. + + Args: + messages: Initial conversation messages. Defaults to None. + tools: Available function tools. Defaults to None. + **kwargs: Additional arguments passed to parent OpenAILLMContext. + """ + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self): + self.llm_needs_settings_update = True + self.llm_needs_initial_messages = True + self._session_instructions = "" + + return + + @staticmethod + def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": + """Upgrade a standard OpenAI LLM context to a realtime context. + + Args: + obj: The OpenAILLMContext instance to upgrade. + + Returns: + The upgraded OpenAIRealtimeLLMContext instance. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): + obj.__class__ = OpenAIRealtimeLLMContext + obj.__setup_local() + return obj + + # todo + # - finish implementing all frames + + def from_standard_message(self, message): + """Convert a standard message format to a realtime conversation item. + + Args: + message: The standard message dictionary to convert. + + Returns: + A ConversationItem instance for the realtime API. + """ + if message.get("role") == "user": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + return events.ConversationItem( + role="user", + type="message", + content=[events.ItemContent(type="input_text", text=content)], + ) + if message.get("role") == "assistant" and message.get("tool_calls"): + tc = message.get("tool_calls")[0] + return events.ConversationItem( + type="function_call", + call_id=tc["id"], + name=tc["function"]["name"], + arguments=tc["function"]["arguments"], + ) + logger.error(f"Unhandled message type in from_standard_message: {message}") + + def get_messages_for_initializing_history(self): + """Get conversation items for initializing the realtime session history. + + Converts the context's messages to a format suitable for the realtime API, + handling system instructions and conversation history packaging. + + Returns: + List of conversation items for session initialization. + """ + # We can't load a long conversation history into the openai realtime api yet. (The API/model + # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So + # our general strategy until this is fixed is just to put everything into a first "user" + # message as a single input. + if not self.messages: + return [] + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into session + # "instructions" + if messages[0].get("role") == "system": + self.llm_needs_settings_update = True + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + self._session_instructions = content + elif isinstance(content, list): + self._session_instructions = content[0].get("text") + if not messages: + return [] + + # If we have just a single "user" item, we can just send it normally + if len(messages) == 1 and messages[0].get("role") == "user": + return [self.from_standard_message(messages[0])] + + # Otherwise, let's pack everything into a single "user" message with a bit of + # explanation for the LLM + intro_text = """ + This is a previously saved conversation. Please treat this conversation history as a + starting point for the current conversation.""" + + trailing_text = """ + This is the end of the previously saved conversation. Please continue the conversation + from here. If the last message is a user instruction or question, act on that instruction + or answer the question. If the last message is an assistant response, simple say that you + are ready to continue the conversation.""" + + return [ + { + "role": "user", + "type": "message", + "content": [ + { + "type": "input_text", + "text": "\n\n".join( + [intro_text, json.dumps(messages, indent=2), trailing_text] + ), + } + ], + } + ] + + def add_user_content_item_as_message(self, item): + """Add a user content item as a standard message to the context. + + Args: + item: The conversation item to add as a user message. + """ + message = { + "role": "user", + "content": [{"type": "text", "text": item.content[0].transcript}], + } + self.add_message(message) + + class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): """User context aggregator for OpenAI Realtime API. diff --git a/src/pipecat/services/openai/realtime/frames.py b/src/pipecat/services/openai/realtime/frames.py index 8617c6efd..39cfd9757 100644 --- a/src/pipecat/services/openai/realtime/frames.py +++ b/src/pipecat/services/openai/realtime/frames.py @@ -4,7 +4,28 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Custom frame types for OpenAI Realtime API integration.""" +"""Custom frame types for OpenAI Realtime API integration. + +.. deprecated:: 0.0.92 + OpenAI Realtime no longer uses types from this module under the hood. + + It now works more like most LLM services in Pipecat, relying on updates to + its context, pushed by context aggregators, to update its internal state. + + Listen for `LLMContextFrame`s for context updates. +""" + +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai.realtime.frames are deprecated. \n" + "OpenAI Realtime no longer uses types from this module under the hood. \n\n" + "It now works more like other LLM services in Pipecat, relying on updates to \n" + "its context, pushed by context aggregators, to update its internal state.\n\n" + "Listen for `LLMContextFrame`s for context updates.\n" + ) from dataclasses import dataclass from typing import TYPE_CHECKING diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index a3421cb75..eb2ba5ef4 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -48,6 +48,7 @@ from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, ) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, @@ -60,11 +61,6 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt from . import events -from .context import ( - OpenAIRealtimeAssistantContextAggregator, - OpenAIRealtimeUserContextAggregator, -) -from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame try: from websockets.asyncio.client import connect as websocket_connect @@ -832,9 +828,14 @@ class OpenAIRealtimeLLMService(LLMService): *, user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), - ) -> OpenAIContextAggregatorPair: + ) -> LLMContextAggregatorPair: """Create an instance of OpenAIContextAggregatorPair from an OpenAILLMContext. + NOTE: this method exists only for backward compatibility. New code + should instead do: + context = LLMContext(...) + context_aggregator = LLMContextAggregatorPair(context) + Constructor keyword arguments for both the user and assistant aggregators can be provided. Args: @@ -847,11 +848,7 @@ class OpenAIRealtimeLLMService(LLMService): the user and one for the assistant, encapsulated in an OpenAIContextAggregatorPair. """ - context.set_llm_adapter(self.get_llm_adapter()) - - OpenAIRealtimeLLMContext.upgrade_to_realtime(context) - user = OpenAIRealtimeUserContextAggregator(context, params=user_params) - - assistant_params.expect_stripped_words = False - assistant = OpenAIRealtimeAssistantContextAggregator(context, params=assistant_params) - return OpenAIContextAggregatorPair(_user=user, _assistant=assistant) + context = LLMContext.from_openai_context(context) + return LLMContextAggregatorPair( + context, user_params=user_params, assistant_params=assistant_params + ) From b34461bf937030be3bcccf148d884ac60eb14314 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 20 Oct 2025 17:36:30 -0400 Subject: [PATCH 67/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). --- CHANGELOG.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 359b2a2f4..a152022f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,85 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Expanded support for univeral `LLMContext` to `OpenAIRealtimeLLMService`. + As a reminder, the context-setup pattern when using `LLMContext` is: + + ```python + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + ``` + + (Note that even though `OpenAIRealtimeLLMService` now supports the universal + `LLMContext`, it is not meant to be swapped out for another LLM service at + runtime.) + + Note: `TranscriptionFrame`s now go upstream from `OpenAIRealtimeLLMService`, + so if you're using `TranscriptProcessor`, say, you'll want to adjust + accordingly: + + ```python + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + + # BEFORE + llm, + transcript.user(), + + # AFTER + transcript.user(), + llm, + + transport.output(), + transcript.assistant(), + context_aggregator.assistant(), + ] + ) + ``` + + Also worth noting: whether or not you use the new context-setup pattern with + `OpenAIRealtimeLLMService`, some types have changed under the hood: + + ```python + ## BEFORE: + + # Context aggregator type + context_aggregator: OpenAIContextAggregatorPair + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: OpenAIRealtimeLLMContext + # or + context: OpenAILLMContext + + # Reading messages from context + messages = context.messages + + ## AFTER: + + # Context aggregator type + context_aggregator: LLMContextAggregatorPair + + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + + # Reading messages from context + messages = context.get_messages() + ``` + + Also note that `RealtimeMessagesUpdateFrame` and + `RealtimeFunctionCallResultFrame` have been deprecated, since they're no + longer used by `OpenAIRealtimeLLMService`. OpenAI Realtime now works more + like other LLM services in Pipecat, relying on updates to its context, pushed + by context aggregators, to update its internal state. Listen for + `LLMContextFrame`s for context updates. + - Expanded support for universal `LLMContext` to `GeminiLiveLLMService`. As a reminder, the context-setup pattern when using `LLMContext` is: @@ -79,6 +158,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +- The `send_transcription_frames` argument to `OpenAIRealtimeLLMService` is + deprecated. Transcription frames are now always sent. They go upstream, to be + handled by the user context aggregator. See "Added" section for details. + +- Types in `pipecat.services.openai.realtime.context` and + `pipecat.services.openai.realtime.frames` are deprecated, as they're no + longer used by `OpenAIRealtimeLLMService`. See "Added" section for details. + - `SimliVideoService` `simli_config` parameter is deprecated. Use `api_key` and `face_id` parameters instead. @@ -200,8 +287,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 deprecated. Transcription frames are now always sent. They go upstream, to be handled by the user context aggregator. See "Added" section for details. -- Types in `pipecat.services.aws.nova_sonic.context` have been deprecated due - to changes to support `LLMContext`. See "Changed" section for details. +- Types in `pipecat.services.aws.nova_sonic.context` are deprecated, as they're + no longer used by `AWSNovaSonicLLMService`. See "Added" section for + details. ### Fixed @@ -5375,4 +5463,4 @@ a bit. ## [0.0.2] - 2024-03-12 -Initial public release. \ No newline at end of file +Initial public release. From 19770b76b499a8e2c2783b22f4819845912e7950 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 10:15:11 -0400 Subject: [PATCH 68/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Add back file that was removed, when it should've just been deprecated. Also, fix version numbers in deprecation messages to match the next expected release. --- .../services/openai/realtime/context.py | 5 +++-- src/pipecat/services/openai/realtime/frames.py | 2 +- src/pipecat/services/openai/realtime/llm.py | 2 +- .../services/openai_realtime/context.py | 18 ++++++++++++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 src/pipecat/services/openai_realtime/context.py diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py index 57979406c..96a714565 100644 --- a/src/pipecat/services/openai/realtime/context.py +++ b/src/pipecat/services/openai/realtime/context.py @@ -6,7 +6,7 @@ """OpenAI Realtime LLM context and aggregator implementations. -.. deprecated:: 0.0.92 +.. deprecated:: 0.0.91 OpenAI Realtime no longer uses types from this module under the hood. It now uses `LLMContext` and `LLMContextAggregatorPair`. Using the new patterns should allow you to not need types from this module. @@ -57,7 +57,8 @@ import warnings with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( - "Types in pipecat.services.openai.realtime.llm are deprecated. \n" + "Types in pipecat.services.openai.realtime.llm (or " + "pipecat.services.openai_realtime.llm) are deprecated. \n" "OpenAI Realtime no longer uses types from this module under the hood. \n" "It now uses `LLMContext` and `LLMContextAggregatorPair`. \n" "Using the new patterns should allow you to not need types from this module.\n\n" diff --git a/src/pipecat/services/openai/realtime/frames.py b/src/pipecat/services/openai/realtime/frames.py index 39cfd9757..1f800af89 100644 --- a/src/pipecat/services/openai/realtime/frames.py +++ b/src/pipecat/services/openai/realtime/frames.py @@ -6,7 +6,7 @@ """Custom frame types for OpenAI Realtime API integration. -.. deprecated:: 0.0.92 +.. deprecated:: 0.0.91 OpenAI Realtime no longer uses types from this module under the hood. It now works more like most LLM services in Pipecat, relying on updates to diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index eb2ba5ef4..5e042cff3 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -121,7 +121,7 @@ class OpenAIRealtimeLLMService(LLMService): start_audio_paused: Whether to start with audio input paused. Defaults to False. send_transcription_frames: Whether to emit transcription frames. - .. deprecated:: 0.0.92 + .. deprecated:: 0.0.91 This parameter is deprecated and will be removed in a future version. Transcription frames are always sent. diff --git a/src/pipecat/services/openai_realtime/context.py b/src/pipecat/services/openai_realtime/context.py new file mode 100644 index 000000000..79a01b980 --- /dev/null +++ b/src/pipecat/services/openai_realtime/context.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""OpenAI Realtime LLM context and aggregator implementations. + +.. deprecated:: 0.0.91 + OpenAI Realtime no longer uses types from this module under the hood. + It now uses `LLMContext` and `LLMContextAggregatorPair`. + Using the new patterns should allow you to not need types from this module. + + See deprecation warning in pipecat.services.openai.realtime.context for + more details. +""" + +from pipecat.services.openai.realtime.context import * From 46e97c57c26c0ecb17f5d94d6c8175abacf59616 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 11:14:40 -0400 Subject: [PATCH 69/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Update 20b example to use new `LLMContext` pattern. --- .../20b-persistent-context-openai-realtime.py | 78 ++++++++----------- .../services/open_ai_realtime_adapter.py | 2 +- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 629a17c67..33328ac24 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -13,11 +13,15 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) @@ -97,14 +101,12 @@ async def load_conversation(params: FunctionCallParams): asyncio.create_task(_reset()) -tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { +tools = ToolsSchema( + standard_tools=[ + FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ "location": { "type": "string", "description": "The city and state, e.g. San Francisco, CA", @@ -115,45 +117,33 @@ tools = [ "description": "The temperature unit to use. Infer this from the users location.", }, }, - "required": ["location", "format"], - }, - }, - { - "type": "function", - "name": "save_conversation", - "description": "Save the current conversatione. Use this function to persist the current conversation to external storage.", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - { - "type": "function", - "name": "get_saved_conversation_filenames", - "description": "Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - { - "type": "function", - "name": "load_conversation", - "description": "Load a conversation history. Use this function to load a conversation history into the current session.", - "parameters": { - "type": "object", - "properties": { + required=["location", "format"], + ), + FunctionSchema( + name="save_conversation", + description="Save the current conversatione. Use this function to persist the current conversation to external storage.", + properties={}, + required=[], + ), + FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], + ), + FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ "filename": { "type": "string", "description": "The filename of the conversation history to load.", } }, - "required": ["filename"], - }, - }, -] + required=["filename"], + ), + ] +) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -224,8 +214,8 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext([], tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext([], tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 58cf284b1..7a6fc4d02 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -93,7 +93,7 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): # message as a single input. if not universal_context_messages: - return self.ConvertedMessages() + return self.ConvertedMessages(messages=[]) messages = copy.deepcopy(universal_context_messages) system_instruction = None From 376180414664aba011d1f3a36783e474f3033a22 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 11:41:02 -0400 Subject: [PATCH 70/93] Make `OpenAIRealtimeLLMService`'s websocket send method more resilient. Previously, it was possible for a websocket send attempt to occur during a disconnect. --- src/pipecat/services/openai/realtime/llm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 5e042cff3..3d64f6ccb 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -456,10 +456,14 @@ class OpenAIRealtimeLLMService(LLMService): async def _ws_send(self, realtime_message): try: - if self._websocket: + if not self._disconnecting and self._websocket: await self._websocket.send(json.dumps(realtime_message)) except Exception as e: - if self._disconnecting: + if self._disconnecting or not self._websocket: + # We're in the process of disconnecting. + # (If not self._websocket, that could indicate that we + # somehow *started* the websocket send attempt while we still + # had a connection) return logger.error(f"Error sending message to websocket: {e}") # In server-to-server contexts, a WebSocket error should be quite rare. Given how hard From 42d0a097c5d7f80a8ffeea3c88f64f00946c0cd4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 16:29:13 -0400 Subject: [PATCH 71/93] Tweaks to 20b example --- .../foundational/20b-persistent-context-openai-realtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 33328ac24..c0aa9bc95 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -73,7 +73,7 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: @@ -214,7 +214,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = LLMContext([], tools) + context = LLMContext([{"role": "user", "content": "Say hello!"}], tools) context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( From b6a1886daef5d444760d7b2774a2a0c1edf5209c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 17:00:40 -0400 Subject: [PATCH 72/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). --- .../20b-persistent-context-openai-realtime.py | 6 +++++- src/pipecat/services/openai/realtime/llm.py | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index c0aa9bc95..346a5b4bd 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -77,7 +77,7 @@ async def save_conversation(params: FunctionCallParams): ) try: with open(filename, "w") as file: - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) @@ -94,6 +94,10 @@ async def load_conversation(params: FunctionCallParams): with open(filename, "r") as file: params.context.set_messages(json.load(file)) await params.llm.reset_conversation() + # NOTE: we manually create a response here rather than relying + # on the function callback to trigger one since we've reset the + # conversation so the remote service doesn't know about the + # in-progress tool call. await params.llm._create_response() except Exception as e: await params.result_callback({"success": False, "error": str(e)}) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 3d64f6ccb..1abbce01f 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -401,9 +401,10 @@ class OpenAIRealtimeLLMService(LLMService): # Run the LLM at next opportunity await self._create_response() else: - # We got an updated context + # We got an updated context. + # This may contain a new user message or tool call result. self._context = context - # Send results for any newly-completed function calls + # Send results for newly-completed function calls, if any. await self._process_completed_function_calls(send_new_results=True) async def _handle_messages_append(self, frame): @@ -758,7 +759,11 @@ class OpenAIRealtimeLLMService(LLMService): """ logger.debug("Resetting conversation") await self._disconnect() + + # Prepare to setup server-side conversation from local context again self._llm_needs_conversation_setup = True + await self._process_completed_function_calls(send_new_results=False) + await self._connect() @traced_openai_realtime(operation="llm_request") @@ -771,6 +776,10 @@ class OpenAIRealtimeLLMService(LLMService): # Configure the LLM for this session if needed if self._llm_needs_conversation_setup: + logger.debug( + f"Setting up conversation on OpenAI Realtime LLM service with initial messages: {adapter.get_messages_for_logging(self._context)}" + ) + # Send initial messages llm_invocation_params = adapter.get_llm_invocation_params(self._context) messages = llm_invocation_params["messages"] @@ -785,7 +794,7 @@ class OpenAIRealtimeLLMService(LLMService): # We're done configuring the LLM for this session self._llm_needs_conversation_setup = False - logger.debug(f"Creating response: {adapter.get_messages_for_logging(self._context)}") + logger.debug(f"Creating response") await self.push_frame(LLMFullResponseStartFrame()) await self.start_processing_metrics() @@ -809,8 +818,8 @@ class OpenAIRealtimeLLMService(LLMService): await self._send_tool_result(tool_call_id, message.get("content")) self._completed_tool_calls.add(tool_call_id) - # If we sent any new tool call results to the service, trigger another - # response + # If we reported any new tool call results to the service, trigger + # another response if sent_new_result: await self._create_response() From 6140fdb2c98b3b65dc526ed004a3d4d58072e013 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 21 Oct 2025 17:40:59 -0400 Subject: [PATCH 73/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). In anticipation of `messages` property being added to `LLMContext` (in another PR), remove warnings about the need to use `get_messages()` instead. --- src/pipecat/services/openai/realtime/context.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py index 96a714565..b5d68b8b4 100644 --- a/src/pipecat/services/openai/realtime/context.py +++ b/src/pipecat/services/openai/realtime/context.py @@ -27,9 +27,6 @@ context: OpenAIRealtimeLLMContext # or context: OpenAILLMContext - - # Reading messages from context - messages = context.messages ``` AFTER: @@ -46,9 +43,6 @@ # Context type context: LLMContext - - # Reading messages from context - messages = context.get_messages() ``` """ @@ -75,8 +69,6 @@ with warnings.catch_warnings(): "context: OpenAIRealtimeLLMContext\n" "# or\n" "context: OpenAILLMContext\n\n" - "# Reading messages from context\n" - "messages = context.messages\n" "```\n\n" "AFTER:\n" "```\n" @@ -89,8 +81,6 @@ with warnings.catch_warnings(): "frame: LLMContextFrame\n\n" "# Context type\n" "context: LLMContext\n\n" - "# Reading messages from context\n" - "messages = context.get_messages()\n" "```\n", ) From 9bc02afd0d82bbabc73e069742d66e27a91ed8f5 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 09:34:06 -0400 Subject: [PATCH 74/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). CHANGELOG tweak. --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a152022f1..2449276ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,9 +63,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # or context: OpenAILLMContext - # Reading messages from context - messages = context.messages - ## AFTER: # Context aggregator type @@ -76,9 +73,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # Context type context: LLMContext - - # Reading messages from context - messages = context.get_messages() ``` Also note that `RealtimeMessagesUpdateFrame` and From 0495de52b60ec7c8e22e711a21b8e9a5e251b120 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 09:57:44 -0400 Subject: [PATCH 75/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Log warning about transcription frame direction change. --- CHANGELOG.md | 6 +++--- src/pipecat/services/openai/realtime/llm.py | 22 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2449276ab..4a85fc19d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `LLMContext`, it is not meant to be swapped out for another LLM service at runtime.) - Note: `TranscriptionFrame`s now go upstream from `OpenAIRealtimeLLMService`, - so if you're using `TranscriptProcessor`, say, you'll want to adjust - accordingly: + Note: `TranscriptionFrame`s and `InterimTranscriptionFrame`s now go upstream + from `OpenAIRealtimeLLMService`, so if you're using `TranscriptProcessor`, + say, you'll want to adjust accordingly: ```python pipeline = Pipeline( diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 1abbce01f..36ec7930c 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -139,6 +139,28 @@ class OpenAIRealtimeLLMService(LLMService): stacklevel=2, ) + # Log warning about transcription frame direction change in 0.0.92 + logger.warning( + "As of version 0.0.92, TranscriptionFrames and InterimTranscriptionFrames " + "now go upstream from OpenAIRealtimeLLMService, so if you're using " + "TranscriptProcessor, say, you'll want to adjust accordingly:\n\n" + "pipeline = Pipeline(\n" + " [\n" + " transport.input(),\n" + " context_aggregator.user(),\n\n" + " # BEFORE\n" + " llm,\n" + " transcript.user(),\n\n" + " # AFTER\n" + " transcript.user(),\n" + " llm,\n\n" + " transport.output(),\n" + " transcript.assistant(),\n" + " context_aggregator.assistant(),\n" + " ]\n" + ")" + ) + full_url = f"{base_url}?model={model}" super().__init__(base_url=full_url, **kwargs) From e42cf78e79760d9ac6c48ce5cceb16e097ea6e6e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 10:02:45 -0400 Subject: [PATCH 76/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Update deprecation versions. --- src/pipecat/services/openai/realtime/context.py | 2 +- src/pipecat/services/openai/realtime/frames.py | 2 +- src/pipecat/services/openai/realtime/llm.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py index b5d68b8b4..91c6e74d5 100644 --- a/src/pipecat/services/openai/realtime/context.py +++ b/src/pipecat/services/openai/realtime/context.py @@ -6,7 +6,7 @@ """OpenAI Realtime LLM context and aggregator implementations. -.. deprecated:: 0.0.91 +.. deprecated:: 0.0.92 OpenAI Realtime no longer uses types from this module under the hood. It now uses `LLMContext` and `LLMContextAggregatorPair`. Using the new patterns should allow you to not need types from this module. diff --git a/src/pipecat/services/openai/realtime/frames.py b/src/pipecat/services/openai/realtime/frames.py index 1f800af89..39cfd9757 100644 --- a/src/pipecat/services/openai/realtime/frames.py +++ b/src/pipecat/services/openai/realtime/frames.py @@ -6,7 +6,7 @@ """Custom frame types for OpenAI Realtime API integration. -.. deprecated:: 0.0.91 +.. deprecated:: 0.0.92 OpenAI Realtime no longer uses types from this module under the hood. It now works more like most LLM services in Pipecat, relying on updates to diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 36ec7930c..bf5fe7679 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -121,7 +121,7 @@ class OpenAIRealtimeLLMService(LLMService): start_audio_paused: Whether to start with audio input paused. Defaults to False. send_transcription_frames: Whether to emit transcription frames. - .. deprecated:: 0.0.91 + .. deprecated:: 0.0.92 This parameter is deprecated and will be removed in a future version. Transcription frames are always sent. From df19011080e2e072cbe6d78d0ddc5236734f23ba Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 11:17:00 -0400 Subject: [PATCH 77/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Improve warning about transcription frame direction change. --- src/pipecat/services/openai/realtime/llm.py | 52 ++++++++++++--------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index bf5fe7679..284d19a90 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -139,28 +139,6 @@ class OpenAIRealtimeLLMService(LLMService): stacklevel=2, ) - # Log warning about transcription frame direction change in 0.0.92 - logger.warning( - "As of version 0.0.92, TranscriptionFrames and InterimTranscriptionFrames " - "now go upstream from OpenAIRealtimeLLMService, so if you're using " - "TranscriptProcessor, say, you'll want to adjust accordingly:\n\n" - "pipeline = Pipeline(\n" - " [\n" - " transport.input(),\n" - " context_aggregator.user(),\n\n" - " # BEFORE\n" - " llm,\n" - " transcript.user(),\n\n" - " # AFTER\n" - " transcript.user(),\n" - " llm,\n\n" - " transport.output(),\n" - " transcript.assistant(),\n" - " context_aggregator.assistant(),\n" - " ]\n" - ")" - ) - full_url = f"{base_url}?model={model}" super().__init__(base_url=full_url, **kwargs) @@ -883,6 +861,36 @@ class OpenAIRealtimeLLMService(LLMService): the user and one for the assistant, encapsulated in an OpenAIContextAggregatorPair. """ + # Log warning about transcription frame direction change in 0.0.92. + # We're putting this warning here rather than in the constructor so + # that it shows up for folks who haven't updated their code at all + # since 0.0.92, gives them a way to acknowledge and dismiss the + # warning, and encourages adoption of a new preferred pattern. + logger.warning( + "As of version 0.0.92, TranscriptionFrames and InterimTranscriptionFrames " + "now go upstream from OpenAIRealtimeLLMService, so if you're using " + "TranscriptProcessor, say, you'll want to adjust accordingly:\n\n" + "pipeline = Pipeline(\n" + " [\n" + " transport.input(),\n" + " context_aggregator.user(),\n\n" + " # BEFORE\n" + " llm,\n" + " transcript.user(),\n\n" + " # AFTER\n" + " transcript.user(),\n" + " llm,\n\n" + " transport.output(),\n" + " transcript.assistant(),\n" + " context_aggregator.assistant(),\n" + " ]\n" + ")\n\n" + "Once you've done that (if needed), you can dismiss this warning " + "by updating to the new context-setup pattern:\n\n" + " context = LLMContext(messages, tools)\n" + " context_aggregator = LLMContextAggregatorPair(context)\n" + ) + context = LLMContext.from_openai_context(context) return LLMContextAggregatorPair( context, user_params=user_params, assistant_params=assistant_params From 95be1510ac47cec294882a5f191b96ebd1664ce4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 11:34:33 -0400 Subject: [PATCH 78/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Improve `OpenAIRealtimeLLMAdapter.get_messages_for_logging()`. --- .../services/open_ai_realtime_adapter.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 7a6fc4d02..3cdfefc86 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -75,7 +75,25 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): Returns: List of messages in a format ready for logging about OpenAI Realtime. """ - return self._from_universal_context_messages(self.get_messages(context)).messages + # NOTE: this is the same as in OpenAIAdapter, as that's what it was + # prior to a refactor. Worth noting that for OpenAI Realtime + # specifically, not everything handled here is necessarily supported + # (or supported yet). + msgs = [] + for message in self.get_messages(context): + msg = copy.deepcopy(message) + if "content" in msg: + if isinstance(msg["content"], list): + for item in msg["content"]: + if item["type"] == "image_url": + if item["image_url"]["url"].startswith("data:image/"): + item["image_url"]["url"] = "data:image/..." + if item["type"] == "input_audio": + item["input_audio"]["data"] = "..." + if "mime_type" in msg and msg["mime_type"].startswith("image/"): + msg["data"] = "..." + msgs.append(msg) + return msgs @dataclass class ConvertedMessages: From 75b3ea9c96dfd3233136dd5846bd2527eb845e21 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 12:01:58 -0400 Subject: [PATCH 79/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Fix tracing. --- src/pipecat/utils/tracing/service_decorators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/utils/tracing/service_decorators.py b/src/pipecat/utils/tracing/service_decorators.py index cf1ba912c..3935a4afc 100644 --- a/src/pipecat/utils/tracing/service_decorators.py +++ b/src/pipecat/utils/tracing/service_decorators.py @@ -905,7 +905,9 @@ def traced_openai_realtime(operation: str) -> Callable: # Capture context messages being sent if hasattr(self, "_context") and self._context: try: - messages = self._context.get_messages_for_logging() + messages = self.get_llm_adapter().get_messages_for_logging( + self._context + ) if messages: operation_attrs["context_messages"] = json.dumps(messages) except Exception as e: From 8ac421f8fd45127e76350f394196e3e0d1011de8 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 12:03:45 -0400 Subject: [PATCH 80/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Remove unused imports. --- examples/foundational/19-openai-realtime.py | 1 - examples/foundational/19b-openai-realtime-text.py | 1 - .../foundational/20b-persistent-context-openai-realtime.py | 3 --- 3 files changed, 5 deletions(-) diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 3bf12f916..e883322ef 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -21,7 +21,6 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py index fab029bb6..c1f33b7bf 100644 --- a/examples/foundational/19b-openai-realtime-text.py +++ b/examples/foundational/19b-openai-realtime-text.py @@ -20,7 +20,6 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 346a5b4bd..e3f018c16 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -22,9 +22,6 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService From 15aa76efbaabbe7e760242522772eab70836161b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 12:49:48 -0400 Subject: [PATCH 81/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Maintain backward compatibility with functions specified in dict format. --- src/pipecat/adapters/schemas/tools_schema.py | 3 +++ .../services/aws_nova_sonic_adapter.py | 18 ++++++++++++++++-- .../services/open_ai_realtime_adapter.py | 18 ++++++++++++++++-- .../processors/aggregators/llm_context.py | 12 ++++++++++-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/pipecat/adapters/schemas/tools_schema.py b/src/pipecat/adapters/schemas/tools_schema.py index 05710616d..d0d798569 100644 --- a/src/pipecat/adapters/schemas/tools_schema.py +++ b/src/pipecat/adapters/schemas/tools_schema.py @@ -22,9 +22,12 @@ class AdapterType(Enum): Parameters: GEMINI: Google Gemini adapter - currently the only service supporting custom tools. + SHIM: Backward compatibility shim for creating ToolsSchemas from lists of tools in + any format, used by LLMContext.from_openai_context. """ GEMINI = "gemini" # that is the only service where we are able to add custom tools for now + SHIM = "shim" # for use as backward compatibility shim for creating ToolsSchemas from list of tools in any format class ToolsSchema: diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py index 60f12798b..dcc42ba68 100644 --- a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -16,7 +16,7 @@ from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage @@ -210,4 +210,18 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): List of dictionaries in AWS Nova Sonic function format. """ functions_schema = tools_schema.standard_tools - return [self._to_aws_nova_sonic_function_format(func) for func in functions_schema] + standard_tools = [ + self._to_aws_nova_sonic_function_format(func) for func in functions_schema + ] + + # For backward compatibility, AWS Nova Sonic can still be used with + # tools in dict format, even though it always uses `LLMContext` under + # the hood (via `LLMContext.from_openai_context()`). + # To support this behavior, we use "shimmed" custom tools here. + # (We maintain this backward compatibility because users aren't + # *knowingly* opting into the new `LLMContext`.) + shimmed_tools = [] + if tools_schema.custom_tools: + shimmed_tools = tools_schema.custom_tools.get(AdapterType.SHIM, []) + + return standard_tools + shimmed_tools diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 3cdfefc86..3d3650633 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -15,7 +15,7 @@ from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage from pipecat.services.openai.realtime import events @@ -225,4 +225,18 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): List of function definitions in OpenAI Realtime format. """ functions_schema = tools_schema.standard_tools - return [self._to_openai_realtime_function_format(func) for func in functions_schema] + standard_tools = [ + self._to_openai_realtime_function_format(func) for func in functions_schema + ] + + # For backward compatibility, OpenAI Realtime can still be used with + # tools in dict format, even though it always uses `LLMContext` under + # the hood (via `LLMContext.from_openai_context()`). + # To support this behavior, we use "shimmed" custom tools here. + # (We maintain this backward compatibility because users aren't + # *knowingly* opting into the new `LLMContext`.) + shimmed_tools = [] + if tools_schema.custom_tools: + shimmed_tools = tools_schema.custom_tools.get(AdapterType.SHIM, []) + + return standard_tools + shimmed_tools diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 913566909..61241729e 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -29,7 +29,7 @@ from openai.types.chat import ( ) from PIL import Image -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.frames.frames import AudioRawFrame if TYPE_CHECKING: @@ -83,9 +83,17 @@ class LLMContext: Returns: New LLMContext instance with converted messages and settings. """ + # Convert tools to ToolsSchema if needed. + # If the tools are already a ToolsSchema, this is a no-op. + # Otherwise, we wrap them in a shim ToolsSchema. + converted_tools = openai_context.tools + if isinstance(converted_tools, list): + converted_tools = ToolsSchema( + standard_tools=[], custom_tools={AdapterType.SHIM: converted_tools} + ) return LLMContext( messages=openai_context.get_messages(), - tools=openai_context.tools, + tools=converted_tools, tool_choice=openai_context.tool_choice, ) From 8c03df1463a537e00454ea48321c5f9124dfef3e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 14:32:32 -0400 Subject: [PATCH 82/93] Update some docstring arg descriptions to be a bit more current or accurate --- src/pipecat/services/azure/realtime/llm.py | 2 +- src/pipecat/services/openai/realtime/llm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/azure/realtime/llm.py b/src/pipecat/services/azure/realtime/llm.py index 1193b82d4..66ba95eea 100644 --- a/src/pipecat/services/azure/realtime/llm.py +++ b/src/pipecat/services/azure/realtime/llm.py @@ -38,7 +38,7 @@ class AzureRealtimeLLMService(OpenAIRealtimeLLMService): Args: api_key: The API key for the Azure OpenAI service. base_url: The full Azure WebSocket endpoint URL including api-version and deployment. - Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2025-04-01-preview&deployment=my-realtime-deployment" **kwargs: Additional arguments passed to parent OpenAIRealtimeLLMService. """ super().__init__(base_url=base_url, api_key=api_key, **kwargs) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 284d19a90..213c27df7 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -113,7 +113,7 @@ class OpenAIRealtimeLLMService(LLMService): Args: api_key: OpenAI API key for authentication. - model: OpenAI model name. Defaults to "gpt-4o-realtime-preview-2025-06-03". + model: OpenAI model name. Defaults to "gpt-realtime". base_url: WebSocket base URL for the realtime API. Defaults to "wss://api.openai.com/v1/realtime". session_properties: Configuration properties for the realtime session. From 917ea273525a229ce29ac25de340a405b2a9fb55 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 14:36:22 -0400 Subject: [PATCH 83/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Update `AzureRealtimeLLMService` example (19a) to use new `LLMContext` pattern. --- examples/foundational/19a-azure-realtime.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index c4b0fc02a..7b07985be 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService @@ -155,10 +156,10 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm.register_function("get_current_weather", fetch_weather_from_api) llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) - # Create a standard OpenAI LLM context object using the normal messages format. The + # Create a standard LLM context object using the normal messages format. The # OpenAIRealtimeBetaLLMService will convert this internally to messages that the # openai WebSocket API can understand. - context = OpenAILLMContext( + context = LLMContext( [{"role": "user", "content": "Say hello!"}], # [{"role": "user", "content": [{"type": "text", "text": "Say hello!"}]}], # [ @@ -173,7 +174,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re tools, ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ From 0282033208c5099dbe79442b8fd8a07f3ed61773 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 15:08:58 -0400 Subject: [PATCH 84/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Add `LLMContext.get_messages_for_persistent_storage()` for compatibility with `OpenAILLMContext`, to avoid tripping up users who we're unknowingly migrating to using `LLMContext`. --- .../processors/aggregators/llm_context.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 61241729e..8dc79fb50 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -15,7 +15,6 @@ service-specific adapter. """ import base64 -import copy import io from dataclasses import dataclass from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union @@ -127,6 +126,33 @@ class LLMContext: """ return self.get_messages() + def get_messages_for_persistent_storage(self) -> List[LLMContextMessage]: + """Get messages suitable for persistent storage. + + NOTE: the only reason this method exists is because we're "silently" + switching from OpenAILLMContext to LLMContext under the hood in some + services and don't want to trip up users who may have been relying on + this method, which is part of the public API of OpenAILLMContext but + doesn't need to be for LLMContext. + + .. deprecated:: + Use `get_messages()` instead. + + Returns: + List of conversation messages. + """ + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "get_messages_for_persistent_storage() is deprecated, use get_messages() instead.", + DeprecationWarning, + stacklevel=2, + ) + + return self.get_messages() + def get_messages(self, llm_specific_filter: Optional[str] = None) -> List[LLMContextMessage]: """Get the current messages list. From 1f96cdf9707f1d690f83e1fe39e5a5f84144e080 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 16:05:41 -0400 Subject: [PATCH 85/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make `LLMUserAggregator` push the `LLMSetToolsFrame`s, in case a speech-to-speech service that needs to handle the frame itself—like `OpenAIRealtimeLLMService`—is downstream. As far as I can tell, pushing `LLMSetToolsFrame` should otherwise have no unwanted side effects. --- examples/foundational/19-openai-realtime.py | 37 ++++++++++++++++--- .../aggregators/llm_response_universal.py | 6 +++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index e883322ef..5f215a07b 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -5,6 +5,7 @@ # +import asyncio import os from datetime import datetime @@ -14,7 +15,7 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage +from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame, TranscriptionMessage from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -53,6 +54,18 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def get_news(params: FunctionCallParams): + await params.result_callback( + { + "news": [ + "Massive UFO currently hovering above New York City", + "Stock markets reach all-time highs", + "Living dinosaur species discovered in the Amazon rainforest", + ], + } + ) + + async def fetch_restaurant_recommendation(params: FunctionCallParams): await params.result_callback({"name": "The Golden Dragon"}) @@ -74,6 +87,13 @@ weather_function = FunctionSchema( required=["location", "format"], ) +get_news_function = FunctionSchema( + name="get_news", + description="Get the current news.", + properties={}, + required=[], +) + restaurant_function = FunctionSchema( name="get_restaurant_recommendation", description="Get a restaurant recommendation", @@ -141,10 +161,6 @@ even if you're asked about them. You are participating in a voice conversation. Keep your responses concise, short, and to the point unless specifically asked to elaborate on a topic. -You have access to the following tools: -- get_current_weather: Get the current weather for a given location. -- get_restaurant_recommendation: Get a restaurant recommendation for a given location. - Remember, your responses should be short. Just one or two sentences, usually. Respond in English.""", ) @@ -158,6 +174,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re # llm.register_function(None, fetch_weather_from_api) llm.register_function("get_current_weather", fetch_weather_from_api) llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + llm.register_function("get_news", get_news) transcript = TranscriptProcessor() @@ -199,6 +216,16 @@ Remember, your responses should be short. Just one or two sentences, usually. Re # Kick off the conversation. await task.queue_frames([LLMRunFrame()]) + async def set_tools_after_delay(): + await asyncio.sleep(15) + new_tools = ToolsSchema( + standard_tools=[weather_function, restaurant_function, get_news_function] + ) + logger.info("Registering new tool with LLMSetToolsFrame") + await task.queue_frames([LLMSetToolsFrame(tools=new_tools)]) + + asyncio.create_task(set_tools_after_delay()) + @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 69a8dd280..9f1e04fe0 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -290,6 +290,12 @@ class LLMUserAggregator(LLMContextAggregator): await self._handle_llm_messages_update(frame) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) + # Push the LLMSetToolsFrame as well, since speech-to-speech LLM + # services (like OpenAI Realtime) may need to know about tool + # changes; unlike text-based LLM services they won't just "pick up + # the change" on the next LLM run, as the LLM is continuously + # running. + await self.push_frame(frame, direction) elif isinstance(frame, LLMSetToolChoiceFrame): self.set_tool_choice(frame.tool_choice) elif isinstance(frame, SpeechControlParamsFrame): From 8894db429026742a077fd4cc1ec7f0237c142090 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 16:39:13 -0400 Subject: [PATCH 86/93] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Add warning about no longer pushing `TTSTextFrame`s. --- CHANGELOG.md | 4 ++++ src/pipecat/services/openai/realtime/llm.py | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a85fc19d..f208bf1d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 by context aggregators, to update its internal state. Listen for `LLMContextFrame`s for context updates. + Finally, `LLMTextFrame`s are no longer pushed from `OpenAIRealtimeLLMService` + when it's configured with `output_modalities=['audio']`. If you need + to process its output, listen for `TTSTextFrame`s instead. + - Expanded support for universal `LLMContext` to `GeminiLiveLLMService`. As a reminder, the context-setup pattern when using `LLMContext` is: diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 213c27df7..c46f3435d 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -885,8 +885,11 @@ class OpenAIRealtimeLLMService(LLMService): " context_aggregator.assistant(),\n" " ]\n" ")\n\n" - "Once you've done that (if needed), you can dismiss this warning " - "by updating to the new context-setup pattern:\n\n" + "Also, LLMTextFrames are no longer pushed from " + "OpenAIRealtimeLLMService when it's configured with " + "output_modalities=['audio']. Listen for TTSTextFrames instead.\n\n" + "Once you've made the appropriate changes (if needed), you can" + "dismiss this warning by updating to the new context-setup pattern:\n\n" " context = LLMContext(messages, tools)\n" " context_aggregator = LLMContextAggregatorPair(context)\n" ) From d0f52feba3a36f7ecf9c85dd200bc00594230ed1 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 16:15:16 -0400 Subject: [PATCH 87/93] OpenAI Realtime needs the assistant context aggregator to have `expect_stripped_words=False` --- CHANGELOG.md | 8 +++++++- examples/foundational/19-openai-realtime.py | 8 +++++++- examples/foundational/19a-azure-realtime.py | 8 +++++++- src/pipecat/services/openai/realtime/llm.py | 1 + 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f208bf1d9..6ff37d47c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ```python context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) + context_aggregator = LLMContextAggregatorPair( + context, + # This part is `OpenAIRealtimeLLMService`-specific. + # `expect_stripped_words=False` needed when OpenAI Realtime used with + # "audio" modality (the default). + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) ``` (Note that even though `OpenAIRealtimeLLMService` now supports the universal diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 5f215a07b..3451c9da8 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -21,6 +21,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments @@ -186,7 +187,12 @@ Remember, your responses should be short. Just one or two sentences, usually. Re tools, ) - context_aggregator = LLMContextAggregatorPair(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when OpenAI Realtime used with + # "audio" modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index 7b07985be..7d9cf1b4b 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -19,6 +19,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import LLMAssistantAggregatorParams from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -174,7 +175,12 @@ Remember, your responses should be short. Just one or two sentences, usually. Re tools, ) - context_aggregator = LLMContextAggregatorPair(context) + context_aggregator = LLMContextAggregatorPair( + context, + # `expect_stripped_words=False` needed when OpenAI Realtime used with + # "audio" modality (the default) + assistant_params=LLMAssistantAggregatorParams(expect_stripped_words=False), + ) pipeline = Pipeline( [ diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index c46f3435d..f9e8b1a3f 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -895,6 +895,7 @@ class OpenAIRealtimeLLMService(LLMService): ) context = LLMContext.from_openai_context(context) + assistant_params.expect_stripped_words = False return LLMContextAggregatorPair( context, user_params=user_params, assistant_params=assistant_params ) From ddac24e6c9a44c8c658783fa2805415ea0e8e2d1 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 16:17:05 -0400 Subject: [PATCH 88/93] Fix a missing space in a warning message --- src/pipecat/services/openai/realtime/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index f9e8b1a3f..012604eb8 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -888,7 +888,7 @@ class OpenAIRealtimeLLMService(LLMService): "Also, LLMTextFrames are no longer pushed from " "OpenAIRealtimeLLMService when it's configured with " "output_modalities=['audio']. Listen for TTSTextFrames instead.\n\n" - "Once you've made the appropriate changes (if needed), you can" + "Once you've made the appropriate changes (if needed), you can " "dismiss this warning by updating to the new context-setup pattern:\n\n" " context = LLMContext(messages, tools)\n" " context_aggregator = LLMContextAggregatorPair(context)\n" From 89e9acf0e1b5673e00764b88dbd28535fd9876bb Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 16:21:04 -0400 Subject: [PATCH 89/93] CHANGELOG and code comment tweaks --- CHANGELOG.md | 6 +++--- examples/foundational/19-openai-realtime.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ff37d47c..5ab7c7121 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (Note that even though `OpenAIRealtimeLLMService` now supports the universal `LLMContext`, it is not meant to be swapped out for another LLM service at - runtime.) + runtime with `LLMSwitcher`.) Note: `TranscriptionFrame`s and `InterimTranscriptionFrame`s now go upstream from `OpenAIRealtimeLLMService`, so if you're using `TranscriptProcessor`, @@ -108,7 +108,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (Note that even though `GeminiLiveLLMService` now supports the universal `LLMContext`, it is not meant to be swapped out for another LLM service at - runtime.) + runtime with `LLMSwitcher`.) Worth noting: whether or not you use the new context-setup pattern with `GeminiLiveLLMService`, some types have changed under the hood: @@ -212,7 +212,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (Note that even though `AWSNovaSonicLLMService` now supports the universal `LLMContext`, it is not meant to be swapped out for another LLM service at - runtime.) + runtime with `LLMSwitcher`.) Worth noting: whether or not you use the new context-setup pattern with `AWSNovaSonicLLMService`, some types have changed under the hood: diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 3451c9da8..4604fbd02 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -222,6 +222,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re # Kick off the conversation. await task.queue_frames([LLMRunFrame()]) + # Add a new tool at runtime after a delay. async def set_tools_after_delay(): await asyncio.sleep(15) new_tools = ToolsSchema( From 8f15980c674f69d1349936b085b365b8f0a815b6 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Oct 2025 16:23:50 -0400 Subject: [PATCH 90/93] Get rid of unnecessary new task in example file --- examples/foundational/19-openai-realtime.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 4604fbd02..6907ec196 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -223,15 +223,11 @@ Remember, your responses should be short. Just one or two sentences, usually. Re await task.queue_frames([LLMRunFrame()]) # Add a new tool at runtime after a delay. - async def set_tools_after_delay(): - await asyncio.sleep(15) - new_tools = ToolsSchema( - standard_tools=[weather_function, restaurant_function, get_news_function] - ) - logger.info("Registering new tool with LLMSetToolsFrame") - await task.queue_frames([LLMSetToolsFrame(tools=new_tools)]) - - asyncio.create_task(set_tools_after_delay()) + await asyncio.sleep(15) + new_tools = ToolsSchema( + standard_tools=[weather_function, restaurant_function, get_news_function] + ) + await task.queue_frames([LLMSetToolsFrame(tools=new_tools)]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): From bcffa590a32ebe5117579efb6cf7f97738292418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 23 Oct 2025 13:20:16 -0700 Subject: [PATCH 91/93] DailyTransport: update start_dialout/start_recording return values --- CHANGELOG.md | 4 + src/pipecat/transports/daily/transport.py | 378 +++++++++++++++------- 2 files changed, 269 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab7c7121..920fff5be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `DailyTransport` updates: `start_dialout()` now returns two values: + `session_id` and `error`. `start_recording()` now returns two values: + `stream_id` and `error`. + - Updated `daily-python` to 0.21.0. - `SimliVideoService` now accepts `api_key` and `face_id` parameters directly, diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 2e48c4d2e..e7e3ab8ef 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -16,7 +16,7 @@ import time from concurrent.futures import CancelledError as FuturesCancelledError from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from typing import Any, Awaitable, Callable, Dict, Mapping, Optional +from typing import Any, Awaitable, Callable, Dict, Mapping, Optional, Tuple import aiohttp from loguru import logger @@ -419,6 +419,11 @@ class DailyAudioTrack: track: CustomAudioTrack +# This is just a type alias for the errors returned by daily-python. Right now +# they are just a string. +CallClientError = str + + class DailyTransportClient(EventHandler): """Core client for interacting with Daily's API. @@ -553,14 +558,17 @@ class DailyTransportClient(EventHandler): async def send_message( self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame - ): + ) -> Optional[CallClientError]: """Send an application message to participants. Args: frame: The message frame to send. + + Returns: + error: An error description or None. """ if not self._joined: - return + return "Unable to send messages before joining." participant_id = None if isinstance( @@ -572,7 +580,7 @@ class DailyTransportClient(EventHandler): self._client.send_app_message( frame.message, participant_id, completion=completion_callback(future) ) - await future + return await future async def read_next_audio_frame(self) -> Optional[InputAudioRawFrame]: """Reads the next 20ms audio frame from the virtual speaker.""" @@ -754,9 +762,6 @@ class DailyTransportClient(EventHandler): logger.info(f"Joined {self._room_url}") - if self._params.transcription_enabled: - await self.start_transcription(self._params.transcription_settings) - await self._callbacks.on_joined(data) self._joined_event.set() @@ -842,9 +847,6 @@ class DailyTransportClient(EventHandler): # Call callback before leaving. await self._callbacks.on_before_leave() - if self._params.transcription_enabled: - await self.stop_transcription() - # Remove any custom tracks, if any. for track_name, _ in self._custom_audio_tracks.items(): await self.remove_custom_audio_track(track_name) @@ -873,7 +875,7 @@ class DailyTransportClient(EventHandler): self._client.release() self._client = None - def participants(self): + def participants(self) -> Mapping[str, Any]: """Get current participants in the room. Returns: @@ -881,7 +883,7 @@ class DailyTransportClient(EventHandler): """ return self._client.participants() - def participant_counts(self): + def participant_counts(self) -> Mapping[str, Any]: """Get participant count information. Returns: @@ -889,165 +891,173 @@ class DailyTransportClient(EventHandler): """ return self._client.participant_counts() - async def start_dialout(self, settings): + async def start_dialout(self, settings) -> Tuple[str, Optional[CallClientError]]: """Start a dial-out call to a phone number. Args: settings: Dial-out configuration settings. - """ - logger.debug(f"Starting dialout: settings={settings}") + Returns: + session_id: Dail-out session ID. + error: An error description or None. + """ future = self._get_event_loop().create_future() self._client.start_dialout(settings, completion=completion_callback(future)) - error = await future - if error: - logger.error(f"Unable to start dialout: {error}") + return await future - async def stop_dialout(self, participant_id): + async def stop_dialout(self, participant_id) -> Optional[CallClientError]: """Stop a dial-out call for a specific participant. Args: participant_id: ID of the participant to stop dial-out for. - """ - logger.debug(f"Stopping dialout: participant_id={participant_id}") + Returns: + error: An error description or None. + """ future = self._get_event_loop().create_future() self._client.stop_dialout(participant_id, completion=completion_callback(future)) - error = await future - if error: - logger.error(f"Unable to stop dialout: {error}") + return await future - async def send_dtmf(self, settings): + async def send_dtmf(self, settings) -> Optional[CallClientError]: """Send DTMF tones during a call. Args: settings: DTMF settings including tones and target session. + + Returns: + error: An error description or None. """ session_id = settings.get("sessionId") or self._dial_out_session_id if not session_id: - logger.error("Unable to send DTMF: 'sessionId' is not set") - return + return "Can't send DTMF if 'sessionId' is not set" # Update 'sessionId' field. settings["sessionId"] = session_id future = self._get_event_loop().create_future() self._client.send_dtmf(settings, completion=completion_callback(future)) - await future + return await future - async def sip_call_transfer(self, settings): + async def sip_call_transfer(self, settings) -> Optional[CallClientError]: """Transfer a SIP call to another destination. Args: settings: SIP call transfer settings. + + Returns: + error: An error description or None. """ session_id = ( settings.get("sessionId") or self._dial_out_session_id or self._dial_in_session_id ) if not session_id: - logger.error("Unable to transfer SIP call: 'sessionId' is not set") - return + return "Can't transfer SIP call if 'sessionId' is not set" # Update 'sessionId' field. settings["sessionId"] = session_id future = self._get_event_loop().create_future() self._client.sip_call_transfer(settings, completion=completion_callback(future)) - await future + return await future - async def sip_refer(self, settings): + async def sip_refer(self, settings) -> Optional[CallClientError]: """Send a SIP REFER request. Args: settings: SIP REFER settings. + + Returns: + error: An error description or None. """ future = self._get_event_loop().create_future() self._client.sip_refer(settings, completion=completion_callback(future)) - await future + return await future - async def start_recording(self, streaming_settings, stream_id, force_new): + async def start_recording( + self, streaming_settings, stream_id, force_new + ) -> Tuple[str, Optional[CallClientError]]: """Start recording the call. Args: streaming_settings: Recording configuration settings. stream_id: Unique identifier for the recording stream. force_new: Whether to force a new recording session. - """ - logger.debug( - f"Starting recording: stream_id={stream_id} force_new={force_new} settings={streaming_settings}" - ) + Returns: + stream_id: Unique identifier for the recording stream. + error: An error description or None. + """ future = self._get_event_loop().create_future() self._client.start_recording( streaming_settings, stream_id, force_new, completion=completion_callback(future) ) - error = await future - if error: - logger.error(f"Unable to start recording: {error}") + return await future - async def stop_recording(self, stream_id): + async def stop_recording(self, stream_id) -> Optional[CallClientError]: """Stop recording the call. Args: stream_id: Unique identifier for the recording stream to stop. - """ - logger.debug(f"Stopping recording: stream_id={stream_id}") + Returns: + error: An error description or None. + """ future = self._get_event_loop().create_future() self._client.stop_recording(stream_id, completion=completion_callback(future)) - error = await future - if error: - logger.error(f"Unable to stop recording: {error}") + return await future - async def start_transcription(self, settings): + async def start_transcription(self, settings) -> Optional[CallClientError]: """Start transcription for the call. Args: settings: Transcription configuration settings. + + Returns: + error: An error description or None. """ if not self._token: - logger.warning("Transcription can't be started without a room token") - return - - logger.debug(f"Starting transcription: settings={settings}") + return "Transcription can't be started without a room token" future = self._get_event_loop().create_future() self._client.start_transcription( settings=self._params.transcription_settings.model_dump(exclude_none=True), completion=completion_callback(future), ) - error = await future - if error: - logger.error(f"Unable to start transcription: {error}") + return await future - async def stop_transcription(self): - """Stop transcription for the call.""" + async def stop_transcription(self) -> Optional[CallClientError]: + """Stop transcription for the call. + + Returns: + error: An error description or None. + """ if not self._token: - return - - logger.debug(f"Stopping transcription") + return "Transcription can't be stopped without a room token" future = self._get_event_loop().create_future() self._client.stop_transcription(completion=completion_callback(future)) - error = await future - if error: - logger.error(f"Unable to stop transcription: {error}") + return await future - async def send_prebuilt_chat_message(self, message: str, user_name: Optional[str] = None): + async def send_prebuilt_chat_message( + self, message: str, user_name: Optional[str] = None + ) -> Optional[CallClientError]: """Send a chat message to Daily's Prebuilt main room. Args: message: The chat message to send. user_name: Optional user name that will appear as sender of the message. + + Returns: + error: An error description or None. """ if not self._joined: - return + return "Can't send message if not joined" future = self._get_event_loop().create_future() self._client.send_prebuilt_chat_message( message, user_name=user_name, completion=completion_callback(future) ) - await future + return await future async def capture_participant_transcription(self, participant_id: str): """Enable transcription capture for a specific participant. @@ -1167,38 +1177,51 @@ class DailyTransportClient(EventHandler): return track - async def remove_custom_audio_track(self, track_name: str): + async def remove_custom_audio_track(self, track_name: str) -> Optional[CallClientError]: """Remove a custom audio track. Args: track_name: Name of the custom audio track to remove. + + Returns: + error: An error description or None. """ future = self._get_event_loop().create_future() self._client.remove_custom_audio_track( track_name=track_name, completion=completion_callback(future), ) - await future + return await future - async def update_transcription(self, participants=None, instance_id=None): + async def update_transcription( + self, participants=None, instance_id=None + ) -> Optional[CallClientError]: """Update transcription settings for specific participants. Args: participants: List of participant IDs to enable transcription for. instance_id: Optional transcription instance ID. + + Returns: + error: An error description or None. """ future = self._get_event_loop().create_future() self._client.update_transcription( participants, instance_id, completion=completion_callback(future) ) - await future + return await future - async def update_subscriptions(self, participant_settings=None, profile_settings=None): + async def update_subscriptions( + self, participant_settings=None, profile_settings=None + ) -> Optional[CallClientError]: """Update media subscription settings. Args: participant_settings: Per-participant subscription settings. profile_settings: Global subscription profile settings. + + Returns: + error: An error description or None. """ future = self._get_event_loop().create_future() self._client.update_subscriptions( @@ -1206,32 +1229,42 @@ class DailyTransportClient(EventHandler): profile_settings=profile_settings, completion=completion_callback(future), ) - await future + return await future - async def update_publishing(self, publishing_settings: Mapping[str, Any]): + async def update_publishing( + self, publishing_settings: Mapping[str, Any] + ) -> Optional[CallClientError]: """Update media publishing settings. Args: publishing_settings: Publishing configuration settings. + + Returns: + error: An error description or None. """ future = self._get_event_loop().create_future() self._client.update_publishing( publishing_settings=publishing_settings, completion=completion_callback(future), ) - await future + return await future - async def update_remote_participants(self, remote_participants: Mapping[str, Any]): + async def update_remote_participants( + self, remote_participants: Mapping[str, Any] + ) -> Optional[CallClientError]: """Update settings for remote participants. Args: remote_participants: Remote participant configuration settings. + + Returns: + error: An error description or None. """ future = self._get_event_loop().create_future() self._client.update_remote_participants( remote_participants=remote_participants, completion=completion_callback(future) ) - await future + return await future # # @@ -1922,7 +1955,9 @@ class DailyOutputTransport(BaseOutputTransport): Args: frame: The transport message frame to send. """ - await self._client.send_message(frame) + error = await self._client.send_message(frame) + if error: + logger.error(f"Unable to send message: {error}") async def register_video_destination(self, destination: str): """Register a video output destination. @@ -2166,7 +2201,7 @@ class DailyTransport(BaseTransport): if self._output: await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM) - def participants(self): + def participants(self) -> Mapping[str, Any]: """Get current participants in the room. Returns: @@ -2174,7 +2209,7 @@ class DailyTransport(BaseTransport): """ return self._client.participants() - def participant_counts(self): + def participant_counts(self) -> Mapping[str, Any]: """Get participant count information. Returns: @@ -2182,76 +2217,155 @@ class DailyTransport(BaseTransport): """ return self._client.participant_counts() - async def start_dialout(self, settings=None): + async def start_dialout(self, settings=None) -> Tuple[str, Optional[CallClientError]]: """Start a dial-out call to a phone number. Args: settings: Dial-out configuration settings. - """ - await self._client.start_dialout(settings) - async def stop_dialout(self, participant_id): + Returns: + session_id: Dail-out session ID. + error: An error description or None. + """ + logger.debug(f"Starting dialout: settings={settings}") + + session_id, error = await self._client.start_dialout(settings) + if error: + logger.error(f"Unable to start dialout: {error}") + return session_id, error + + async def stop_dialout(self, participant_id) -> Optional[CallClientError]: """Stop a dial-out call for a specific participant. Args: participant_id: ID of the participant to stop dial-out for. - """ - await self._client.stop_dialout(participant_id) - async def sip_call_transfer(self, settings): + Returns: + error: An error description or None. + """ + logger.debug(f"Stopping dialout: participant_id={participant_id}") + + error = await self._client.stop_dialout(participant_id) + if error: + logger.error(f"Unable to stop dialout: {error}") + return error + + async def sip_call_transfer(self, settings) -> Optional[CallClientError]: """Transfer a SIP call to another destination. Args: settings: SIP call transfer settings. - """ - await self._client.sip_call_transfer(settings) - async def sip_refer(self, settings): + Returns: + error: An error description or None. + """ + logger.debug(f"Staring SIP call transfer: settings={settings}") + + error = await self._client.sip_call_transfer(settings) + if error: + logger.error(f"Unable to transfer SIP call: {error}") + return error + + async def sip_refer(self, settings) -> Optional[CallClientError]: """Send a SIP REFER request. Args: settings: SIP REFER settings. - """ - await self._client.sip_refer(settings) - async def start_recording(self, streaming_settings=None, stream_id=None, force_new=None): + Returns: + error: An error description or None. + """ + logger.debug(f"Staring SIP REFER: settings={settings}") + + error = await self._client.sip_refer(settings) + if error: + logger.error(f"Unable to perform SIP REFER: {error}") + return error + + async def start_recording( + self, streaming_settings=None, stream_id=None, force_new=None + ) -> Tuple[str, Optional[CallClientError]]: """Start recording the call. Args: streaming_settings: Recording configuration settings. stream_id: Unique identifier for the recording stream. force_new: Whether to force a new recording session. - """ - await self._client.start_recording(streaming_settings, stream_id, force_new) - async def stop_recording(self, stream_id=None): + Returns: + stream_id: Unique identifier for the recording stream. + error: An error description or None. + """ + logger.debug( + f"Starting recording: stream_id={stream_id} force_new={force_new} settings={streaming_settings}" + ) + + r_id, error = await self._client.start_recording(streaming_settings, stream_id, force_new) + if error: + logger.error(f"Unable to start recording: {error}") + return r_id, error + + async def stop_recording(self, stream_id=None) -> Optional[CallClientError]: """Stop recording the call. Args: stream_id: Unique identifier for the recording stream to stop. - """ - await self._client.stop_recording(stream_id) - async def start_transcription(self, settings=None): + Returns: + error: An error description or None. + """ + logger.debug(f"Stopping recording: stream_id={stream_id}") + + error = await self._client.stop_recording(stream_id) + if error: + logger.error(f"Unable to stop recording: {error}") + return error + + async def start_transcription(self, settings=None) -> Optional[CallClientError]: """Start transcription for the call. Args: settings: Transcription configuration settings. + + Returns: + error: An error description or None. """ - await self._client.start_transcription(settings) + logger.debug(f"Starting transcription: settings={settings}") - async def stop_transcription(self): - """Stop transcription for the call.""" - await self._client.stop_transcription() + error = await self._client.start_transcription(settings) + if error: + logger.error(f"Unable to start transcription: {error}") + return error - async def send_prebuilt_chat_message(self, message: str, user_name: Optional[str] = None): + async def stop_transcription(self) -> Optional[CallClientError]: + """Stop transcription for the call. + + Returns: + error: An error description or None. + """ + logger.debug(f"Stopping transcription") + + error = await self._client.stop_transcription() + if error: + logger.error(f"Unable to stop transcription: {error}") + return error + + async def send_prebuilt_chat_message( + self, message: str, user_name: Optional[str] = None + ) -> Optional[CallClientError]: """Send a chat message to Daily's Prebuilt main room. Args: message: The chat message to send. user_name: Optional user name that will appear as sender of the message. + + Returns: + error: An error description or None. """ - await self._client.send_prebuilt_chat_message(message, user_name) + error = await self._client.send_prebuilt_chat_message(message, user_name) + if error: + logger.error(f"Unable to send prebuilt chat message: {error}") + return error async def capture_participant_transcription(self, participant_id: str): """Enable transcription capture for a specific participant. @@ -2297,32 +2411,66 @@ class DailyTransport(BaseTransport): participant_id, framerate, video_source, color_format ) - async def update_publishing(self, publishing_settings: Mapping[str, Any]): + async def update_publishing( + self, publishing_settings: Mapping[str, Any] + ) -> Optional[CallClientError]: """Update media publishing settings. Args: publishing_settings: Publishing configuration settings. - """ - await self._client.update_publishing(publishing_settings=publishing_settings) - async def update_subscriptions(self, participant_settings=None, profile_settings=None): + Returns: + error: An error description or None. + """ + logger.debug(f"Updating publishing settings: settings={publishing_settings}") + + error = await self._client.update_publishing(publishing_settings=publishing_settings) + if error: + logger.error(f"Unable to update publishing settings: {error}") + return error + + async def update_subscriptions( + self, participant_settings=None, profile_settings=None + ) -> Optional[CallClientError]: """Update media subscription settings. Args: participant_settings: Per-participant subscription settings. profile_settings: Global subscription profile settings. + + Returns: + error: An error description or None. """ - await self._client.update_subscriptions( - participant_settings=participant_settings, profile_settings=profile_settings + logger.debug( + f"Updating subscriptions: participant_settings={participant_settings} profile_settings={profile_settings}" ) - async def update_remote_participants(self, remote_participants: Mapping[str, Any]): + error = await self._client.update_subscriptions( + participant_settings=participant_settings, profile_settings=profile_settings + ) + if error: + logger.error(f"Unable to update subscription settings: {error}") + return error + + async def update_remote_participants( + self, remote_participants: Mapping[str, Any] + ) -> Optional[CallClientError]: """Update settings for remote participants. Args: remote_participants: Remote participant configuration settings. + + Returns: + error: An error description or None. """ - await self._client.update_remote_participants(remote_participants=remote_participants) + logger.debug(f"Updating remote participants: remote_participants={remote_participants}") + + error = await self._client.update_remote_participants( + remote_participants=remote_participants + ) + if error: + logger.error(f"Unable to update remote participants: {error}") + return error async def _on_active_speaker_changed(self, participant: Any): """Handle active speaker change events.""" @@ -2330,6 +2478,8 @@ class DailyTransport(BaseTransport): async def _on_joined(self, data): """Handle room joined events.""" + if self._params.transcription_enabled: + await self.start_transcription(self._params.transcription_settings) await self._call_event_handler("on_joined", data) async def _on_left(self): @@ -2338,6 +2488,8 @@ class DailyTransport(BaseTransport): async def _on_before_leave(self): """Handle before leave room events.""" + if self._params.transcription_enabled: + await self.stop_transcription() await self._call_event_handler("on_before_leave") async def _on_error(self, error): From 6299b9db87b18745d6e646005e2fd51c5db90775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 19:49:38 -0700 Subject: [PATCH 92/93] DailyTransport: trigger "on_error" if transcription fails to start/stop --- CHANGELOG.md | 3 +++ src/pipecat/transports/daily/transport.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 920fff5be..a612c66cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `DailyTransport` triggers `on_error` event if transcription can't be started + or stopped. + - `DailyTransport` updates: `start_dialout()` now returns two values: `session_id` and `error`. `start_recording()` now returns two values: `stream_id` and `error`. diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index e7e3ab8ef..18c36ee56 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -2479,7 +2479,11 @@ class DailyTransport(BaseTransport): async def _on_joined(self, data): """Handle room joined events.""" if self._params.transcription_enabled: - await self.start_transcription(self._params.transcription_settings) + # We report an error because we are starting transcription + # internally and if it fails we need to know. + error = await self.start_transcription(self._params.transcription_settings) + if error: + await self._on_error(f"Unable to start transcription: {error}") await self._call_event_handler("on_joined", data) async def _on_left(self): @@ -2489,7 +2493,11 @@ class DailyTransport(BaseTransport): async def _on_before_leave(self): """Handle before leave room events.""" if self._params.transcription_enabled: - await self.stop_transcription() + # We report an error because we are stopping transcription + # internally and if it fails we need to know. + error = await self.stop_transcription() + if error: + await self._on_error(f"Unable to stop transcription: {error}") await self._call_event_handler("on_before_leave") async def _on_error(self, error): From abf34bcccfab15d329dca5c46c05539aa3cb76de Mon Sep 17 00:00:00 2001 From: Roshan Date: Wed, 29 Oct 2025 18:29:51 -0700 Subject: [PATCH 93/93] address pr comments --- CHANGELOG.md | 2 +- src/pipecat/services/cartesia/tts.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6928867..daeae5d61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `generation_config` parameter support to `CartesiaTTSService` and `CartesiaHttpTTSService` for Cartesia Sonic-3 models. Includes a new - `GenerationConfig` dataclass with `volume` (0.5-2.0), `speed` (0.6-1.5), + `GenerationConfig` class with `volume` (0.5-2.0), `speed` (0.6-1.5), and `emotion` (60+ options) parameters for fine-grained speech generation control. diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 4e785a374..c2185a355 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -126,6 +126,7 @@ class CartesiaTTSService(AudioContextWordTTSService): .. deprecated:: 0.0.68 The `emotion` parameter is deprecated and will be removed in a future version. + generation_config: Generation configuration for Sonic-3 models. Includes volume, speed (numeric), and emotion (string) parameters. """ @@ -322,7 +323,9 @@ class CartesiaTTSService(AudioContextWordTTSService): msg["speed"] = self._settings["speed"] if self._settings["generation_config"]: - msg["generation_config"] = self._settings["generation_config"].model_dump(exclude_none=True) + msg["generation_config"] = self._settings["generation_config"].model_dump( + exclude_none=True + ) return json.dumps(msg) @@ -514,6 +517,7 @@ class CartesiaHttpTTSService(TTSService): .. deprecated:: 0.0.68 The `emotion` parameter is deprecated and will be removed in a future version. + generation_config: Generation configuration for Sonic-3 models. Includes volume, speed (numeric), and emotion (string) parameters. """ @@ -664,7 +668,9 @@ class CartesiaHttpTTSService(TTSService): payload["speed"] = self._settings["speed"] if self._settings["generation_config"]: - payload["generation_config"] = self._settings["generation_config"].model_dump(exclude_none=True) + payload["generation_config"] = self._settings["generation_config"].model_dump( + exclude_none=True + ) yield TTSStartedFrame()