From ffa16dd1364edc5e957a4915cfbd3e2d33d38312 Mon Sep 17 00:00:00 2001 From: Nischal Jain <55933460+nischalj10@users.noreply.github.com> Date: Thu, 22 May 2025 20:08:48 -0700 Subject: [PATCH 01/35] added deepwiki badge for weekly repo refresh --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae6c3cc53..20b7b9007 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ pipecat -[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) +[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/pipecat-ai/pipecat) # 🎙️ Pipecat: Real-Time Voice & Multimodal AI Agents From cbdbdee4c05091e98f27ffed9b0e2876c364d322 Mon Sep 17 00:00:00 2001 From: Adithya Suresh Date: Thu, 4 Sep 2025 16:58:58 +1000 Subject: [PATCH 02/35] Initial StrandsAgentsProcessor implementation --- .../07m-interruptible-aws-strands.py | 169 ++++++++++++++++++ .../processors/frameworks/strands_agents.py | 112 ++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 examples/foundational/07m-interruptible-aws-strands.py create mode 100644 src/pipecat/processors/frameworks/strands_agents.py diff --git a/examples/foundational/07m-interruptible-aws-strands.py b/examples/foundational/07m-interruptible-aws-strands.py new file mode 100644 index 000000000..82d7192c2 --- /dev/null +++ b/examples/foundational/07m-interruptible-aws-strands.py @@ -0,0 +1,169 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + LLMAssistantContextAggregator, + LLMUserContextAggregator, +) +from pipecat.processors.frameworks.strands_agents import StrandsAgentsProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.stt import AWSTranscribeSTTService +from pipecat.services.aws.tts import AWSPollyTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +# Strands agent setup +try: + from strands import Agent, tool + from strands.models import BedrockModel +except ImportError: + logger.warning("Strands not installed. Please install with: pip install strands-agents") + Agent = None + BedrockModel = None + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + +def build_agent(model_id: str, max_tokens: int): + """Create and configure a Strands agent for NAB customer service coaching. + + Args: + model_id: The AWS Bedrock model ID to use + max_tokens: Maximum tokens for the model + + Returns: + Configured Strands Agent + """ + @tool + def check_weather(location: str) -> str: + if location.lower() == "san francisco": + return "The weather in San Francisco is sunny and 30 degrees." + elif location.lower() == "sydney": + return "The weather in Sydney is cloudy and 20 degrees." + else: + return "I'm not sure about the weather in that location." + + agent = Agent( + model=BedrockModel( + model_id=model_id, + max_tokens=max_tokens, + ), + tools=[check_weather], + system_prompt="You are a helpful assistant that can check the weather in a given location." + ) + + return agent + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = AWSTranscribeSTTService() + + tts = AWSPollyTTSService( + region="us-west-2", # only specific regions support generative TTS + voice_id="Joanna", + params=AWSPollyTTSService.InputParams(engine="generative", rate="1.1"), + ) + + # Create Strands agent processor + try: + agent = build_agent( + model_id="us.anthropic.claude-3-5-haiku-20241022-v1:0", + max_tokens=8000 + ) + llm = StrandsAgentsProcessor(agent=agent) + logger.info("Successfully created Strands agent for NAB customer service coaching") + except Exception as e: + logger.error(f"Failed to create Strands agent: {e}") + raise ValueError( + "Unable to create Strands processor. Please ensure you have properly " + "installed strands-agents and configured your AWS credentials." + ) + + # Setup context aggregators for message handling + context = OpenAILLMContext() + tma_in = LLMUserContextAggregator(context=context) + tma_out = LLMAssistantContextAggregator(context=context) + + pipeline = Pipeline([ + transport.input(), # Transport user input + stt, # Speech-to-text + tma_in, # User context aggregator + llm, # Strands Agents processor + tts, # Text-to-speech + transport.output(), # Transport bot output + tma_out # Assistant context aggregator + ]) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "user", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py new file mode 100644 index 000000000..5c2011b43 --- /dev/null +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -0,0 +1,112 @@ +""" +Strands Agent integration for Pipecat. + +This module provides integration with Strands Agents for handling conversational AI +interactions. It supports both single agent and multi-agent graphs. +""" + +from typing import Optional + +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + +try: + from strands import Agent + from strands.multiagent.graph import Graph +except ModuleNotFoundError as e: + logger.exception("In order to use Strands Agents, you need to `pip install strands-agents`.") + raise Exception(f"Missing module: {e}") + + +class StrandsAgentsProcessor(FrameProcessor): + """Processor that integrates Strands Agents with Pipecat's frame pipeline. + + This processor takes LLM message frames, extracts the latest user message, + and processes it through either a single Strands Agent or a multi-agent Graph. + The response is streamed back as text frames with appropriate response markers. + + Supports both single agent streaming and graph-based multi-agent workflows. + """ + + def __init__( + self, + agent: Optional[Agent] = None, + graph: Optional[Graph] = None, + graph_exit_node: Optional[str] = None, + ): + """Initialize the Strands Agents processor. + + Args: + agent: The Strands Agent to use for single-agent processing. + graph: The Strands multi-agent Graph to use for graph-based processing. + graph_exit_node: The exit node name when using graph-based processing. + + Raises: + AssertionError: If neither agent nor graph is provided, or if graph is + provided without a graph_exit_node. + """ + super().__init__() + self.agent = agent + self.graph = graph + self.graph_exit_node = graph_exit_node + + assert self.agent or self.graph, "Either agent or graph must be provided" + + if self.graph: + assert self.graph_exit_node, "graph_exit_node must be provided if graph is provided" + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle LLM message frames. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + if isinstance(frame, OpenAILLMContextFrame): + text = frame.context.messages[-1]["content"] + await self._ainvoke(str(text).strip()) + else: + await self.push_frame(frame, direction) + + async def _ainvoke(self, text: str): + """Invoke the Strands agent with the provided text and stream results as Pipecat frames. + + Args: + text: The user input text to process through the agent or graph. + """ + logger.debug(f"Invoking Strands agent with: {text}") + await self.push_frame(LLMFullResponseStartFrame()) + try: + if self.graph: + # Graph does not stream; await full result then emit assistant text + graph_result = await self.graph.invoke_async(text) + try: + node_result = graph_result.results[self.graph_exit_node] + for agent_result in node_result.get_agent_results(): + message = getattr(agent_result, "message", None) + if isinstance(message, dict) and "content" in message: + for block in message["content"]: + if isinstance(block, dict) and "text" in block: + await self.push_frame(LLMTextFrame(str(block["text"]))) + except Exception as parse_err: + logger.warning(f"Failed to extract messages from GraphResult: {parse_err}") + else: + # Agent supports streaming events via async iterator + async for event in self.agent.stream_async(text): + if isinstance(event, dict) and "data" in event: + await self.push_frame(LLMTextFrame(str(event["data"]))) + except GeneratorExit: + logger.warning(f"{self} generator was closed prematurely") + except Exception as e: + logger.exception(f"{self} an unknown error occurred: {e}") + finally: + await self.push_frame(LLMFullResponseEndFrame()) \ No newline at end of file From 446d99d19400b29918e67569282cac57eb18e3f2 Mon Sep 17 00:00:00 2001 From: Adithya Suresh Date: Thu, 4 Sep 2025 17:08:16 +1000 Subject: [PATCH 03/35] Bug fixes --- examples/foundational/07m-interruptible-aws-strands.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/examples/foundational/07m-interruptible-aws-strands.py b/examples/foundational/07m-interruptible-aws-strands.py index 82d7192c2..584153ba2 100644 --- a/examples/foundational/07m-interruptible-aws-strands.py +++ b/examples/foundational/07m-interruptible-aws-strands.py @@ -9,12 +9,11 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response import ( LLMAssistantContextAggregator, LLMUserContextAggregator, ) @@ -143,9 +142,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # Kick off the conversation. - messages.append({"role": "user", "content": "Please introduce yourself to the user."}) - await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): From d1f72c1c0b25ea446d90ae1a278047edecb698c7 Mon Sep 17 00:00:00 2001 From: Adithya Suresh Date: Mon, 8 Sep 2025 14:44:25 +1000 Subject: [PATCH 04/35] Added usage metrics --- .../07m-interruptible-aws-strands.py | 37 +++++++------ .../processors/frameworks/strands_agents.py | 55 ++++++++++++++++++- 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/examples/foundational/07m-interruptible-aws-strands.py b/examples/foundational/07m-interruptible-aws-strands.py index 584153ba2..eca2947fc 100644 --- a/examples/foundational/07m-interruptible-aws-strands.py +++ b/examples/foundational/07m-interruptible-aws-strands.py @@ -12,11 +12,11 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.llm_response import ( LLMAssistantContextAggregator, LLMUserContextAggregator, ) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frameworks.strands_agents import StrandsAgentsProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -58,16 +58,18 @@ transport_params = { ), } + def build_agent(model_id: str, max_tokens: int): """Create and configure a Strands agent for NAB customer service coaching. - + Args: model_id: The AWS Bedrock model ID to use max_tokens: Maximum tokens for the model - + Returns: Configured Strands Agent """ + @tool def check_weather(location: str) -> str: if location.lower() == "san francisco": @@ -76,14 +78,14 @@ def build_agent(model_id: str, max_tokens: int): return "The weather in Sydney is cloudy and 20 degrees." else: return "I'm not sure about the weather in that location." - + agent = Agent( model=BedrockModel( model_id=model_id, max_tokens=max_tokens, ), tools=[check_weather], - system_prompt="You are a helpful assistant that can check the weather in a given location." + system_prompt="You are a helpful assistant that can check the weather in a given location.", ) return agent @@ -102,10 +104,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create Strands agent processor try: - agent = build_agent( - model_id="us.anthropic.claude-3-5-haiku-20241022-v1:0", - max_tokens=8000 - ) + agent = build_agent(model_id="us.anthropic.claude-3-5-haiku-20241022-v1:0", max_tokens=8000) llm = StrandsAgentsProcessor(agent=agent) logger.info("Successfully created Strands agent for NAB customer service coaching") except Exception as e: @@ -120,15 +119,17 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tma_in = LLMUserContextAggregator(context=context) tma_out = LLMAssistantContextAggregator(context=context) - pipeline = Pipeline([ - transport.input(), # Transport user input - stt, # Speech-to-text - tma_in, # User context aggregator - llm, # Strands Agents processor - tts, # Text-to-speech - transport.output(), # Transport bot output - tma_out # Assistant context aggregator - ]) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # Speech-to-text + tma_in, # User context aggregator + llm, # Strands Agents processor + tts, # Text-to-speech + transport.output(), # Transport bot output + tma_out, # Assistant context aggregator + ] + ) task = PipelineTask( pipeline, diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py index 5c2011b43..05aef0f17 100644 --- a/src/pipecat/processors/frameworks/strands_agents.py +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -15,6 +15,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMTextFrame, ) +from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -84,29 +85,79 @@ class StrandsAgentsProcessor(FrameProcessor): text: The user input text to process through the agent or graph. """ logger.debug(f"Invoking Strands agent with: {text}") - await self.push_frame(LLMFullResponseStartFrame()) try: + await self.push_frame(LLMFullResponseStartFrame()) + await self.start_processing_metrics() + await self.start_ttfb_metrics() + ttfb_tracking = True + if self.graph: # Graph does not stream; await full result then emit assistant text graph_result = await self.graph.invoke_async(text) + if ttfb_tracking: + await self.stop_ttfb_metrics() + ttfb_tracking = False try: node_result = graph_result.results[self.graph_exit_node] + logger.debug(f"Node result: {node_result}") for agent_result in node_result.get_agent_results(): + # Push to TTS service message = getattr(agent_result, "message", None) if isinstance(message, dict) and "content" in message: for block in message["content"]: if isinstance(block, dict) and "text" in block: await self.push_frame(LLMTextFrame(str(block["text"]))) + # Update usage metrics + await self._report_usage_metrics( + agent_result.metrics.accumulated_usage.get('inputTokens', 0), + agent_result.metrics.accumulated_usage.get('outputTokens', 0), + agent_result.metrics.accumulated_usage.get('totalTokens', 0) + ) except Exception as parse_err: logger.warning(f"Failed to extract messages from GraphResult: {parse_err}") else: # Agent supports streaming events via async iterator async for event in self.agent.stream_async(text): + # Push to TTS service if isinstance(event, dict) and "data" in event: await self.push_frame(LLMTextFrame(str(event["data"]))) + if ttfb_tracking: + await self.stop_ttfb_metrics() + ttfb_tracking = False + + # Update usage metrics + if isinstance(event, dict) and "event" in event and "metadata" in event['event']: + if 'usage' in event['event']['metadata']: + usage = event['event']['metadata']['usage'] + await self._report_usage_metrics(usage.get('inputTokens', 0), usage.get('outputTokens', 0), usage.get('totalTokens', 0)) except GeneratorExit: logger.warning(f"{self} generator was closed prematurely") except Exception as e: logger.exception(f"{self} an unknown error occurred: {e}") finally: - await self.push_frame(LLMFullResponseEndFrame()) \ No newline at end of file + if ttfb_tracking: + await self.stop_ttfb_metrics() + ttfb_tracking = False + await self.stop_processing_metrics() + await self.push_frame(LLMFullResponseEndFrame()) + + def can_generate_metrics(self) -> bool: + """Check if this service can generate performance metrics. + + Returns: + True as this service supports metrics generation. + """ + return True + + async def _report_usage_metrics( + self, + prompt_tokens: int, + completion_tokens: int, + total_tokens: int + ): + tokens = LLMTokenUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens + ) + await self.start_llm_usage_metrics(tokens) \ No newline at end of file From d0b573e44fe4a6f886818317b464de0b520714f9 Mon Sep 17 00:00:00 2001 From: Jin Kim Date: Thu, 18 Sep 2025 19:40:45 +0900 Subject: [PATCH 05/35] Fix audio buffer flush and silence handling --- .../audio/audio_buffer_processor.py | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 276051919..fefbafc9b 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -229,9 +229,12 @@ class AudioBufferProcessor(FrameProcessor): # Save time of frame so we can compute silence. self._last_bot_frame_at = time.time() - if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size: + if self._buffer_size > 0 and ( + len(self._user_audio_buffer) >= self._buffer_size + or len(self._bot_audio_buffer) >= self._buffer_size + ): await self._call_on_audio_data_handler() - self._reset_recording() + self._clear_primary_audio_buffers() # Process turn recording with preprocessed data. if self._enable_turn_audio: @@ -272,9 +275,15 @@ class AudioBufferProcessor(FrameProcessor): async def _call_on_audio_data_handler(self): """Call the audio data event handlers with buffered audio.""" - if not self.has_audio() or not self._recording: + if not self._recording: return + if len(self._user_audio_buffer) == 0 and len(self._bot_audio_buffer) == 0: + return + + self._align_track_buffers() + flush_time = time.time() + # Call original handler with merged audio merged_audio = self.merge_audio_buffers() await self._call_event_handler( @@ -290,6 +299,9 @@ class AudioBufferProcessor(FrameProcessor): self._num_channels, ) + self._last_user_frame_at = flush_time + self._last_bot_frame_at = flush_time + def _buffer_has_audio(self, buffer: bytearray) -> bool: """Check if a buffer contains audio data.""" return buffer is not None and len(buffer) > 0 @@ -307,6 +319,26 @@ class AudioBufferProcessor(FrameProcessor): self._user_turn_audio_buffer = bytearray() self._bot_turn_audio_buffer = bytearray() + def _clear_primary_audio_buffers(self): + """Clear user and bot buffers while preserving turn buffers and timestamps.""" + self._user_audio_buffer = bytearray() + self._bot_audio_buffer = bytearray() + + def _align_track_buffers(self): + """Pad the shorter track with silence so both tracks stay in sync.""" + user_len = len(self._user_audio_buffer) + bot_len = len(self._bot_audio_buffer) + if user_len == bot_len: + return + + target_len = max(user_len, bot_len) + if user_len < target_len: + self._user_audio_buffer.extend(b"\x00" * (target_len - user_len)) + self._last_user_frame_at = max(self._last_user_frame_at, self._last_bot_frame_at) + if bot_len < target_len: + self._bot_audio_buffer.extend(b"\x00" * (target_len - bot_len)) + self._last_bot_frame_at = max(self._last_bot_frame_at, self._last_user_frame_at) + async def _resample_input_audio(self, frame: InputAudioRawFrame) -> bytes: """Resample audio frame to the target sample rate.""" return await self._input_resampler.resample( From 58f70e7e0da0f38478096f7f93b550dda550e19f Mon Sep 17 00:00:00 2001 From: Jin Kim Date: Thu, 18 Sep 2025 22:19:32 +0900 Subject: [PATCH 06/35] Add tests for audio buffer processor flush alignment --- tests/test_audio_buffer_processor.py | 111 +++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/test_audio_buffer_processor.py diff --git a/tests/test_audio_buffer_processor.py b/tests/test_audio_buffer_processor.py new file mode 100644 index 000000000..24ae42265 --- /dev/null +++ b/tests/test_audio_buffer_processor.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import struct +import unittest + +from pipecat.frames.frames import InputAudioRawFrame, OutputAudioRawFrame, StartFrame +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor + + +class _PassthroughResampler: + async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes: # pragma: no cover - trivial + return audio + + +class TestAudioBufferProcessor(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.processor = AudioBufferProcessor(sample_rate=16000, num_channels=2, buffer_size=4) + self.processor._input_resampler = _PassthroughResampler() + self.processor._output_resampler = _PassthroughResampler() + self.processor._update_sample_rate(StartFrame(audio_out_sample_rate=16000)) + await self.processor.start_recording() + + async def asyncTearDown(self): + if getattr(self.processor, "_recording", False): + await self.processor.stop_recording() + await self.processor.cleanup() + + async def test_flush_user_audio_pads_bot_track(self): + user_audio = struct.pack(" Date: Mon, 22 Sep 2025 15:38:09 -0400 Subject: [PATCH 07/35] update latency observer logging to be in unique funcs, so others could extend and overwrite --- .../loggers/user_bot_latency_log_observer.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py index 505a0a807..e17387019 100644 --- a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py +++ b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py @@ -61,17 +61,27 @@ class UserBotLatencyLogObserver(BaseObserver): elif isinstance(data.frame, UserStoppedSpeakingFrame): self._user_stopped_time = time.time() elif isinstance(data.frame, (EndFrame, CancelFrame)): - if self._latencies: - avg_latency = mean(self._latencies) - min_latency = min(self._latencies) - max_latency = max(self._latencies) - logger.info( - f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING - Avg: {avg_latency:.3f}s, Min: {min_latency:.3f}s, Max: {max_latency:.3f}s" - ) + self._log_summary() elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time: latency = time.time() - self._user_stopped_time self._user_stopped_time = 0 self._latencies.append(latency) - logger.debug( - f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s" - ) + self._log_latency(latency) + + async def _log_summary(self): + if not self._latencies: + return + avg_latency = mean(self._latencies) + min_latency = min(self._latencies) + max_latency = max(self._latencies) + logger.info( + f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING - Avg: {avg_latency:.3f}s, Min: {min_latency:.3f}s, Max: {max_latency:.3f}s" + ) + + async def _log_latency(self, latency: float): + """Log the latency. + + Args: + latency: The latency to log. + """ + logger.debug(f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s") From 272532a3ea307978b55937165793ec0cad9e42a0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 22 Sep 2025 15:06:44 -0400 Subject: [PATCH 08/35] Update examples, wherever possible, to use `LLMContext` and associated machinery instead of `OpenAILLMContext` and associated machinery. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With all these examples updated, we no longer need dedicated examples illustrating `LLMContext`, so they're removed. Here’s where we *don’t* yet use `LLMContext` and associated machinery: - Realtime services: OpenAI Realtime, Gemini Live, and AWS Nova Sonic (support coming soon) - `GoogleLLMOpenAIBetaService` (it’s deprecated, so we didn’t bother adding support) - `LLMLogObserver` (support coming soon) - `GatedOpenAILLMContextAggregator` (support coming soon) - `LangchainProcessor` (support coming soon) - `Mem0MemoryService` (support coming soon) - Examples that use LLM-specific tools definitions as opposed to `ToolsSchema` (these will be updated soon) - Examples that rely `GoogleLLMContext.upgrade_to_google` (TBD what to do with these) Examples that use `LLMLogObserver`: - 30- Examples that use `GatedOpenAILLMContextAggregator`: - 22- Examples that use `LangchainProcessor`: - 07b- Examples that use `Mem0MemoryService`: - 37- Examples that need updating to use `ToolsSchema`: - 15- - 15a- - 20a- - 20c- - 20d- - 22b- - 22c- - 33- - 36- Examples that use `GoogleLLMContext.upgrade_to_google`: - 22d- - 25- --- examples/foundational/02-llm-say-one-thing.py | 9 +- .../04-transports-small-webrtc.py | 7 +- examples/foundational/04a-transports-daily.py | 7 +- .../foundational/04b-transports-livekit.py | 7 +- .../foundational/05-sync-speech-and-image.py | 9 +- .../05a-local-sync-speech-and-image.py | 9 +- .../foundational/06-listen-and-respond.py | 7 +- examples/foundational/06a-image-sync.py | 7 +- .../07-interruptible-cartesia-http.py | 7 +- examples/foundational/07-interruptible.py | 7 +- .../07a-interruptible-speechmatics-vad.py | 7 +- .../07a-interruptible-speechmatics.py | 7 +- .../foundational/07aa-interruptible-soniox.py | 7 +- .../07ab-interruptible-inworld-http.py | 7 +- .../07ac-interruptible-asyncai-http.py | 7 +- .../07ac-interruptible-asyncai.py | 7 +- .../07ad-interruptible-aicoustics.py | 7 +- .../07c-interruptible-deepgram-vad.py | 7 +- .../07c-interruptible-deepgram.py | 7 +- .../07d-interruptible-elevenlabs-http.py | 7 +- .../07d-interruptible-elevenlabs.py | 7 +- .../07e-interruptible-playht-http.py | 7 +- .../foundational/07e-interruptible-playht.py | 7 +- .../foundational/07f-interruptible-azure.py | 7 +- .../foundational/07g-interruptible-openai.py | 7 +- .../07h-interruptible-openpipe.py | 7 +- .../foundational/07i-interruptible-xtts.py | 7 +- .../foundational/07j-interruptible-gladia.py | 7 +- .../foundational/07k-interruptible-lmnt.py | 7 +- .../foundational/07l-interruptible-groq.py | 7 +- .../foundational/07m-interruptible-aws.py | 7 +- .../foundational/07n-interruptible-gemini.py | 7 +- .../foundational/07n-interruptible-google.py | 7 +- .../07o-interruptible-assemblyai.py | 7 +- .../foundational/07p-interruptible-krisp.py | 7 +- .../07q-interruptible-rime-http.py | 7 +- .../foundational/07q-interruptible-rime.py | 7 +- .../07r-interruptible-riva-nim.py | 7 +- .../07s-interruptible-google-audio-in.py | 7 +- .../foundational/07t-interruptible-fish.py | 7 +- .../07v-interruptible-neuphonic-http.py | 7 +- .../07v-interruptible-neuphonic.py | 7 +- .../foundational/07w-interruptible-fal.py | 7 +- .../foundational/07x-interruptible-local.py | 7 +- .../foundational/07y-interruptible-minimax.py | 7 +- .../07z-interruptible-sarvam-http.py | 7 +- .../foundational/07z-interruptible-sarvam.py | 7 +- examples/foundational/08-bots-arguing.py | 10 +- examples/foundational/10-wake-phrase.py | 7 +- examples/foundational/11-sound-effects.py | 13 +- examples/foundational/14-function-calling.py | 7 +- .../14a-function-calling-anthropic.py | 7 +- ...-function-calling-aws-universal-context.py | 219 ---------------- .../14b-function-calling-anthropic-video.py | 7 +- .../14c-function-calling-together.py | 7 +- .../14d-function-calling-video.py | 7 +- .../14e-function-calling-google.py | 7 +- .../foundational/14f-function-calling-groq.py | 7 +- .../foundational/14g-function-calling-grok.py | 7 +- .../14h-function-calling-azure.py | 7 +- .../14i-function-calling-fireworks.py | 7 +- .../foundational/14j-function-calling-nim.py | 7 +- .../14k-function-calling-cerebras.py | 7 +- .../14l-function-calling-deepseek.py | 7 +- .../14m-function-calling-openrouter.py | 7 +- .../14n-function-calling-perplexity.py | 7 +- .../14p-function-calling-gemini-vertex-ai.py | 7 +- .../foundational/14q-function-calling-qwen.py | 7 +- .../foundational/14r-function-calling-aws.py | 7 +- .../14s-function-calling-sambanova.py | 7 +- .../14t-function-calling-direct.py | 7 +- .../14u-function-calling-ollama.py | 7 +- .../14v-function-calling-openai.py | 7 +- .../14w-function-calling-mistral.py | 7 +- .../14x-function-calling-universal-context.py | 176 ------------- ...nction-calling-google-universal-context.py | 234 ------------------ ...ion-calling-anthropic-universal-context.py | 216 ---------------- examples/foundational/15-switch-voices.py | 7 +- examples/foundational/15a-switch-languages.py | 7 +- .../16-gpu-container-local-bot.py | 7 +- examples/foundational/17-detect-user-idle.py | 7 +- examples/foundational/21-tavus-transport.py | 7 +- .../foundational/21a-tavus-video-service.py | 7 +- .../foundational/23-bot-background-sound.py | 7 +- examples/foundational/24-stt-mute-filter.py | 7 +- examples/foundational/27-simli-layer.py | 7 +- .../28-transcription-processor.py | 7 +- .../foundational/29-turn-tracking-observer.py | 7 +- .../32-gemini-grounding-metadata.py | 7 +- examples/foundational/34-audio-recording.py | 7 +- .../35-pattern-pair-voice-switching.py | 7 +- examples/foundational/38-smart-turn-fal.py | 7 +- .../38a-smart-turn-local-coreml.py | 7 +- examples/foundational/38b-smart-turn-local.py | 7 +- examples/foundational/39-mcp-stdio.py | 7 +- examples/foundational/39a-mcp-run-sse.py | 7 +- examples/foundational/39b-multiple-mcp.py | 7 +- examples/foundational/39c-mcp-run-http.py | 7 +- examples/foundational/41a-text-only-webrtc.py | 7 +- .../foundational/41b-text-and-audio-webrtc.py | 7 +- .../foundational/42-interruption-config.py | 7 +- .../foundational/43a-heygen-video-service.py | 7 +- .../foundational/44-voicemail-detection.py | 7 +- .../45-before-and-after-events.py | 7 +- examples/quickstart/bot.py | 7 +- scripts/evals/run-release-evals.py | 19 -- 106 files changed, 405 insertions(+), 1181 deletions(-) delete mode 100644 examples/foundational/14aa-function-calling-aws-universal-context.py delete mode 100644 examples/foundational/14x-function-calling-universal-context.py delete mode 100644 examples/foundational/14y-function-calling-google-universal-context.py delete mode 100644 examples/foundational/14z-function-calling-anthropic-universal-context.py diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index d04ff6df7..3141a69d6 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -9,14 +9,11 @@ import os from dotenv import load_dotenv from loguru import logger -from pipecat.frames.frames import EndFrame +from pipecat.frames.frames import EndFrame, LLMContextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -63,7 +60,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Register an event handler so we can play the audio when the client joins @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): - await task.queue_frames([OpenAILLMContextFrame(OpenAILLMContext(messages)), EndFrame()]) + await task.queue_frames([LLMContextFrame(LLMContext(messages)), EndFrame()]) runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) diff --git a/examples/foundational/04-transports-small-webrtc.py b/examples/foundational/04-transports-small-webrtc.py index 54c63117e..997b917f9 100644 --- a/examples/foundational/04-transports-small-webrtc.py +++ b/examples/foundational/04-transports-small-webrtc.py @@ -25,7 +25,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService @@ -80,8 +81,8 @@ async def run_example(webrtc_connection: SmallWebRTCConnection): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/04a-transports-daily.py b/examples/foundational/04a-transports-daily.py index 2b8169eb5..5d2855450 100644 --- a/examples/foundational/04a-transports-daily.py +++ b/examples/foundational/04a-transports-daily.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.daily import configure from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.openai.llm import OpenAILLMService @@ -64,8 +65,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/04b-transports-livekit.py b/examples/foundational/04b-transports-livekit.py index 9c4724266..4e5a7e820 100644 --- a/examples/foundational/04b-transports-livekit.py +++ b/examples/foundational/04b-transports-livekit.py @@ -26,7 +26,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.livekit import configure from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService @@ -73,8 +74,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index c4968245a..cdcd05350 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -14,6 +14,7 @@ from loguru import logger from pipecat.frames.frames import ( DataFrame, Frame, + LLMContextFrame, LLMFullResponseStartFrame, TextFrame, ) @@ -21,10 +22,8 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.sync_parallel_pipeline import SyncParallelPipeline from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.sentence import SentenceAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments @@ -156,7 +155,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): } ] frames.append(MonthFrame(month=month)) - frames.append(OpenAILLMContextFrame(OpenAILLMContext(messages))) + frames.append(LLMContextFrame(LLMContext(messages))) task = PipelineTask( pipeline, diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index f40d2a163..48fed091e 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -15,6 +15,7 @@ from loguru import logger from pipecat.frames.frames import ( Frame, + LLMContextFrame, OutputAudioRawFrame, TextFrame, TTSAudioRawFrame, @@ -24,10 +25,8 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.sync_parallel_pipeline import SyncParallelPipeline from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.sentence import SentenceAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaHttpTTSService @@ -140,7 +139,7 @@ async def main(): ) task = PipelineTask(pipeline) - await task.queue_frame(OpenAILLMContextFrame(OpenAILLMContext(messages))) + await task.queue_frame(LLMContextFrame(LLMContext(messages))) await task.stop_when_done() await runner.run(task) diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index b48933e00..92e04e25a 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -23,7 +23,8 @@ from pipecat.metrics.metrics import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -103,8 +104,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index c5cbcff81..4f8816df6 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -24,7 +24,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -116,8 +117,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) image_sync_aggregator = ImageSyncAggregator( os.path.join(os.path.dirname(__file__), "assets", "speaking.png"), diff --git a/examples/foundational/07-interruptible-cartesia-http.py b/examples/foundational/07-interruptible-cartesia-http.py index d2e0f9c08..d121e40ba 100644 --- a/examples/foundational/07-interruptible-cartesia-http.py +++ b/examples/foundational/07-interruptible-cartesia-http.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaHttpTTSService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 5beddf7e1..81ba692c7 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -73,8 +74,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07a-interruptible-speechmatics-vad.py b/examples/foundational/07a-interruptible-speechmatics-vad.py index 09777c9b7..55514017f 100644 --- a/examples/foundational/07a-interruptible-speechmatics-vad.py +++ b/examples/foundational/07a-interruptible-speechmatics-vad.py @@ -13,10 +13,11 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMUserAggregatorParams, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.elevenlabs.tts import ElevenLabsTTSService @@ -123,8 +124,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), ) diff --git a/examples/foundational/07a-interruptible-speechmatics.py b/examples/foundational/07a-interruptible-speechmatics.py index 1d5806340..3d1e639b9 100644 --- a/examples/foundational/07a-interruptible-speechmatics.py +++ b/examples/foundational/07a-interruptible-speechmatics.py @@ -17,10 +17,11 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMUserAggregatorParams, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.elevenlabs.tts import ElevenLabsTTSService @@ -112,8 +113,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), ) diff --git a/examples/foundational/07aa-interruptible-soniox.py b/examples/foundational/07aa-interruptible-soniox.py index 16180e77c..addfde49a 100644 --- a/examples/foundational/07aa-interruptible-soniox.py +++ b/examples/foundational/07aa-interruptible-soniox.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -73,8 +74,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ab-interruptible-inworld-http.py b/examples/foundational/07ab-interruptible-inworld-http.py index 5fd761bfb..907a358d2 100644 --- a/examples/foundational/07ab-interruptible-inworld-http.py +++ b/examples/foundational/07ab-interruptible-inworld-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -84,8 +85,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ac-interruptible-asyncai-http.py b/examples/foundational/07ac-interruptible-asyncai-http.py index d3f286393..7a92f0f7c 100644 --- a/examples/foundational/07ac-interruptible-asyncai-http.py +++ b/examples/foundational/07ac-interruptible-asyncai-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.asyncai.tts import AsyncAIHttpTTSService @@ -79,8 +80,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ac-interruptible-asyncai.py b/examples/foundational/07ac-interruptible-asyncai.py index 851fa4338..8216c317b 100644 --- a/examples/foundational/07ac-interruptible-asyncai.py +++ b/examples/foundational/07ac-interruptible-asyncai.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.asyncai.tts import AsyncAITTSService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index 2c6e653b1..aa647ff33 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -98,8 +99,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index 3f9330bbd..fc43bb134 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -71,8 +72,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index 9c4cdb204..7dd46124d 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -72,8 +73,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index 27bd212fc..a6b0f1fd7 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -79,8 +80,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index ca40f6e4c..da2a8eb00 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07e-interruptible-playht-http.py b/examples/foundational/07e-interruptible-playht-http.py index be5769fbf..0026afc86 100644 --- a/examples/foundational/07e-interruptible-playht-http.py +++ b/examples/foundational/07e-interruptible-playht-http.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 6b2b8fde8..e854a61cb 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index 50a589951..f829063d3 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.azure.llm import AzureLLMService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index c73f074f7..d3e5ee373 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.openai.llm import OpenAILLMService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 1fbb716a4..313843b31 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -80,8 +81,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 675588f41..029961040 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -78,8 +79,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 3e24a7614..20ddc3f66 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -84,8 +85,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index a9da69f2c..daa944cce 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -71,8 +72,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07l-interruptible-groq.py b/examples/foundational/07l-interruptible-groq.py index 7a57bfe3e..6f90d0d8d 100644 --- a/examples/foundational/07l-interruptible-groq.py +++ b/examples/foundational/07l-interruptible-groq.py @@ -18,8 +18,9 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.groq.llm import GroqLLMService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05) ) diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index 754fd3b69..9343797a9 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -16,7 +16,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.aws.llm import AWSBedrockLLMService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07n-interruptible-gemini.py b/examples/foundational/07n-interruptible-gemini.py index f12693bd6..4da14f908 100644 --- a/examples/foundational/07n-interruptible-gemini.py +++ b/examples/foundational/07n-interruptible-gemini.py @@ -36,7 +36,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.llm import GoogleLLMService @@ -112,8 +113,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index f4ef03aee..203099c59 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.llm import GoogleLLMService @@ -84,8 +85,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 57e12e7fc..1b7a1daea 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.assemblyai.stt import AssemblyAISTTService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 3af964e41..3cbe7f28a 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07q-interruptible-rime-http.py b/examples/foundational/07q-interruptible-rime-http.py index 82b581d16..2e1732c6b 100644 --- a/examples/foundational/07q-interruptible-rime-http.py +++ b/examples/foundational/07q-interruptible-rime-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -80,8 +81,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 18fb459ec..9ba070151 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index ae1556a58..4f451fd53 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.nim.llm import NimLLMService @@ -71,8 +72,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 06e59b76d..40759f262 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -31,7 +31,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -244,8 +245,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) audio_collector = UserAudioCollector(context, context_aggregator.user()) pull_transcript_out_of_llm_output = TranscriptExtractor(context) fixup_context_messages = TranscriptionContextFixup(context) diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index 161e5c724..52c4ebbb1 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07v-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py index b63fc2fdf..e22ef734e 100644 --- a/examples/foundational/07v-interruptible-neuphonic-http.py +++ b/examples/foundational/07v-interruptible-neuphonic-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -79,8 +80,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07v-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py index c9265b0f0..c9bb13a70 100644 --- a/examples/foundational/07v-interruptible-neuphonic.py +++ b/examples/foundational/07v-interruptible-neuphonic.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py index dcef8dddf..dc572f652 100644 --- a/examples/foundational/07w-interruptible-fal.py +++ b/examples/foundational/07w-interruptible-fal.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07x-interruptible-local.py b/examples/foundational/07x-interruptible-local.py index 8d3ba8d17..4778d262f 100644 --- a/examples/foundational/07x-interruptible-local.py +++ b/examples/foundational/07x-interruptible-local.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -59,8 +60,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07y-interruptible-minimax.py b/examples/foundational/07y-interruptible-minimax.py index c499b8318..7995ed832 100644 --- a/examples/foundational/07y-interruptible-minimax.py +++ b/examples/foundational/07y-interruptible-minimax.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07z-interruptible-sarvam-http.py b/examples/foundational/07z-interruptible-sarvam-http.py index 7140e4c1f..29851d254 100644 --- a/examples/foundational/07z-interruptible-sarvam-http.py +++ b/examples/foundational/07z-interruptible-sarvam-http.py @@ -18,7 +18,8 @@ from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -79,8 +80,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07z-interruptible-sarvam.py b/examples/foundational/07z-interruptible-sarvam.py index 6b027f3dd..44e0b7844 100644 --- a/examples/foundational/07z-interruptible-sarvam.py +++ b/examples/foundational/07z-interruptible-sarvam.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py index a5d1b98f6..b84e945c3 100644 --- a/examples/foundational/08-bots-arguing.py +++ b/examples/foundational/08-bots-arguing.py @@ -6,13 +6,11 @@ from typing import Tuple import aiohttp from dotenv import load_dotenv -from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, TextFrame +from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMContextFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.aggregators import SentenceAggregator -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.daily import configure from pipecat.services.azure import AzureLLMService, AzureTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService @@ -83,7 +81,7 @@ async def main(): sentence_aggregator = SentenceAggregator() pipeline = Pipeline([llm, sentence_aggregator, tts1], source_queue, sink_queue) - await source_queue.put(OpenAILLMContextFrame(OpenAILLMContext(messages))) + await source_queue.put(LLMContextFrame(LLMContext(messages))) await source_queue.put(EndFrame()) await pipeline.run_pipeline() diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index d86f40241..30a7d6ca9 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.wake_check_filter import WakeCheckFilter from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -76,8 +77,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): hey_robot_filter = WakeCheckFilter(["hey robot", "hey, robot"]) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 61e6326dd..9eae5944f 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -16,6 +16,7 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( Frame, + LLMContextFrame, LLMFullResponseEndFrame, OutputAudioRawFrame, TTSSpeakFrame, @@ -23,10 +24,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.logger import FrameLogger from pipecat.runner.types import RunnerArguments @@ -74,7 +73,7 @@ class InboundSoundEffectWrapper(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, LLMContextFrame): await self.push_frame(sounds["ding2.wav"]) # In case anything else downstream needs it await self.push_frame(frame, direction) @@ -126,8 +125,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) out_sound = OutboundSoundEffectWrapper() in_sound = InboundSoundEffectWrapper() fl = FrameLogger("LLM Out") diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index e9e96a989..42c728bae 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -123,8 +124,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 038d475ce..5f9a0ec06 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService @@ -118,8 +119,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "user", "content": "Say 'hello' to start the conversation."}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14aa-function-calling-aws-universal-context.py b/examples/foundational/14aa-function-calling-aws-universal-context.py deleted file mode 100644 index 1a7960811..000000000 --- a/examples/foundational/14aa-function-calling-aws-universal-context.py +++ /dev/null @@ -1,219 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.aws.llm import AWSBedrockLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.services.daily import DailyParams - -load_dotenv(override=True) - - -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def get_image(params: FunctionCallParams): - question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") - - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, - ) - - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AWSBedrockLLMService( - aws_region="us-west-2", - model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", - # Note: usually, prefer providing latency="optimized" param. - # Here we can't because AWS Bedrock doesn't support it for Claude 3.7, - # which we need for image input. - params=AWSBedrockLLMService.InputParams(temperature=0.8), - ) - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) - - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to two tools: get_weather and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? - -If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise. - """ - - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Start the conversation by introducing yourself."}, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User speech to text - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - global client_id - client_id = get_transport_client_id(transport, client) - - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 2905509ef..009f59500 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -165,8 +166,8 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo {"role": "user", "content": "Start the conversation by introducing yourself."}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 904c7caf3..9e626ef14 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -109,8 +110,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 37bc09f94..48cf95ee9 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -154,8 +155,8 @@ indicate you should use the get_image tool are: {"role": "system", "content": system_prompt}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14e-function-calling-google.py b/examples/foundational/14e-function-calling-google.py index 60b62f22b..528051d8e 100644 --- a/examples/foundational/14e-function-calling-google.py +++ b/examples/foundational/14e-function-calling-google.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -175,8 +176,8 @@ indicate you should use the get_image tool are: {"role": "user", "content": "Say hello."}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 6a9f7989a..568cf43be 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -20,8 +20,9 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -107,8 +108,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05) ) diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 0f332aa4e..a9bba2c4d 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -102,8 +103,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 757020852..ab72e2bcc 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.azure.llm import AzureLLMService @@ -110,8 +111,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 11a0b6fee..96dce71e6 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -113,8 +114,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 678792501..579a21b95 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -107,8 +108,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 99f62ab64..93dbee8b1 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -116,8 +117,8 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index d2ca39755..e7de42a43 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -116,8 +117,8 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index ebcdaf9e8..93221e1da 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.azure.tts import AzureTTSService @@ -110,8 +111,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 4c046135f..2f37768b7 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -24,7 +24,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -80,8 +81,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index 17abe1179..f9ad264eb 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -112,8 +113,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14q-function-calling-qwen.py b/examples/foundational/14q-function-calling-qwen.py index 7f09c412d..650142592 100644 --- a/examples/foundational/14q-function-calling-qwen.py +++ b/examples/foundational/14q-function-calling-qwen.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -108,8 +109,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index 95899dbe8..03aa7bb96 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.aws.llm import AWSBedrockLLMService @@ -123,8 +124,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14s-function-calling-sambanova.py b/examples/foundational/14s-function-calling-sambanova.py index eed121cdb..e425ccd29 100644 --- a/examples/foundational/14s-function-calling-sambanova.py +++ b/examples/foundational/14s-function-calling-sambanova.py @@ -20,8 +20,9 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -113,8 +114,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05) ) diff --git a/examples/foundational/14t-function-calling-direct.py b/examples/foundational/14t-function-calling-direct.py index 978513b4d..872cb2a8e 100644 --- a/examples/foundational/14t-function-calling-direct.py +++ b/examples/foundational/14t-function-calling-direct.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -109,8 +110,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14u-function-calling-ollama.py b/examples/foundational/14u-function-calling-ollama.py index c49ea2cf9..3e87c601a 100644 --- a/examples/foundational/14u-function-calling-ollama.py +++ b/examples/foundational/14u-function-calling-ollama.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -125,8 +126,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14v-function-calling-openai.py b/examples/foundational/14v-function-calling-openai.py index ba4a99117..d259e7131 100644 --- a/examples/foundational/14v-function-calling-openai.py +++ b/examples/foundational/14v-function-calling-openai.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.llm_service import FunctionCallParams @@ -131,8 +132,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14w-function-calling-mistral.py b/examples/foundational/14w-function-calling-mistral.py index f68143073..ce61085e2 100644 --- a/examples/foundational/14w-function-calling-mistral.py +++ b/examples/foundational/14w-function-calling-mistral.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -119,8 +120,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14x-function-calling-universal-context.py b/examples/foundational/14x-function-calling-universal-context.py deleted file mode 100644 index 42c728bae..000000000 --- a/examples/foundational/14x-function-calling-universal-context.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - - -async def fetch_weather_from_api(params: FunctionCallParams): - await params.result_callback({"conditions": "nice", "temperature": "75"}) - - -async def fetch_restaurant_recommendation(params: FunctionCallParams): - await params.result_callback({"name": "The Golden Dragon"}) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) - llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) - - @llm.event_handler("on_function_calls_started") - async def on_function_calls_started(service, function_calls): - await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) - - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location", "format"], - ) - restaurant_function = FunctionSchema( - name="get_restaurant_recommendation", - description="Get a restaurant recommendation", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/14y-function-calling-google-universal-context.py b/examples/foundational/14y-function-calling-google-universal-context.py deleted file mode 100644 index 528051d8e..000000000 --- a/examples/foundational/14y-function-calling-google-universal-context.py +++ /dev/null @@ -1,234 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.google.llm import GoogleLLMService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams - -load_dotenv(override=True) - - -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def fetch_restaurant_recommendation(params: FunctionCallParams): - await params.result_callback({"name": "The Golden Dragon"}) - - -async def get_image(params: FunctionCallParams): - question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") - - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, - ) - - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) - llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) - - @llm.event_handler("on_function_calls_started") - async def on_function_calls_started(service, function_calls): - await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) - - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location", "format"], - ) - restaurant_function = FunctionSchema( - name="get_restaurant_recommendation", - description="Get a restaurant recommendation", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function, restaurant_function]) - - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to three tools: get_weather, get_restaurant_recommendation, and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? -""" - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Say hello."}, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - global client_id - client_id = get_transport_client_id(transport, client) - - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/14z-function-calling-anthropic-universal-context.py b/examples/foundational/14z-function-calling-anthropic-universal-context.py deleted file mode 100644 index 253ebf4d0..000000000 --- a/examples/foundational/14z-function-calling-anthropic-universal-context.py +++ /dev/null @@ -1,216 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.services.daily import DailyParams - -load_dotenv(override=True) - - -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def get_image(params: FunctionCallParams): - question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") - - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, - ) - - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - model="claude-3-7-sonnet-latest", - params=AnthropicLLMService.InputParams(enable_prompt_caching=True), - ) - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) - - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to two tools: get_weather and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? - -If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise. - """ - - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Start the conversation by introducing yourself."}, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User speech to text - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - global client_id - client_id = get_transport_client_id(transport, client) - - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 598f007af..bd544462e 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -20,7 +20,8 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -146,8 +147,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index af8357065..f5e92af4f 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -21,7 +21,8 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -137,8 +138,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index cbf2de9dc..89316e63c 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 5c76f3d48..cc36d891d 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import EndFrame, LLMMessagesAppendFrame, LLMRunFrame, from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.user_idle_processor import UserIdleProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) async def handle_user_idle(user_idle: UserIdleProcessor, retry_count: int) -> bool: if retry_count == 1: diff --git a/examples/foundational/21-tavus-transport.py b/examples/foundational/21-tavus-transport.py index 406c41928..4aabaa23e 100644 --- a/examples/foundational/21-tavus-transport.py +++ b/examples/foundational/21-tavus-transport.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService @@ -64,8 +65,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/21a-tavus-video-service.py b/examples/foundational/21a-tavus-video-service.py index fd4c82e54..0aa2f7e17 100644 --- a/examples/foundational/21a-tavus-video-service.py +++ b/examples/foundational/21a-tavus-video-service.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -83,8 +84,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index a6747f3d6..b34947c2b 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, MixerEnableFrame, MixerUpdateSett from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -93,8 +94,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index 30bf59303..6c4804c37 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -114,8 +115,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 36827b8f3..9479632b5 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/28-transcription-processor.py b/examples/foundational/28-transcription-processor.py index b2a9f435e..6aa3168ac 100644 --- a/examples/foundational/28-transcription-processor.py +++ b/examples/foundational/28-transcription-processor.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage, Transcripti from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -137,8 +138,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) # Create transcript processor and handler transcript = TranscriptProcessor() diff --git a/examples/foundational/29-turn-tracking-observer.py b/examples/foundational/29-turn-tracking-observer.py index 5b6d8a4d5..83faa71dc 100644 --- a/examples/foundational/29-turn-tracking-observer.py +++ b/examples/foundational/29-turn-tracking-observer.py @@ -19,7 +19,8 @@ from pipecat.observers.loggers.user_bot_latency_log_observer import UserBotLaten from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 9912dc818..f782e7c36 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -21,7 +21,8 @@ from pipecat.observers.base_observer import BaseObserver, FramePushed from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -115,7 +116,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tools=tools, ) - context = OpenAILLMContext( + context = LLMContext( [ { "role": "user", @@ -123,7 +124,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): } ], ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index 3f03ec25e..cd5250dfc 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -58,7 +58,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -133,8 +134,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 9b33c36ec..7ed9eb268 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -52,7 +52,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -194,8 +195,8 @@ Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue."" }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) # Create pipeline pipeline = Pipeline( diff --git a/examples/foundational/38-smart-turn-fal.py b/examples/foundational/38-smart-turn-fal.py index 6de739b04..03792c11d 100644 --- a/examples/foundational/38-smart-turn-fal.py +++ b/examples/foundational/38-smart-turn-fal.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/38a-smart-turn-local-coreml.py b/examples/foundational/38a-smart-turn-local-coreml.py index b7bf1c14b..fc11b4137 100644 --- a/examples/foundational/38a-smart-turn-local-coreml.py +++ b/examples/foundational/38a-smart-turn-local-coreml.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -97,8 +98,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/38b-smart-turn-local.py b/examples/foundational/38b-smart-turn-local.py index 81952b2c3..a99b870ef 100644 --- a/examples/foundational/38b-smart-turn-local.py +++ b/examples/foundational/38b-smart-turn-local.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index 2e709304d..c51de3284 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -29,7 +29,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -157,8 +158,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py index c4e5d1df7..1777f9ce5 100644 --- a/examples/foundational/39a-mcp-run-sse.py +++ b/examples/foundational/39a-mcp-run-sse.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService @@ -93,8 +94,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39b-multiple-mcp.py index 9faae1b2b..36ec13d60 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39b-multiple-mcp.py @@ -32,7 +32,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -169,8 +170,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): all_standard_tools = run_tools.standard_tools + tools.standard_tools all_tools = ToolsSchema(standard_tools=all_standard_tools) - context = OpenAILLMContext(messages, all_tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, all_tools) + context_aggregator = LLMContextAggregatorPair(context) mcp_image_processor = UrlToImageProcessor(aiohttp_session=session) pipeline = Pipeline( diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39c-mcp-run-http.py index 3365a9fe2..38234de1c 100644 --- a/examples/foundational/39c-mcp-run-http.py +++ b/examples/foundational/39c-mcp-run-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -97,8 +98,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/41a-text-only-webrtc.py b/examples/foundational/41a-text-only-webrtc.py index 95b14c5db..3fd73f89a 100644 --- a/examples/foundational/41a-text-only-webrtc.py +++ b/examples/foundational/41a-text-only-webrtc.py @@ -16,7 +16,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import ( ActionResult, RTVIAction, @@ -91,8 +92,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) action_llm_append_to_messages = create_action_llm_append_to_messages(context_aggregator) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) diff --git a/examples/foundational/41b-text-and-audio-webrtc.py b/examples/foundational/41b-text-and-audio-webrtc.py index 338987e7e..f2b2da6b1 100644 --- a/examples/foundational/41b-text-and-audio-webrtc.py +++ b/examples/foundational/41b-text-and-audio-webrtc.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import ( ActionResult, RTVIAction, @@ -107,8 +108,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) action_llm_append_to_messages = create_action_llm_append_to_messages(context_aggregator) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) diff --git a/examples/foundational/42-interruption-config.py b/examples/foundational/42-interruption-config.py index 9c5fa0cfc..c3c899b89 100644 --- a/examples/foundational/42-interruption-config.py +++ b/examples/foundational/42-interruption-config.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/43a-heygen-video-service.py b/examples/foundational/43a-heygen-video-service.py index 81aac51ff..9f25a0cfb 100644 --- a/examples/foundational/43a-heygen-video-service.py +++ b/examples/foundational/43a-heygen-video-service.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -86,8 +87,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 2e718b156..080bd1ca6 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import EndTaskFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -78,8 +79,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/45-before-and-after-events.py b/examples/foundational/45-before-and-after-events.py index c84053bbd..fd7bdfa5b 100644 --- a/examples/foundational/45-before-and-after-events.py +++ b/examples/foundational/45-before-and-after-events.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import DataFrame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -85,8 +86,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/quickstart/bot.py b/examples/quickstart/bot.py index ee97175a6..f353ecf6f 100644 --- a/examples/quickstart/bot.py +++ b/examples/quickstart/bot.py @@ -43,7 +43,8 @@ logger.info("Loading pipeline components...") from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 0506136ff..f17a7f6fa 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -135,25 +135,6 @@ TESTS_14 = [ ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14x-function-calling-universal-context.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ( - "14y-function-calling-google-universal-context.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), - ( - "14z-function-calling-anthropic-universal-context.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), - ( - "14aa-function-calling-aws-universal-context.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), # Currently not working. # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), From b04e4943733ba634473248bda900fde9dfc5e73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 22 Sep 2025 16:45:58 -0700 Subject: [PATCH 09/35] DailyTransport: remove deprecated note and double registration --- src/pipecat/transports/daily/transport.py | 3 --- src/pipecat/utils/base_object.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index bf0036950..030f1ad6a 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -1974,9 +1974,6 @@ class DailyTransport(BaseTransport): self._register_event_handler("on_recording_stopped") self._register_event_handler("on_recording_error") self._register_event_handler("on_before_leave", sync=True) - # Deprecated - self._register_event_handler("on_joined") - self._register_event_handler("on_left") # # BaseTransport diff --git a/src/pipecat/utils/base_object.py b/src/pipecat/utils/base_object.py index a1c75f733..f0744f3b8 100644 --- a/src/pipecat/utils/base_object.py +++ b/src/pipecat/utils/base_object.py @@ -139,7 +139,7 @@ class BaseObject(ABC): name=event_name, handlers=[], is_sync=sync ) else: - logger.warning(f"Event handler {event_name} not registered") + logger.warning(f"Event handler {event_name} already registered") async def _call_event_handler(self, event_name: str, *args, **kwargs): """Call all registered handlers for the specified event. From 75c0b089e09b21e6de0c791cf214121aeffce146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 22 Sep 2025 22:38:12 -0700 Subject: [PATCH 10/35] PipelineRunner: use signal.signal() on Windows --- CHANGELOG.md | 3 +++ src/pipecat/pipeline/runner.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d905d43..ee54ce7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `PipelineRunner` issue on Windows where setting up SIGINT and SIGTERM + was raising an exception. + - Fixed an issue where multiple handlers for an event would not run in parallel. - Fixed `DailyTransport.sip_call_transfer()` to automatically use the session diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index e8c325521..9d82fcd88 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -106,13 +106,21 @@ class PipelineRunner(BaseObject): def _setup_sigint(self): """Set up signal handlers for graceful shutdown.""" - loop = asyncio.get_running_loop() - loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler()) + try: + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler()) + except NotImplementedError: + # Windows fallback + signal.signal(signal.SIGINT, lambda s, f: self._sig_handler()) def _setup_sigterm(self): """Set up signal handlers for graceful shutdown.""" - loop = asyncio.get_running_loop() - loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler()) + try: + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler()) + except NotImplementedError: + # Windows fallback + signal.signal(signal.SIGTERM, lambda s, f: self._sig_handler()) def _sig_handler(self): """Handle interrupt signals by cancelling all tasks.""" From 1df36601860f4bd47c5f5f55b00bdd5212304aa0 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 08:03:35 -0300 Subject: [PATCH 11/35] Not storing anymore the last frames received to display them in the idle processor. --- src/pipecat/pipeline/task.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 57e4621ce..35d1971a5 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -777,7 +777,6 @@ class PipelineTask(BasePipelineTask): """ running = True last_frame_time = 0 - frame_buffer = deque(maxlen=10) # Store last 10 frames while running: try: @@ -785,9 +784,6 @@ class PipelineTask(BasePipelineTask): self._idle_queue.get(), timeout=self._idle_timeout_secs ) - if not isinstance(frame, InputAudioRawFrame): - frame_buffer.append(frame) - if isinstance(frame, StartFrame) or isinstance(frame, self._idle_timeout_frames): # If we find a StartFrame or one of the frames that prevents a # time out we update the time. @@ -798,7 +794,7 @@ class PipelineTask(BasePipelineTask): # valid frames. diff_time = time.time() - last_frame_time if diff_time >= self._idle_timeout_secs: - running = await self._idle_timeout_detected(frame_buffer) + running = await self._idle_timeout_detected() # Reset `last_frame_time` so we don't trigger another # immediate idle timeout if we are not cancelling. For # example, we might want to force the bot to say goodbye @@ -808,14 +804,11 @@ class PipelineTask(BasePipelineTask): self._idle_queue.task_done() except asyncio.TimeoutError: - running = await self._idle_timeout_detected(frame_buffer) + running = await self._idle_timeout_detected() - async def _idle_timeout_detected(self, last_frames: Deque[Frame]) -> bool: + async def _idle_timeout_detected(self) -> bool: """Handle idle timeout detection and optional cancellation. - Args: - last_frames: Recent frames received before timeout for debugging. - Returns: Whether the pipeline task should continue running. """ @@ -823,10 +816,7 @@ class PipelineTask(BasePipelineTask): if self._cancelled: return True - logger.warning("Idle timeout detected. Last 10 frames received:") - for i, frame in enumerate(last_frames, 1): - logger.warning(f"Frame {i}: {frame}") - + logger.warning("Idle timeout detected.") await self._call_event_handler("on_idle_timeout") if self._cancel_on_idle_timeout: logger.warning(f"Idle pipeline detected, cancelling pipeline task...") From 8bf6a4c66fca3db74fc6055006dc0837fb4f2b2e Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 08:03:55 -0300 Subject: [PATCH 12/35] Improving memory cleanup inside the SmallWebRTCTransport. --- .../transports/smallwebrtc/transport.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index 2abe2abf5..0ec8bf998 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -309,7 +309,7 @@ class SmallWebRTCClient: # self._webrtc_connection.ask_to_renegotiate() frame = None except MediaStreamError: - logger.warning("Received an unexpected media stream error while reading the audio.") + logger.warning("Received an unexpected media stream error while reading the video.") frame = None if frame is None or not isinstance(frame, VideoFrame): @@ -321,15 +321,21 @@ class SmallWebRTCClient: # Convert frame to NumPy array in its native format frame_array = frame.to_ndarray(format=format_name) frame_rgb = self._convert_frame(frame_array, format_name) + del frame_array # free intermediate array immediately + image_bytes = frame_rgb.tobytes() + del frame_rgb # free RGB array immediately image_frame = UserImageRawFrame( user_id=self._webrtc_connection.pc_id, - image=frame_rgb.tobytes(), + image=image_bytes, size=(frame.width, frame.height), format="RGB", ) image_frame.transport_source = video_source + del frame # free original VideoFrame + del image_bytes # reference kept in image_frame + yield image_frame async def read_audio_frame(self): @@ -364,23 +370,35 @@ class SmallWebRTCClient: resampled_frames = self._pipecat_resampler.resample(frame) for resampled_frame in resampled_frames: # 16-bit PCM bytes - pcm_bytes = resampled_frame.to_ndarray().astype(np.int16).tobytes() + pcm_array = resampled_frame.to_ndarray().astype(np.int16) + pcm_bytes = pcm_array.tobytes() + del pcm_array # free NumPy array immediately + audio_frame = InputAudioRawFrame( audio=pcm_bytes, sample_rate=resampled_frame.sample_rate, num_channels=self._audio_in_channels, ) + del pcm_bytes # reference kept in audio_frame + yield audio_frame else: # 16-bit PCM bytes - pcm_bytes = frame.to_ndarray().astype(np.int16).tobytes() + pcm_array = frame.to_ndarray().astype(np.int16) + pcm_bytes = pcm_array.tobytes() + del pcm_array # free NumPy array immediately + audio_frame = InputAudioRawFrame( audio=pcm_bytes, sample_rate=frame.sample_rate, num_channels=self._audio_in_channels, ) + del pcm_bytes # reference kept in audio_frame + yield audio_frame + del frame # free original AudioFrame + async def write_audio_frame(self, frame: OutputAudioRawFrame): """Write an audio frame to the WebRTC connection. From 7c569b3863fccc13eb4840f085f5d92c5822f162 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 08:04:06 -0300 Subject: [PATCH 13/35] Calling task_done when reading the audio from the queue. --- src/pipecat/transports/base_output.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 0316df194..ed60a4584 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -659,6 +659,7 @@ class BaseOutputTransport(FrameProcessor): self._audio_queue.get(), timeout=vad_stop_secs ) yield frame + self._audio_queue.task_done() except asyncio.TimeoutError: # Notify the bot stopped speaking upstream if necessary. await self._bot_stopped_speaking() @@ -673,6 +674,7 @@ class BaseOutputTransport(FrameProcessor): frame.audio = await self._mixer.mix(frame.audio) last_frame_time = time.time() yield frame + self._audio_queue.task_done() except asyncio.QueueEmpty: # Notify the bot stopped speaking upstream if necessary. diff_time = time.time() - last_frame_time From eaecefe6752eb63cf907328bc35528e09a94883a Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 08:05:15 -0300 Subject: [PATCH 14/35] Refactoring how we are reading the image bytes inside the base llm. --- src/pipecat/services/openai/base_llm.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 6f6376c02..47127b77a 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -281,8 +281,10 @@ class BaseOpenAILLMService(LLMService): # base64 encode any images for message in messages: if message.get("mime_type") == "image/jpeg": - encoded_image = base64.b64encode(message["data"].getvalue()).decode("utf-8") - text = message["content"] + # Avoid .getvalue() which makes a full copy of BytesIO + raw_bytes = message["data"].read() + encoded_image = base64.b64encode(raw_bytes).decode("utf-8") + text = message.get("content", "") message["content"] = [ {"type": "text", "text": text}, { @@ -290,6 +292,7 @@ class BaseOpenAILLMService(LLMService): "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}, }, ] + # Explicit cleanup del message["data"] del message["mime_type"] From 1647b5b665a23be50313d61dcc047ee915586cc1 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 08:05:28 -0300 Subject: [PATCH 15/35] Created a new example using the video processor to make it easier to investigate memory leaks. --- examples/foundational/46-video-processing.py | 175 +++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 examples/foundational/46-video-processing.py diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py new file mode 100644 index 000000000..bfb718ff2 --- /dev/null +++ b/examples/foundational/46-video-processing.py @@ -0,0 +1,175 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +import os + +import cv2 +import numpy as np +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import Frame, InputImageRawFrame, LLMRunFrame, OutputImageRawFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.daily.transport import DailyParams, DailyTransport + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_10ms_chunks=2, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_10ms_chunks=2, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +class EdgeDetectionProcessor(FrameProcessor): + def __init__(self, video_out_width, video_out_height: int): + super().__init__() + self._video_out_width = video_out_width + self._video_out_height = video_out_height + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # Send back the user's camera video with edge detection applied + if isinstance(frame, InputImageRawFrame) and frame.transport_source == "camera": + # Convert bytes to NumPy array + img = np.frombuffer(frame.image, dtype=np.uint8).reshape( + (frame.size[1], frame.size[0], 3) + ) + + # perform edge detection only on camera frames + img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR) + + # convert the size if needed + desired_size = (self._video_out_width, self._video_out_height) + if frame.size != desired_size: + resized_image = cv2.resize(img, desired_size) + out_frame = OutputImageRawFrame(resized_image.tobytes(), desired_size, frame.format) + await self.push_frame(out_frame) + else: + out_frame = OutputImageRawFrame( + image=img.tobytes(), size=frame.size, format=frame.format + ) + await self.push_frame(out_frame) + else: + await self.push_frame(frame, direction) + + +SYSTEM_INSTRUCTION = f""" +"You are Gemini Chatbot, a friendly, helpful robot. + +Your goal is to demonstrate your capabilities in a succinct way. + +Your output will be converted to audio so don't include special characters in your answers. + +Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most. +""" + + +async def run_bot(pipecat_transport): + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck + transcribe_user_audio=True, + system_instruction=SYSTEM_INSTRUCTION, + ) + + messages = [ + { + "role": "user", + "content": "Start by greeting the user warmly and introducing yourself.", + } + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # RTVI events for Pipecat client UI + rtvi = RTVIProcessor() + + pipeline = Pipeline( + [ + pipecat_transport.input(), + context_aggregator.user(), + rtvi, + llm, # LLM + EdgeDetectionProcessor( + pipecat_transport._params.video_out_width, + pipecat_transport._params.video_out_height, + ), # Sending the video back to the user + pipecat_transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + observers=[RTVIObserver(rtvi)], + ) + + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + logger.info("Pipecat client ready.") + await rtvi.set_bot_ready() + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @pipecat_transport.event_handler("on_client_connected") + async def on_client_connected(transport, participant): + logger.info("Pipecat Client connected") + if isinstance(transport, DailyTransport): + await pipecat_transport.capture_participant_video(participant["id"], framerate=30) + else: + await pipecat_transport.capture_participant_video("camera") + + @pipecat_transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info("Pipecat Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False, force_gc=True) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() From a7bfac8d68db6886a57802f427ea7ca63ab42ab3 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 08:09:31 -0300 Subject: [PATCH 16/35] Mentioning the memory cleanups in the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d905d43..ca5d55089 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added memory cleanup improvements to reduce memory peaks. + - Added `on_before_process_frame`, `on_after_process_frame`, `on_before_push_frame` and `on_after_push_frame`. These are synchronous events that get called before and after a frame is processed or pushed. Note that From a5776b20ade37e3f5b09c4745a25a0d2078f492d Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 08:30:17 -0300 Subject: [PATCH 17/35] Monitoring the peer connection while it is in the *connecting* state. --- CHANGELOG.md | 4 ++ .../transports/smallwebrtc/connection.py | 46 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca5d55089..c413d16fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a peer connection monitor to the `SmallWebRTCConnection` that + automatically disconnects if the connection fails to establish within + the timeout (1 minute by default). + - Added memory cleanup improvements to reduce memory peaks. - Added `on_before_process_frame`, `on_after_process_frame`, diff --git a/src/pipecat/transports/smallwebrtc/connection.py b/src/pipecat/transports/smallwebrtc/connection.py index decd8ca58..45b1cffb2 100644 --- a/src/pipecat/transports/smallwebrtc/connection.py +++ b/src/pipecat/transports/smallwebrtc/connection.py @@ -206,11 +206,16 @@ class SmallWebRTCConnection(BaseObject): for real-time audio/video communication. """ - def __init__(self, ice_servers: Optional[Union[List[str], List[IceServer]]] = None): + def __init__( + self, + ice_servers: Optional[Union[List[str], List[IceServer]]] = None, + connection_timeout_secs: int = 60, + ): """Initialize the WebRTC connection. Args: ice_servers: List of ICE servers as URLs or IceServer objects. + connection_timeout_secs: Timeout in seconds for connecting to the peer. Raises: TypeError: If ice_servers contains mixed types or unsupported types. @@ -231,6 +236,7 @@ class SmallWebRTCConnection(BaseObject): VIDEO_TRANSCEIVER_INDEX: self.video_input_track, SCREEN_VIDEO_TRANSCEIVER_INDEX: self.screen_video_input_track, } + self.connection_timeout_secs = connection_timeout_secs self._initialize() @@ -279,6 +285,7 @@ class SmallWebRTCConnection(BaseObject): self._last_received_time = None self._message_queue = [] self._pending_app_messages = [] + self._connecting_timeout_task = None def _setup_listeners(self): """Set up event listeners for the peer connection.""" @@ -499,6 +506,7 @@ class SmallWebRTCConnection(BaseObject): self._message_queue.clear() self._pending_app_messages.clear() self._track_map = {} + self._cancel_monitoring_connecting_state() def get_answer(self): """Get the SDP answer for the current connection. @@ -516,9 +524,45 @@ class SmallWebRTCConnection(BaseObject): "pc_id": self._pc_id, } + def _monitoring_connecting_state(self) -> None: + """Start monitoring the peer connection while it is in the *connecting* state. + + This method schedules a timeout task that will automatically close the + connection if it remains in the connecting state for more than the specified + timeout, default to 60 seconds. + """ + logger.debug("Monitoring connecting state") + + async def timeout_handler(): + # We will close the connection in case we have remained in the connecting state for over 1 minute + await asyncio.sleep(self.connection_timeout_secs) + logger.warning("Timeout establishing the connection to the remote peer. Closing.") + + await self._close() + + # Create and store the timeout task + self._connecting_timeout_task = asyncio.create_task(timeout_handler()) + + def _cancel_monitoring_connecting_state(self) -> None: + """Cancel the ongoing connecting-state timeout task, if any. + + This method should be called once the connection has either succeeded or + transitioned out of the connecting state. If the timeout task is still + pending, it will be canceled and the reference cleared. + """ + if self._connecting_timeout_task and not self._connecting_timeout_task.done(): + logger.debug("Cancelling the connecting timeout task") + self._connecting_timeout_task.cancel() + self._connecting_timeout_task = None + async def _handle_new_connection_state(self): """Handle changes in the peer connection state.""" state = self._pc.connectionState + if state == "connecting": + self._monitoring_connecting_state() + else: + self._cancel_monitoring_connecting_state() + if state == "connected" and not self._connect_invoked: # We are going to wait until the pipeline is ready before triggering the event return From 6b2bf8de64332ecf88371f6b9fb974ab9414f537 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 23 Sep 2025 09:41:50 -0300 Subject: [PATCH 18/35] Fixing the ruff format and making the methods sync. --- .../observers/loggers/user_bot_latency_log_observer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py index e17387019..22e2cc358 100644 --- a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py +++ b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py @@ -67,8 +67,8 @@ class UserBotLatencyLogObserver(BaseObserver): self._user_stopped_time = 0 self._latencies.append(latency) self._log_latency(latency) - - async def _log_summary(self): + + def _log_summary(self): if not self._latencies: return avg_latency = mean(self._latencies) @@ -78,10 +78,12 @@ class UserBotLatencyLogObserver(BaseObserver): f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING - Avg: {avg_latency:.3f}s, Min: {min_latency:.3f}s, Max: {max_latency:.3f}s" ) - async def _log_latency(self, latency: float): + def _log_latency(self, latency: float): """Log the latency. Args: latency: The latency to log. """ - logger.debug(f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s") + logger.debug( + f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s" + ) From f96cbcce22a699aa9dfeb89517d0a66a728f2947 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 23 Sep 2025 09:30:14 -0400 Subject: [PATCH 19/35] Add support for universal `LLMContext` to `LLMLogObserver` --- examples/foundational/30-observer.py | 9 ++++----- src/pipecat/observers/loggers/llm_log_observer.py | 10 +++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index b5d7e51b1..54318884e 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -29,9 +29,8 @@ from pipecat.observers.loggers.llm_log_observer import LLMLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -124,8 +123,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/src/pipecat/observers/loggers/llm_log_observer.py b/src/pipecat/observers/loggers/llm_log_observer.py index 53a0ac484..a88f3ca2b 100644 --- a/src/pipecat/observers/loggers/llm_log_observer.py +++ b/src/pipecat/observers/loggers/llm_log_observer.py @@ -11,6 +11,7 @@ from loguru import logger from pipecat.frames.frames import ( FunctionCallInProgressFrame, FunctionCallResultFrame, + LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, @@ -79,10 +80,13 @@ class LLMLogObserver(BaseObserver): f"🧠 {arrow} {dst} LLM MESSAGES FRAME: {frame.messages} at {time_sec:.2f}s" ) # Log OpenAILLMContextFrame (input) - elif isinstance(frame, OpenAILLMContextFrame): - logger.debug( - f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {frame.context.messages} at {time_sec:.2f}s" + elif isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + messages = ( + frame.context.messages + if isinstance(frame, OpenAILLMContextFrame) + else frame.context.get_messages() ) + logger.debug(f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {messages} at {time_sec:.2f}s") # Log function call result (input) elif isinstance(frame, FunctionCallResultFrame): logger.debug( From 99731ca40ae124a408277314e7c6a7cb48798836 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 23 Sep 2025 09:52:13 -0400 Subject: [PATCH 20/35] Add support for universal `LLMContext` to `GatedOpenAILLMContextAggregator`, renaming it to `GatedLLMContextAggregator` in the process --- .../foundational/22-natural-conversation.py | 17 ++++++++--------- ...nai_llm_context.py => gated_llm_context.py} | 18 +++++++++--------- .../aggregators/gated_open_ai_llm_context.py | 12 ++++++++++++ 3 files changed, 29 insertions(+), 18 deletions(-) rename src/pipecat/processors/aggregators/{gated_openai_llm_context.py => gated_llm_context.py} (81%) create mode 100644 src/pipecat/processors/aggregators/gated_open_ai_llm_context.py diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 4907aa137..98ad23c0b 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -16,8 +16,9 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.gated_openai_llm_context import GatedOpenAILLMContextAggregator -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.gated_llm_context import GatedLLMContextAggregator +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.null_filter import NullFilter from pipecat.processors.filters.wake_notifier_filter import WakeNotifierFilter from pipecat.processors.user_idle_processor import UserIdleProcessor @@ -50,8 +51,8 @@ class TurnDetectionLLM(Pipeline): }, ] - statement_context = OpenAILLMContext(statement_messages) - statement_context_aggregator = statement_llm.create_context_aggregator(statement_context) + statement_context = LLMContext(statement_messages) + statement_context_aggregator = LLMContextAggregatorPair(statement_context) # We have instructed the LLM to return 'YES' if it thinks the user # completed a sentence. So, if it's 'YES' we will return true in this @@ -72,9 +73,7 @@ class TurnDetectionLLM(Pipeline): # This processor keeps the last context and will let it through once the # notifier is woken up. We start with the gate open because we send an # initial context frame to start the conversation. - gated_context_aggregator = GatedOpenAILLMContextAggregator( - notifier=notifier, start_open=True - ) + gated_context_aggregator = GatedLLMContextAggregator(notifier=notifier, start_open=True) # Notify if the user hasn't said anything. async def user_idle_notifier(frame): @@ -147,8 +146,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm_main.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) # LLM + turn detection (with an extra LLM as a judge) llm = TurnDetectionLLM(llm_main, context_aggregator) diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_llm_context.py similarity index 81% rename from src/pipecat/processors/aggregators/gated_openai_llm_context.py rename to src/pipecat/processors/aggregators/gated_llm_context.py index 56423403d..6535bffbb 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_llm_context.py @@ -4,20 +4,20 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Gated OpenAI LLM context aggregator for controlled message flow.""" +"""Gated LLM context aggregator for controlled message flow.""" -from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame +from pipecat.frames.frames import CancelFrame, EndFrame, Frame, LLMContextFrame, StartFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.sync.base_notifier import BaseNotifier -class GatedOpenAILLMContextAggregator(FrameProcessor): - """Aggregator that gates OpenAI LLM context frames until notified. +class GatedLLMContextAggregator(FrameProcessor): + """Aggregator that gates LLM context frames until notified. - This aggregator captures OpenAI LLM context frames and holds them until - a notifier signals that they can be released. This is useful for controlling - the flow of context frames based on external conditions or timing. + This aggregator captures LLM context frames and holds them until a notifier + signals that they can be released. This is useful for controlling the flow + of context frames based on external conditions or timing. """ def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs): @@ -35,7 +35,7 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): self._gate_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames, gating OpenAI LLM context frames. + """Process incoming frames, gating LLM context frames. Args: frame: The frame to process. @@ -49,7 +49,7 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): if isinstance(frame, (EndFrame, CancelFrame)): await self._stop() await self.push_frame(frame) - elif isinstance(frame, OpenAILLMContextFrame): + elif isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): if self._start_open: self._start_open = False await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/aggregators/gated_open_ai_llm_context.py b/src/pipecat/processors/aggregators/gated_open_ai_llm_context.py new file mode 100644 index 000000000..42e38928f --- /dev/null +++ b/src/pipecat/processors/aggregators/gated_open_ai_llm_context.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Gated OpenAI LLM context aggregator for controlled message flow.""" + +from pipecat.processors.aggregators.gated_llm_context import GatedLLMContextAggregator + +# Alias for backward compatibility with the previous name +GatedOpenAILLMContextAggregator = GatedLLMContextAggregator From 97868175e65f39a61b78a84326de739b0af33d38 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 23 Sep 2025 10:00:48 -0400 Subject: [PATCH 21/35] Add support for universal `LLMContext` to `LangchainProcessor` --- .../07b-interruptible-langchain.py | 18 ++++++------------ src/pipecat/processors/frameworks/langchain.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 3ef633846..32b637cf8 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -23,13 +23,8 @@ from pipecat.frames.frames import LLMMessagesUpdateFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantContextAggregator, - LLMUserContextAggregator, -) -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.langchain import LangchainProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -106,19 +101,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) lc = LangchainProcessor(history_chain) - context = OpenAILLMContext() - tma_in = LLMUserContextAggregator(context=context) - tma_out = LLMAssistantContextAggregator(context=context) + context = LLMContext() + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ transport.input(), # Transport user input stt, - tma_in, # User responses + context_aggregator.user(), # User responses lc, # Langchain tts, # TTS transport.output(), # Transport bot output - tma_out, # Assistant spoken responses + context_aggregator.assistant(), # Assistant spoken responses ] ) diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index b67e25105..97a6ce343 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -12,6 +12,7 @@ from loguru import logger from pipecat.frames.frames import ( Frame, + LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, TextFrame, @@ -64,11 +65,16 @@ class LangchainProcessor(FrameProcessor): """ await super().process_frame(frame, direction) - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): # Messages are accumulated on the context as a list of messages. # The last one by the human is the one we want to send to the LLM. logger.debug(f"Got transcription frame {frame}") - text: str = frame.context.messages[-1]["content"] + messages = ( + frame.context.messages + if isinstance(frame, OpenAILLMContextFrame) + else frame.context.get_messages() + ) + text: str = messages[-1]["content"] await self._ainvoke(text.strip()) else: From cbce2075ebcc528198d472be3cab252b9d8338b3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 23 Sep 2025 09:32:28 -0400 Subject: [PATCH 22/35] Add ElevenLabsSTTService --- CHANGELOG.md | 6 +- README.md | 2 +- .../07d-interruptible-elevenlabs-http.py | 6 +- .../13k-elevenlabs-transcription.py | 89 +++++ src/pipecat/services/elevenlabs/stt.py | 339 ++++++++++++++++++ src/pipecat/transcriptions/language.py | 42 ++- 6 files changed, 479 insertions(+), 5 deletions(-) create mode 100644 examples/foundational/13k-elevenlabs-transcription.py create mode 100644 src/pipecat/services/elevenlabs/stt.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c413d16fd..5e4729470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added a peer connection monitor to the `SmallWebRTCConnection` that - automatically disconnects if the connection fails to establish within +- Added `ElevenLabsSTTService` for speech-to-text transcription. + +- Added a peer connection monitor to the `SmallWebRTCConnection` that + automatically disconnects if the connection fails to establish within the timeout (1 minute by default). - Added memory cleanup improvements to reduce memory peaks. diff --git a/README.md b/README.md index 247a1508f..0a76ff5ea 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ You can connect to Pipecat from any platform using our official SDKs: | Category | Services | | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index a6b0f1fd7..cd922ddc2 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -24,6 +24,7 @@ from pipecat.processors.aggregators.llm_response_universal import LLMContextAggr from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.elevenlabs.stt import ElevenLabsSTTService from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -63,7 +64,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = ElevenLabsSTTService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + aiohttp_session=session, + ) tts = ElevenLabsHttpTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), diff --git a/examples/foundational/13k-elevenlabs-transcription.py b/examples/foundational/13k-elevenlabs-transcription.py new file mode 100644 index 000000000..bfac16c3f --- /dev/null +++ b/examples/foundational/13k-elevenlabs-transcription.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import Frame, TranscriptionFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.elevenlabs.stt import ElevenLabsSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +class TranscriptionLogger(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + print(f"Transcription: {frame.text}") + + # Push all frames through + await self.push_frame(frame, direction) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_in_enabled=True, vad_analyzer=SileroVADAnalyzer()), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, vad_analyzer=SileroVADAnalyzer() + ), + "webrtc": lambda: TransportParams(audio_in_enabled=True, vad_analyzer=SileroVADAnalyzer()), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = ElevenLabsSTTService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + aiohttp_session=session, + ) + + tl = TranscriptionLogger() + + pipeline = Pipeline([transport.input(), stt, tl]) + + task = PipelineTask( + pipeline, + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py new file mode 100644 index 000000000..291bad414 --- /dev/null +++ b/src/pipecat/services/elevenlabs/stt.py @@ -0,0 +1,339 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""ElevenLabs speech-to-text service implementation. + +This module provides integration with ElevenLabs' Speech-to-Text API for transcription +using segmented audio processing. The service uploads audio files and receives +transcription results directly. +""" + +import io +from typing import AsyncGenerator, Optional + +import aiohttp +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.stt_service import SegmentedSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + + +def language_to_elevenlabs_language(language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs language code. + + Source: + https://elevenlabs.io/docs/capabilities/speech-to-text + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding ElevenLabs language code, or None if not supported. + """ + BASE_LANGUAGES = { + Language.AF: "afr", # Afrikaans + Language.AM: "amh", # Amharic + Language.AR: "ara", # Arabic + Language.HY: "hye", # Armenian + Language.AS: "asm", # Assamese + Language.AST: "ast", # Asturian + Language.AZ: "aze", # Azerbaijani + Language.BE: "bel", # Belarusian + Language.BN: "ben", # Bengali + Language.BS: "bos", # Bosnian + Language.BG: "bul", # Bulgarian + Language.MY: "mya", # Burmese + Language.YUE: "yue", # Cantonese + Language.CA: "cat", # Catalan + Language.CEB: "ceb", # Cebuano + Language.NY: "nya", # Chichewa + Language.HR: "hrv", # Croatian + Language.CS: "ces", # Czech + Language.DA: "dan", # Danish + Language.NL: "nld", # Dutch + Language.EN: "eng", # English + Language.ET: "est", # Estonian + Language.FIL: "fil", # Filipino + Language.FI: "fin", # Finnish + Language.FR: "fra", # French + Language.FF: "ful", # Fulah + Language.GL: "glg", # Galician + Language.LG: "lug", # Ganda + Language.KA: "kat", # Georgian + Language.DE: "deu", # German + Language.EL: "ell", # Greek + Language.GU: "guj", # Gujarati + Language.HA: "hau", # Hausa + Language.HE: "heb", # Hebrew + Language.HI: "hin", # Hindi + Language.HU: "hun", # Hungarian + Language.IS: "isl", # Icelandic + Language.IG: "ibo", # Igbo + Language.ID: "ind", # Indonesian + Language.GA: "gle", # Irish + Language.IT: "ita", # Italian + Language.JA: "jpn", # Japanese + Language.JV: "jav", # Javanese + Language.KEA: "kea", # Kabuverdianu + Language.KN: "kan", # Kannada + Language.KK: "kaz", # Kazakh + Language.KM: "khm", # Khmer + Language.KO: "kor", # Korean + Language.KU: "kur", # Kurdish + Language.KY: "kir", # Kyrgyz + Language.LO: "lao", # Lao + Language.LV: "lav", # Latvian + Language.LN: "lin", # Lingala + Language.LT: "lit", # Lithuanian + Language.LUO: "luo", # Luo + Language.LB: "ltz", # Luxembourgish + Language.MK: "mkd", # Macedonian + Language.MS: "msa", # Malay + Language.ML: "mal", # Malayalam + Language.MT: "mlt", # Maltese + Language.ZH: "zho", # Mandarin Chinese + Language.MI: "mri", # Māori + Language.MR: "mar", # Marathi + Language.MN: "mon", # Mongolian + Language.NE: "nep", # Nepali + Language.NSO: "nso", # Northern Sotho + Language.NO: "nor", # Norwegian + Language.OC: "oci", # Occitan + Language.OR: "ori", # Odia + Language.PS: "pus", # Pashto + Language.FA: "fas", # Persian + Language.PL: "pol", # Polish + Language.PT: "por", # Portuguese + Language.PA: "pan", # Punjabi + Language.RO: "ron", # Romanian + Language.RU: "rus", # Russian + Language.SR: "srp", # Serbian + Language.SN: "sna", # Shona + Language.SD: "snd", # Sindhi + Language.SK: "slk", # Slovak + Language.SL: "slv", # Slovenian + Language.SO: "som", # Somali + Language.ES: "spa", # Spanish + Language.SW: "swa", # Swahili + Language.SV: "swe", # Swedish + Language.TA: "tam", # Tamil + Language.TG: "tgk", # Tajik + Language.TE: "tel", # Telugu + Language.TH: "tha", # Thai + Language.TR: "tur", # Turkish + Language.UK: "ukr", # Ukrainian + Language.UMB: "umb", # Umbundu + Language.UR: "urd", # Urdu + Language.UZ: "uzb", # Uzbek + Language.VI: "vie", # Vietnamese + Language.CY: "cym", # Welsh + Language.WO: "wol", # Wolof + Language.XH: "xho", # Xhosa + Language.ZU: "zul", # Zulu + } + + result = BASE_LANGUAGES.get(language) + + # If not found in base languages, try to find the base language from a variant + if not result: + lang_str = str(language.value) + base_code = lang_str.split("-")[0].lower() + result = base_code if base_code in BASE_LANGUAGES.values() else None + + return result + + +class ElevenLabsSTTService(SegmentedSTTService): + """Speech-to-text service using ElevenLabs' file-based API. + + This service uses ElevenLabs' Speech-to-Text API to perform transcription on audio + segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. + The service uploads audio files to ElevenLabs and receives transcription results directly. + """ + + class InputParams(BaseModel): + """Configuration parameters for ElevenLabs STT API. + + Parameters: + language: Target language for transcription. + tag_audio_events: Whether to include audio events like (laughter), (coughing), in the transcription. + """ + + language: Optional[Language] = None + tag_audio_events: bool = True + + def __init__( + self, + *, + api_key: str, + aiohttp_session: aiohttp.ClientSession, + base_url: str = "https://api.elevenlabs.io", + model: str = "scribe_v1", + sample_rate: Optional[int] = None, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the ElevenLabs STT service. + + Args: + api_key: ElevenLabs API key for authentication. + aiohttp_session: aiohttp ClientSession for HTTP requests. + base_url: Base URL for ElevenLabs API. + model: Model ID for transcription. Defaults to "scribe_v1". + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. + params: Configuration parameters for the STT service. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ + super().__init__( + sample_rate=sample_rate, + **kwargs, + ) + + params = params or ElevenLabsSTTService.InputParams() + + self._api_key = api_key + self._base_url = base_url + self._session = aiohttp_session + self._model_id = model + self._tag_audio_events = params.tag_audio_events + + self._settings = { + "language": self.language_to_service_language(params.language) + if params.language + else "eng", + } + + def can_generate_metrics(self) -> bool: + """Check if the service can generate processing metrics. + + Returns: + True, as ElevenLabs STT service supports metrics generation. + """ + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs service-specific language code. + + Args: + language: The language to convert. + + Returns: + The ElevenLabs-specific language code, or None if not supported. + """ + return language_to_elevenlabs_language(language) + + async def set_language(self, language: Language): + """Set the transcription language. + + Args: + language: The language to use for speech-to-text transcription. + """ + logger.info(f"Switching STT language to: [{language}]") + self._settings["language"] = self.language_to_service_language(language) + + async def set_model(self, model: str): + """Set the STT model. + + Args: + model: The model name to use for transcription. + + Note: + ElevenLabs STT API does not currently support model selection. + This method is provided for interface compatibility. + """ + await super().set_model(model) + logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model") + + async def _transcribe_audio(self, audio_data: bytes) -> dict: + """Upload audio data to ElevenLabs and get transcription result. + + Args: + audio_data: Raw audio bytes in WAV format. + + Returns: + The transcription result data. + + Raises: + Exception: If transcription fails or returns an error. + """ + url = f"{self._base_url}/v1/speech-to-text" + headers = {"xi-api-key": self._api_key} + + # Create form data with the audio file + data = aiohttp.FormData() + data.add_field( + "file", + io.BytesIO(audio_data), + filename="audio.wav", + content_type="audio/x-wav", + ) + + # Add required model_id, language_code, and tag_audio_events + data.add_field("model_id", self._model_id) + data.add_field("language_code", self._settings["language"]) + data.add_field("tag_audio_events", str(self._tag_audio_events).lower()) + + async with self._session.post(url, data=data, headers=headers) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"ElevenLabs transcription error: {error_text}") + raise Exception(f"Transcription failed with status {response.status}: {error_text}") + + result = await response.json() + return result + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[str] = None + ): + """Handle a transcription result with tracing.""" + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe an audio segment using ElevenLabs' STT API. + + Args: + audio: Raw audio bytes in WAV format (already converted by base class). + + Yields: + Frame: TranscriptionFrame containing the transcribed text, or ErrorFrame on failure. + + Note: + The audio is already in WAV format from the SegmentedSTTService. + Only non-empty transcriptions are yielded. + """ + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + # Upload audio and get transcription result directly + result = await self._transcribe_audio(audio) + + # Extract transcription text + text = result.get("text", "").strip() + if text: + # Use the language_code returned by the API + detected_language = result.get("language_code", "eng") + + await self._handle_transcription(text, True, detected_language) + logger.debug(f"Transcription: [{text}]") + + yield TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + detected_language, + result=result, + ) + + except Exception as e: + logger.error(f"ElevenLabs STT error: {e}") + yield ErrorFrame(f"ElevenLabs STT error: {str(e)}") diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index 182a89321..5ee37bf68 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -68,6 +68,9 @@ class Language(StrEnum): AS = "as" AS_IN = "as-IN" + # Asturian + AST = "ast" + # Azerbaijani AZ = "az" AZ_AZ = "az-AZ" @@ -101,6 +104,9 @@ class Language(StrEnum): CA = "ca" CA_ES = "ca-ES" + # Cebuano + CEB = "ceb" + # Mandarin Chinese CMN = "cmn" CMN_CN = "cmn-CN" @@ -185,6 +191,9 @@ class Language(StrEnum): FA = "fa" FA_IR = "fa-IR" + # Fulah + FF = "ff" + # Finnish FI = "fi" FI_FI = "fi-FI" @@ -251,6 +260,9 @@ class Language(StrEnum): ID = "id" ID_ID = "id-ID" + # Igbo + IG = "ig" + # Icelandic IS = "is" IS_IS = "is-IS" @@ -279,6 +291,9 @@ class Language(StrEnum): KA = "ka" KA_GE = "ka-GE" + # Kabuverdianu + KEA = "kea" + # Kazakh KK = "kk" KK_KZ = "kk-KZ" @@ -295,6 +310,13 @@ class Language(StrEnum): KO = "ko" KO_KR = "ko-KR" + # Kurdish + KU = "ku" + + # Kyrgyz + KY = "ky" + KY_KG = "ky-KG" + # Latin LA = "la" @@ -312,6 +334,12 @@ class Language(StrEnum): LT = "lt" LT_LT = "lt-LT" + # Ganda + LG = "lg" + + # Luo + LUO = "luo" + # Latvian LV = "lv" LV_LV = "lv-LV" @@ -366,6 +394,12 @@ class Language(StrEnum): NL_BE = "nl-BE" NL_NL = "nl-NL" + # Northern Sotho + NSO = "nso" + + # Chichewa + NY = "ny" + # Occitan OC = "oc" @@ -484,6 +518,9 @@ class Language(StrEnum): UK = "uk" UK_UA = "uk-UA" + # Umbundu + UMB = "umb" + # Urdu UR = "ur" UR_IN = "ur-IN" @@ -497,6 +534,9 @@ class Language(StrEnum): VI = "vi" VI_VN = "vi-VN" + # Wolof + WO = "wo" + # Wu Chinese WUU = "wuu" WUU_CN = "wuu-CN" @@ -507,7 +547,7 @@ class Language(StrEnum): # Yoruba YO = "yo" - # Yue Chinese + # Yue Chinese (Cantonese) YUE = "yue" YUE_CN = "yue-CN" From e21ab89509e7a149020c21fd71cccf32919e1b4d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 23 Sep 2025 10:35:54 -0400 Subject: [PATCH 23/35] Add support for universal `LLMContext` to `Mem0MemoryService` --- examples/foundational/37-mem0.py | 7 ++++--- src/pipecat/services/mem0/memory.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/foundational/37-mem0.py b/examples/foundational/37-mem0.py index 57be73df4..436f0a452 100644 --- a/examples/foundational/37-mem0.py +++ b/examples/foundational/37-mem0.py @@ -55,7 +55,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -244,8 +245,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set up conversation context and management # The context_aggregator will automatically collect conversation context - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) pipeline = Pipeline( diff --git a/src/pipecat/services/mem0/memory.py b/src/pipecat/services/mem0/memory.py index b134f5162..a80398728 100644 --- a/src/pipecat/services/mem0/memory.py +++ b/src/pipecat/services/mem0/memory.py @@ -16,7 +16,8 @@ from typing import Any, Dict, List, Optional from loguru import logger from pydantic import BaseModel, Field -from pipecat.frames.frames import ErrorFrame, Frame, LLMMessagesFrame +from pipecat.frames.frames import ErrorFrame, Frame, LLMContextFrame, LLMMessagesFrame +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, @@ -180,11 +181,11 @@ class Mem0MemoryService(FrameProcessor): logger.error(f"Error retrieving memories from Mem0: {e}") return [] - def _enhance_context_with_memories(self, context: OpenAILLMContext, query: str): + def _enhance_context_with_memories(self, context: LLMContext | OpenAILLMContext, query: str): """Enhance the LLM context with relevant memories. Args: - context: The OpenAILLMContext to enhance with memory information. + context: The LLM context to enhance with memory information. query: The query to search for relevant memories. """ # Skip if this is the same query we just processed @@ -222,11 +223,11 @@ class Mem0MemoryService(FrameProcessor): context = None messages = None - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): context = frame.context elif isinstance(frame, LLMMessagesFrame): messages = frame.messages - context = OpenAILLMContext.from_messages(messages) + context = LLMContext(messages) if context: try: From 98d3686861d2399aba194d7d6f13cd6bde0005ce Mon Sep 17 00:00:00 2001 From: lshaun <74459820+lshaun@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:58:09 +0000 Subject: [PATCH 24/35] update imports to avoid deprecated module --- examples/foundational/41a-text-only-webrtc.py | 3 +-- examples/foundational/41b-text-and-audio-webrtc.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/foundational/41a-text-only-webrtc.py b/examples/foundational/41a-text-only-webrtc.py index 3fd73f89a..f8e5414b0 100644 --- a/examples/foundational/41a-text-only-webrtc.py +++ b/examples/foundational/41a-text-only-webrtc.py @@ -29,8 +29,7 @@ from pipecat.processors.frameworks.rtvi import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.openai import OpenAIContextAggregatorPair -from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.llm import OpenAIContextAggregatorPair, OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams load_dotenv(override=True) diff --git a/examples/foundational/41b-text-and-audio-webrtc.py b/examples/foundational/41b-text-and-audio-webrtc.py index f2b2da6b1..af604fc84 100644 --- a/examples/foundational/41b-text-and-audio-webrtc.py +++ b/examples/foundational/41b-text-and-audio-webrtc.py @@ -32,8 +32,7 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.openai import OpenAIContextAggregatorPair -from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.llm import OpenAIContextAggregatorPair, OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams load_dotenv(override=True) From d4b1e1ab41941f10ec0cfc4d624bf7d7e5fe4d26 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 23 Sep 2025 12:14:10 -0400 Subject: [PATCH 25/35] Update more examples to use universal `LLMContext`. Specifically, update examples we didn't update before because they weren't using `ToolsSchema` for their tool definitions, which is a requirement for using `LLMContext`. NOTE: oops! Turns out some of these files had *already* been updated to use universal `LLMContext` even though they weren't yet using `ToolsSchema`. This commit should fix those examples. --- examples/foundational/15-switch-voices.py | 33 ++++++------- examples/foundational/15a-switch-languages.py | 32 +++++-------- examples/foundational/33-gemini-rag.py | 48 ++++++++----------- .../foundational/36-user-email-gathering.py | 43 ++++++++--------- 4 files changed, 67 insertions(+), 89 deletions(-) diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index bd544462e..afa51289c 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -9,8 +9,9 @@ import os from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -121,25 +122,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) llm.register_function("switch_voice", tts.switch_voice) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "switch_voice", - "description": "Switch your voice only when the user asks you to", - "parameters": { - "type": "object", - "properties": { - "voice": { - "type": "string", - "description": "The voice the user wants you to use", - }, - }, - "required": ["voice"], - }, + switch_voice_function = FunctionSchema( + name="switch_voice", + description="Switch your voice only when the user asks you to", + properties={ + "voice": { + "type": "string", + "description": "The voice the user wants you to use", }, - ) - ] + }, + required=["voice"], + ) + tools = ToolsSchema(standard_tools=[switch_voice_function]) + messages = [ { "role": "system", diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index f5e92af4f..7e70b0f2d 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -10,8 +10,9 @@ import os from deepgram import LiveOptions from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -112,25 +113,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) llm.register_function("switch_language", tts.switch_language) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "switch_language", - "description": "Switch to another language when the user asks you to", - "parameters": { - "type": "object", - "properties": { - "language": { - "type": "string", - "description": "The language the user wants you to speak", - }, - }, - "required": ["language"], - }, + switch_language_function = FunctionSchema( + name="switch_language", + description="Switch to another language when the user asks you to", + properties={ + "language": { + "type": "string", + "description": "The language the user wants you to speak", }, - ) - ] + }, + required=["language"], + ) + tools = ToolsSchema(standard_tools=[switch_language_function]) messages = [ { "role": "system", diff --git a/examples/foundational/33-gemini-rag.py b/examples/foundational/33-gemini-rag.py index b9293ae23..e2e88e390 100644 --- a/examples/foundational/33-gemini-rag.py +++ b/examples/foundational/33-gemini-rag.py @@ -55,6 +55,8 @@ from dotenv import load_dotenv from google import genai from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -63,7 +65,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -121,11 +124,7 @@ async def query_knowledge_base(params: FunctionCallParams): # for our case, the first two messages are the instructions and the user message # so we remove them. - conversation_turns = params.context.messages[2:] - # convert to standard messages - messages = [] - for turn in conversation_turns: - messages.extend(params.context.to_standard_messages(turn)) + conversation_turns = params.context.get_messages()[2:] def _is_tool_call(turn): if turn.get("role", None) == "tool": @@ -135,7 +134,7 @@ async def query_knowledge_base(params: FunctionCallParams): return False # filter out tool calls - messages = [turn for turn in messages if not _is_tool_call(turn)] + messages = [turn for turn in conversation_turns if not _is_tool_call(turn)] # use the last 3 turns as the conversation history/context messages = messages[-3:] messages_json = json.dumps(messages, ensure_ascii=False, indent=2) @@ -199,25 +198,20 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): api_key=os.getenv("GOOGLE_API_KEY"), ) llm.register_function("query_knowledge_base", query_knowledge_base) - tools = [ - { - "function_declarations": [ - { - "name": "query_knowledge_base", - "description": "Query the knowledge base for the answer to the question.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question to query the knowledge base with.", - }, - }, - }, - }, - ], + + query_function = FunctionSchema( + name="query_knowledge_base", + description="Query the knowledge base for the answer to the question.", + properties={ + "question": { + "type": "string", + "description": "The question to query the knowledge base with.", + }, }, - ] + required=["question"], + ) + tools = ToolsSchema(standard_tools=[query_function]) + system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. @@ -230,8 +224,8 @@ Your response will be turned into speech so use only simple words and punctuatio {"role": "user", "content": "Greet the user."}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py index cc0c9f29f..4cf20b750 100644 --- a/examples/foundational/36-user-email-gathering.py +++ b/examples/foundational/36-user-email-gathering.py @@ -9,8 +9,9 @@ import os from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,14 +20,14 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.services.rime.tts import RimeHttpTTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -90,26 +91,20 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # sent to the same callback with an additional function_name parameter. llm.register_function("store_user_emails", store_user_emails) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "store_user_emails", - "description": "Store user emails when confirmed", - "parameters": { - "type": "object", - "properties": { - "emails": { - "type": "array", - "description": "The list of user emails", - "items": {"type": "string"}, - }, - }, - "required": ["emails"], - }, + store_emails_function = FunctionSchema( + name="store_user_emails", + description="Store user emails when confirmed", + properties={ + "emails": { + "type": "array", + "description": "The list of user emails", + "items": {"type": "string"}, }, - ) - ] + }, + required=["emails"], + ) + tools = ToolsSchema(standard_tools=[store_emails_function]) + messages = [ { "role": "system", @@ -120,8 +115,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ From ed64716219001d7ea82997ccbb10c8a8af8240ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 23 Sep 2025 10:38:31 -0700 Subject: [PATCH 26/35] StrandsAgentsProcessor: fix formatting --- src/pipecat/processors/frameworks/strands_agents.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py index 05aef0f17..67005d74b 100644 --- a/src/pipecat/processors/frameworks/strands_agents.py +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -1,5 +1,4 @@ -""" -Strands Agent integration for Pipecat. +"""Strands Agent integration for Pipecat. This module provides integration with Strands Agents for handling conversational AI interactions. It supports both single agent and multi-agent graphs. From f6b4db42efd70ae3529cae95583c62938a01d389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 23 Sep 2025 10:39:06 -0700 Subject: [PATCH 27/35] pyproject: add strands-agents --- pyproject.toml | 7 +++-- uv.lock | 83 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c26b7744c..d0e727ba1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-ope livekit = [ "livekit~=1.0.13", "livekit-api~=1.0.5", "tenacity>=8.2.3,<10.0.0" ] lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] -mcp = [ "mcp[cli]~=1.9.4" ] +mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] mistral = [] mlx-whisper = [ "mlx-whisper~=0.4.2" ] @@ -95,13 +95,14 @@ sambanova = [] sarvam = [ "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk~=2.23.1" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] -local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1, <2" ] +local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] remote-smart-turn = [] -silero = [ "onnxruntime>=1.20.1, <2" ] +silero = [ "onnxruntime>=1.20.1,<2" ] simli = [ "simli-ai~=0.1.10"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.0" ] speechmatics = [ "speechmatics-rt>=0.4.0" ] +strands = [ "strands-agents>=1.9.1,<2" ] tavus=[] together = [] tracing = [ "opentelemetry-sdk>=1.33.0", "opentelemetry-api>=1.33.0", "opentelemetry-instrumentation>=0.54b0" ] diff --git a/uv.lock b/uv.lock index 63381d8a7..4767acbd4 100644 --- a/uv.lock +++ b/uv.lock @@ -3048,22 +3048,24 @@ wheels = [ [[package]] name = "mcp" -version = "1.9.4" +version = "1.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "httpx" }, { name = "httpx-sse" }, + { name = "jsonschema" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, { name = "sse-starlette" }, { name = "starlette" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/f2/dc2450e566eeccf92d89a00c3e813234ad58e2ba1e31d11467a09ac4f3b9/mcp-1.9.4.tar.gz", hash = "sha256:cfb0bcd1a9535b42edaef89947b9e18a8feb49362e1cc059d6e7fc636f2cb09f", size = 333294, upload-time = "2025-06-12T08:20:30.158Z" } +sdist = { url = "https://files.pythonhosted.org/packages/48/e9/242096400d702924b49f8d202c6ded7efb8841cacba826b5d2e6183aef7b/mcp-1.14.1.tar.gz", hash = "sha256:31c4406182ba15e8f30a513042719c3f0a38c615e76188ee5a736aaa89e20134", size = 454944, upload-time = "2025-09-18T13:37:19.971Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/97/fc/80e655c955137393c443842ffcc4feccab5b12fa7cb8de9ced90f90e6998/mcp-1.9.4-py3-none-any.whl", hash = "sha256:7fcf36b62936adb8e63f89346bccca1268eeca9bf6dfb562ee10b1dfbda9dac0", size = 130232, upload-time = "2025-06-12T08:20:28.551Z" }, + { url = "https://files.pythonhosted.org/packages/8e/11/d334fbb7c2aeddd2e762b86d7a619acffae012643a5738e698f975a2a9e2/mcp-1.14.1-py3-none-any.whl", hash = "sha256:3b7a479e8e5cbf5361bdc1da8bc6d500d795dc3aff44b44077a363a7f7e945a4", size = 163809, upload-time = "2025-09-18T13:37:18.165Z" }, ] [package.optional-dependencies] @@ -3890,6 +3892,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/db/5ff1cd6c5ca1d12ecf1b73be16fbb2a8af2114ee46d4b0e6d4b23f4f4db7/opentelemetry_instrumentation-0.58b0-py3-none-any.whl", hash = "sha256:50f97ac03100676c9f7fc28197f8240c7290ca1baa12da8bfbb9a1de4f34cc45", size = 33019, upload-time = "2025-09-11T11:41:00.624Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-threading" +version = "0.58b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/a9/3888cb0470e6eb48ea17b6802275ae71df411edd6382b9a8e8f391936fda/opentelemetry_instrumentation_threading-0.58b0.tar.gz", hash = "sha256:f68c61f77841f9ff6270176f4d496c10addbceacd782af434d705f83e4504862", size = 8770, upload-time = "2025-09-11T11:42:56.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/54/add1076cb37980e617723a96e29c84006983e8ad6fc589dde7f69ddc57d4/opentelemetry_instrumentation_threading-0.58b0-py3-none-any.whl", hash = "sha256:eacc072881006aceb5b9b6831bcdce718c67ef6f31ac0b32bd6a23a94d979b4a", size = 9312, upload-time = "2025-09-11T11:41:58.603Z" }, +] + [[package]] name = "opentelemetry-sdk" version = "1.37.0" @@ -4408,6 +4424,9 @@ soundfile = [ speechmatics = [ { name = "speechmatics-rt" }, ] +strands = [ + { name = "strands-agents" }, +] tracing = [ { name = "opentelemetry-api" }, { name = "opentelemetry-instrumentation" }, @@ -4492,7 +4511,7 @@ requires-dist = [ { name = "livekit-api", marker = "extra == 'livekit'", specifier = "~=1.0.5" }, { name = "loguru", specifier = "~=0.7.3" }, { name = "markdown", specifier = ">=3.7,<4" }, - { name = "mcp", extras = ["cli"], marker = "extra == 'mcp'", specifier = "~=1.9.4" }, + { name = "mcp", extras = ["cli"], marker = "extra == 'mcp'", specifier = ">=1.11.0,<2" }, { name = "mem0ai", marker = "extra == 'mem0'", specifier = "~=0.1.94" }, { name = "mlx-whisper", marker = "extra == 'mlx-whisper'", specifier = "~=0.4.2" }, { name = "nltk", specifier = ">=3.9.1,<4" }, @@ -4543,6 +4562,7 @@ requires-dist = [ { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.0" }, { name = "soxr", specifier = "~=0.5.0" }, { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.4.0" }, + { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, { name = "torch", marker = "extra == 'local-smart-turn'", specifier = ">=2.5.0,<3" }, @@ -4556,7 +4576,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "silero", "simli", "soniox", "soundfile", "speechmatics", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "deepseek", "daily", "deepgram", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "grok", "groq", "gstreamer", "heygen", "inworld", "krisp", "koala", "langchain", "livekit", "lmnt", "local", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nim", "neuphonic", "noisereduce", "openai", "openpipe", "openrouter", "perplexity", "playht", "qwen", "rime", "riva", "runner", "sambanova", "sarvam", "sentry", "local-smart-turn", "local-smart-turn-v3", "remote-smart-turn", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] [package.metadata.requires-dev] dev = [ @@ -6856,6 +6876,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/fd/901cfa59aaa5b30a99e16876f11abe38b59a1a2c51ffb3d7142bb6089069/starlette-0.47.3-py3-none-any.whl", hash = "sha256:89c0778ca62a76b826101e7c709e70680a1699ca7da6b44d38eb0a7e61fe4b51", size = 72991, upload-time = "2025-08-24T13:36:40.887Z" }, ] +[[package]] +name = "strands-agents" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "docstring-parser" }, + { name = "mcp" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation-threading" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic" }, + { name = "typing-extensions" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/80/de6a83822d2113cddf7d8f69ba4236042195089d4d18c6b8babfbdae83fc/strands_agents-1.9.1.tar.gz", hash = "sha256:224e2736c1562da28378ee928934c3b618558f0af6bb930e0befb7ecb9246701", size = 397329, upload-time = "2025-09-19T19:44:45.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/72/2ffc2a3972188137b6cdf82dca2e5da667040c61e3b0694583eb8527dc89/strands_agents-1.9.1-py3-none-any.whl", hash = "sha256:aee4944b3fda370b198a7bdaec97e55aa5d8475f7df4b52f2f0e33945d149a4a", size = 206723, upload-time = "2025-09-19T19:44:43.21Z" }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -7465,6 +7506,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/90/56/0f88040567af7ff376ec9eaabe18fd980a4f5089d3bf8c7a32598ef06b8d/wait_for2-0.4.1-py3-none-any.whl", hash = "sha256:c694503e8c7420929e8a86bcffd9b00d55acaec2c14223a2b1e92bdc2ebf2154", size = 10985, upload-time = "2025-06-13T19:44:58.82Z" }, ] +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/56/90994d789c61df619bfc5ce2ecdabd5eeff564e1eb47512bd01b5e019569/watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26", size = 96390, upload-time = "2024-11-01T14:06:24.793Z" }, + { url = "https://files.pythonhosted.org/packages/55/46/9a67ee697342ddf3c6daa97e3a587a56d6c4052f881ed926a849fcf7371c/watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112", size = 88389, upload-time = "2024-11-01T14:06:27.112Z" }, + { url = "https://files.pythonhosted.org/packages/44/65/91b0985747c52064d8701e1075eb96f8c40a79df889e59a399453adfb882/watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3", size = 89020, upload-time = "2024-11-01T14:06:29.876Z" }, + { url = "https://files.pythonhosted.org/packages/e0/24/d9be5cd6642a6aa68352ded4b4b10fb0d7889cb7f45814fb92cecd35f101/watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", size = 96393, upload-time = "2024-11-01T14:06:31.756Z" }, + { url = "https://files.pythonhosted.org/packages/63/7a/6013b0d8dbc56adca7fdd4f0beed381c59f6752341b12fa0886fa7afc78b/watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2", size = 88392, upload-time = "2024-11-01T14:06:32.99Z" }, + { url = "https://files.pythonhosted.org/packages/d1/40/b75381494851556de56281e053700e46bff5b37bf4c7267e858640af5a7f/watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", size = 89019, upload-time = "2024-11-01T14:06:34.963Z" }, + { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, + { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, + { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, + { url = "https://files.pythonhosted.org/packages/30/ad/d17b5d42e28a8b91f8ed01cb949da092827afb9995d4559fd448d0472763/watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881", size = 87902, upload-time = "2024-11-01T14:06:53.119Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ca/c3649991d140ff6ab67bfc85ab42b165ead119c9e12211e08089d763ece5/watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11", size = 88380, upload-time = "2024-11-01T14:06:55.19Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + [[package]] name = "watchfiles" version = "1.1.0" From 781366627c9b703c0bf3e8bf4e36ea6bac79b7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 23 Sep 2025 10:41:22 -0700 Subject: [PATCH 28/35] updated CHANGELOG with Strands Agents --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 755687856..1b3db3e24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `StrandsAgentProcessor` which allows you to use the Strands Agents + framework to build your voice agents. + See https://strandsagents.com + - Added `ElevenLabsSTTService` for speech-to-text transcription. - Added a peer connection monitor to the `SmallWebRTCConnection` that From c3a2fa100c6d415c274922901548d336943649dd Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 23 Sep 2025 12:46:36 -0400 Subject: [PATCH 29/35] Update CHANGELOG with additional `LLMContext` support --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3db3e24..bdbb634f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for using universal `LLMContext` with: + + - `LLMLogObserver` + - `GatedLLMContextAggregator` (formerly `GatedOpenAILLMContextAggregator`) + - `LangchainProcessor` + - `Mem0MemoryService` + - Added `StrandsAgentProcessor` which allows you to use the Strands Agents framework to build your voice agents. See https://strandsagents.com From 3fc5214c15b2818db67aee37357f77ccb3280aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 22 Sep 2025 17:28:40 -0700 Subject: [PATCH 30/35] BaseOutputTransport: only push downstream if transport write successful Fixes #2589 --- CHANGELOG.md | 7 ++++ .../foundational/04b-transports-livekit.py | 4 +- src/pipecat/transports/base_output.py | 36 +++++++++++++----- src/pipecat/transports/daily/transport.py | 36 +++++++++++++----- src/pipecat/transports/livekit/transport.py | 13 +++++-- src/pipecat/transports/local/audio.py | 7 +++- src/pipecat/transports/local/tk.py | 13 ++++++- .../transports/smallwebrtc/transport.py | 30 +++++++++++---- src/pipecat/transports/tavus/transport.py | 16 +++++--- src/pipecat/transports/websocket/client.py | 38 ++++++++++++++++++- src/pipecat/transports/websocket/fastapi.py | 9 ++++- src/pipecat/transports/websocket/server.py | 9 ++++- 12 files changed, 172 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdbb634f1..ad49ae5d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `BaseOutputTransport` methods `write_audio_frame` and `write_video_frame` now + return a boolean to indicate if the transport implementation was able to write + the given frame or not. + - Updated Silero VAD model to v6. - Updated `livekit` to 1.0.13. @@ -98,6 +102,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `BaseOutputTransport` issue that could produce large saved + `AudioBufferProcessor` files when using an audio mixer. + - Fixed a `PipelineRunner` issue on Windows where setting up SIGINT and SIGTERM was raising an exception. diff --git a/examples/foundational/04b-transports-livekit.py b/examples/foundational/04b-transports-livekit.py index 4e5a7e820..402fd79ca 100644 --- a/examples/foundational/04b-transports-livekit.py +++ b/examples/foundational/04b-transports-livekit.py @@ -18,8 +18,8 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( InterruptionFrame, - TextFrame, TranscriptionFrame, + TTSSpeakFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -103,7 +103,7 @@ async def main(): async def on_first_participant_joined(transport, participant_id): await asyncio.sleep(1) await task.queue_frame( - TextFrame( + TTSSpeakFrame( "Hello there! How are you doing today? Would you like to talk about the weather?" ) ) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index ed60a4584..9d1b2b3bb 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -202,21 +202,27 @@ class BaseOutputTransport(FrameProcessor): """ pass - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the transport. Args: frame: The output video frame to write. - """ - pass - async def write_audio_frame(self, frame: OutputAudioRawFrame): + Returns: + True if the video frame was written successfully, False otherwise. + """ + return False + + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the transport. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ - pass + return False async def write_dtmf(self, frame: OutputDTMFFrame | OutputDTMFUrgentFrame): """Write a DTMF tone using the transport's preferred method. @@ -740,12 +746,22 @@ class BaseOutputTransport(FrameProcessor): # Handle frame. await self._handle_frame(frame) - # Also, push frame downstream in case anyone else needs it. - await self._transport.push_frame(frame) + # If we are not able to write to the transport we shouldn't + # pushb downstream. + push_downstream = True - # Send audio. - if isinstance(frame, OutputAudioRawFrame): - await self._transport.write_audio_frame(frame) + # Try to send audio to the transport. + try: + if isinstance(frame, OutputAudioRawFrame): + push_downstream = await self._transport.write_audio_frame(frame) + except Exception as e: + logger.error(f"{self} Error writing {frame} to transport: {e}") + push_downstream = False + + # If we were able to send to the transport, push the frame + # downstream in case anyone else needs it. + if push_downstream: + await self._transport.push_frame(frame) # # Video handling diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 030f1ad6a..edb25531f 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -506,11 +506,14 @@ class DailyTransportClient(EventHandler): self._custom_audio_tracks[destination] = await self.add_custom_audio_track(destination) self._client.update_publishing({"customAudio": {destination: True}}) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the appropriate audio track. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ future = self._get_event_loop().create_future() @@ -526,18 +529,24 @@ class DailyTransportClient(EventHandler): audio_source.write_frames(frame.audio, completion=completion_callback(future)) else: logger.warning(f"{self} unable to write audio frames to destination [{destination}]") - future.set_result(None) + future.set_result(0) - await future + num_frames = await future + return num_frames > 0 - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the camera device. Args: frame: The image frame to write. + + Returns: + True if the video frame was written successfully, False otherwise. """ if not frame.transport_destination and self._camera: self._camera.write_frame(frame.image) + return True + return False async def setup(self, setup: FrameProcessorSetup): """Setup the client with task manager and event queues. @@ -1835,24 +1844,33 @@ class DailyOutputTransport(BaseOutputTransport): Args: destination: The destination identifier to register. + + Returns: + True if the audio frame was written successfully, False otherwise. """ await self._client.register_audio_destination(destination) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the Daily call. Args: frame: The audio frame to write. - """ - await self._client.write_audio_frame(frame) - async def write_video_frame(self, frame: OutputImageRawFrame): + Returns: + True if the audio frame was written successfully, False otherwise. + """ + return await self._client.write_audio_frame(frame) + + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the Daily call. Args: frame: The video frame to write. + + Returns: + True if the video frame was written successfully, False otherwise. """ - await self._client.write_video_frame(frame) + return await self._client.write_video_frame(frame) def _supports_native_dtmf(self) -> bool: """Daily supports native DTMF via telephone events. diff --git a/src/pipecat/transports/livekit/transport.py b/src/pipecat/transports/livekit/transport.py index 227a9f4f9..334d48ecb 100644 --- a/src/pipecat/transports/livekit/transport.py +++ b/src/pipecat/transports/livekit/transport.py @@ -329,19 +329,21 @@ class LiveKitTransportClient: except Exception as e: logger.error(f"Error sending DTMF tone {digit}: {e}") - async def publish_audio(self, audio_frame: rtc.AudioFrame): + async def publish_audio(self, audio_frame: rtc.AudioFrame) -> bool: """Publish an audio frame to the room. Args: audio_frame: The LiveKit audio frame to publish. """ if not self._connected or not self._audio_source: - return + return False try: await self._audio_source.capture_frame(audio_frame) + return True except Exception as e: logger.error(f"Error publishing audio: {e}") + return False def get_participants(self) -> List[str]: """Get list of participant IDs in the room. @@ -849,14 +851,17 @@ class LiveKitOutputTransport(BaseOutputTransport): else: await self._client.send_data(message.encode()) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the LiveKit room. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ livekit_audio = self._convert_pipecat_audio_to_livekit(frame.audio) - await self._client.publish_audio(livekit_audio) + return await self._client.publish_audio(livekit_audio) def _supports_native_dtmf(self) -> bool: """LiveKit supports native DTMF via telephone events. diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index b38d1dd8e..7f809b5d4 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -172,16 +172,21 @@ class LocalAudioOutputTransport(BaseOutputTransport): self._out_stream.close() self._out_stream = None - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the output stream. Args: frame: The audio frame to write to the output device. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._out_stream: await self.get_event_loop().run_in_executor( self._executor, self._out_stream.write, frame.audio ) + return True + return False class LocalAudioTransport(BaseTransport): diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index c687370ce..6cceb2cf1 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -191,24 +191,33 @@ class TkOutputTransport(BaseOutputTransport): self._out_stream.close() self._out_stream = None - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the output stream. Args: frame: The audio frame to write to the output device. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._out_stream: await self.get_event_loop().run_in_executor( self._executor, self._out_stream.write, frame.audio ) + return True + return False - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the Tkinter display. Args: frame: The video frame to display in the Tkinter window. + + Returns: + True if the video frame was written successfully, False otherwise. """ self.get_event_loop().call_soon(self._write_frame_to_tk, frame) + return True def _write_frame_to_tk(self, frame: OutputImageRawFrame): """Write frame data to the Tkinter image label.""" diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index 0ec8bf998..4a9ba9341 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -399,23 +399,33 @@ class SmallWebRTCClient: del frame # free original AudioFrame - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebRTC connection. Args: frame: The audio frame to transmit. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._can_send() and self._audio_output_track: await self._audio_output_track.add_audio_bytes(frame.audio) + return True + return False - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the WebRTC connection. Args: frame: The video frame to transmit. + + Returns: + True if the video frame was written successfully, False otherwise. """ if self._can_send() and self._video_output_track: self._video_output_track.add_video_frame(frame) + return True + return False async def setup(self, _params: TransportParams, frame): """Set up the client with transport parameters. @@ -818,21 +828,27 @@ class SmallWebRTCOutputTransport(BaseOutputTransport): """ await self._client.send_message(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebRTC connection. Args: frame: The output audio frame to transmit. - """ - await self._client.write_audio_frame(frame) - async def write_video_frame(self, frame: OutputImageRawFrame): + Returns: + True if the audio frame was written successfully, False otherwise. + """ + return await self._client.write_audio_frame(frame) + + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the WebRTC connection. Args: frame: The output video frame to transmit. + + Returns: + True if the video frame was written successfully, False otherwise. """ - await self._client.write_video_frame(frame) + return await self._client.write_video_frame(frame) class SmallWebRTCTransport(BaseTransport): diff --git a/src/pipecat/transports/tavus/transport.py b/src/pipecat/transports/tavus/transport.py index 23e14f97b..b199de09c 100644 --- a/src/pipecat/transports/tavus/transport.py +++ b/src/pipecat/transports/tavus/transport.py @@ -395,15 +395,18 @@ class TavusTransportClient: participant_settings=participant_settings, profile_settings=profile_settings ) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the transport. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if not self._client: - return - await self._client.write_audio_frame(frame) + return False + return await self._client.write_audio_frame(frame) async def register_audio_destination(self, destination: str): """Register an audio destination for output. @@ -625,15 +628,18 @@ class TavusOutputTransport(BaseOutputTransport): """Handle interruption events by sending interrupt message.""" await self._client.send_interrupt_message() - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the Tavus transport. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ # This is the custom track destination expected by Tavus frame.transport_destination = self._transport_destination - await self._client.write_audio_frame(frame) + return await self._client.write_audio_frame(frame) async def register_audio_destination(self, destination: str): """Register an audio destination. diff --git a/src/pipecat/transports/websocket/client.py b/src/pipecat/transports/websocket/client.py index d141b52f3..c18c0d22a 100644 --- a/src/pipecat/transports/websocket/client.py +++ b/src/pipecat/transports/websocket/client.py @@ -150,17 +150,39 @@ class WebsocketClientSession: await self._websocket.close() self._websocket = None - async def send(self, message: websockets.Data): + async def send(self, message: websockets.Data) -> bool: """Send a message through the WebSocket connection. Args: message: The message data to send. """ + result = False try: if self._websocket: await self._websocket.send(message) + result = True except Exception as e: logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})") + finally: + return result + + @property + def is_connected(self) -> bool: + """Check if the WebSocket is currently connected. + + Returns: + True if the WebSocket is in connected state. + """ + return self._websocket.state == websockets.State.OPEN if self._websocket else False + + @property + def is_closing(self) -> bool: + """Check if the WebSocket is currently closing. + + Returns: + True if the WebSocket is in the process of closing. + """ + return self._websocket.state == websockets.State.CLOSING if self._websocket else False async def _client_task_handler(self): """Handle incoming messages from the WebSocket connection.""" @@ -371,12 +393,18 @@ class WebsocketClientOutputTransport(BaseOutputTransport): """ await self._write_frame(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebSocket with optional WAV header. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ + if self._session.is_closing or not self._session.is_connected: + return False + frame = OutputAudioRawFrame( audio=frame.audio, sample_rate=self.sample_rate, @@ -402,10 +430,16 @@ class WebsocketClientOutputTransport(BaseOutputTransport): # Simulate audio playback with a sleep. await self._write_audio_sleep() + return True + async def _write_frame(self, frame: Frame): """Write a frame to the WebSocket after serialization.""" + if self._session.is_closing or not self._session.is_connected: + return + if not self._params.serializer: return + payload = await self._params.serializer.serialize(frame) if payload: await self._session.send(payload) diff --git a/src/pipecat/transports/websocket/fastapi.py b/src/pipecat/transports/websocket/fastapi.py index cfa68f5cb..72654204a 100644 --- a/src/pipecat/transports/websocket/fastapi.py +++ b/src/pipecat/transports/websocket/fastapi.py @@ -410,14 +410,17 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): """ await self._write_frame(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebSocket with timing simulation. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._client.is_closing or not self._client.is_connected: - return + return False frame = OutputAudioRawFrame( audio=frame.audio, @@ -444,6 +447,8 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): # Simulate audio playback with a sleep. await self._write_audio_sleep() + return True + async def _write_frame(self, frame: Frame): """Serialize and send a frame through the WebSocket.""" if self._client.is_closing or not self._client.is_connected: diff --git a/src/pipecat/transports/websocket/server.py b/src/pipecat/transports/websocket/server.py index 67631ab04..095407422 100644 --- a/src/pipecat/transports/websocket/server.py +++ b/src/pipecat/transports/websocket/server.py @@ -346,14 +346,17 @@ class WebsocketServerOutputTransport(BaseOutputTransport): """ await self._write_frame(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebSocket client with timing control. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if not self._websocket: - return + return False frame = OutputAudioRawFrame( audio=frame.audio, @@ -380,6 +383,8 @@ class WebsocketServerOutputTransport(BaseOutputTransport): # Simulate audio playback with a sleep. await self._write_audio_sleep() + return True + async def _write_frame(self, frame: Frame): """Serialize and send a frame to the WebSocket client.""" if not self._params.serializer: From 17ea0afa6fa5136fc3092031cffe5283a29a8a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 23 Sep 2025 11:03:43 -0700 Subject: [PATCH 31/35] StrandsAgentsProcessor: more formatting fixes --- .../processors/frameworks/strands_agents.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py index 67005d74b..d129c2063 100644 --- a/src/pipecat/processors/frameworks/strands_agents.py +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -84,11 +84,11 @@ class StrandsAgentsProcessor(FrameProcessor): text: The user input text to process through the agent or graph. """ logger.debug(f"Invoking Strands agent with: {text}") + ttfb_tracking = True try: await self.push_frame(LLMFullResponseStartFrame()) await self.start_processing_metrics() await self.start_ttfb_metrics() - ttfb_tracking = True if self.graph: # Graph does not stream; await full result then emit assistant text @@ -108,9 +108,9 @@ class StrandsAgentsProcessor(FrameProcessor): await self.push_frame(LLMTextFrame(str(block["text"]))) # Update usage metrics await self._report_usage_metrics( - agent_result.metrics.accumulated_usage.get('inputTokens', 0), - agent_result.metrics.accumulated_usage.get('outputTokens', 0), - agent_result.metrics.accumulated_usage.get('totalTokens', 0) + agent_result.metrics.accumulated_usage.get("inputTokens", 0), + agent_result.metrics.accumulated_usage.get("outputTokens", 0), + agent_result.metrics.accumulated_usage.get("totalTokens", 0), ) except Exception as parse_err: logger.warning(f"Failed to extract messages from GraphResult: {parse_err}") @@ -123,12 +123,20 @@ class StrandsAgentsProcessor(FrameProcessor): if ttfb_tracking: await self.stop_ttfb_metrics() ttfb_tracking = False - + # Update usage metrics - if isinstance(event, dict) and "event" in event and "metadata" in event['event']: - if 'usage' in event['event']['metadata']: - usage = event['event']['metadata']['usage'] - await self._report_usage_metrics(usage.get('inputTokens', 0), usage.get('outputTokens', 0), usage.get('totalTokens', 0)) + if ( + isinstance(event, dict) + and "event" in event + and "metadata" in event["event"] + ): + if "usage" in event["event"]["metadata"]: + usage = event["event"]["metadata"]["usage"] + await self._report_usage_metrics( + usage.get("inputTokens", 0), + usage.get("outputTokens", 0), + usage.get("totalTokens", 0), + ) except GeneratorExit: logger.warning(f"{self} generator was closed prematurely") except Exception as e: @@ -139,7 +147,7 @@ class StrandsAgentsProcessor(FrameProcessor): ttfb_tracking = False await self.stop_processing_metrics() await self.push_frame(LLMFullResponseEndFrame()) - + def can_generate_metrics(self) -> bool: """Check if this service can generate performance metrics. @@ -149,14 +157,11 @@ class StrandsAgentsProcessor(FrameProcessor): return True async def _report_usage_metrics( - self, - prompt_tokens: int, - completion_tokens: int, - total_tokens: int + self, prompt_tokens: int, completion_tokens: int, total_tokens: int ): tokens = LLMTokenUsage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, - total_tokens=total_tokens + total_tokens=total_tokens, ) - await self.start_llm_usage_metrics(tokens) \ No newline at end of file + await self.start_llm_usage_metrics(tokens) From 780e91eb91d9c482bf3d78589b2e7631fa5b5f38 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 23 Sep 2025 12:50:35 -0400 Subject: [PATCH 32/35] Update persistent conversation storage examples to use universal `LLMContext`. Note that `LLMContext` doesn't have a `get_messages_for_persistent_storage()`; the messages are already in the "standard" format so they can be used directly for storage. --- .../20a-persistent-context-openai.py | 128 +++++++-------- .../20c-persistent-context-anthropic.py | 116 ++++++------- .../20d-persistent-context-gemini.py | 155 +++++++++--------- 3 files changed, 193 insertions(+), 206 deletions(-) diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 9cae10573..93c1fa438 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -12,6 +12,8 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,9 +22,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -66,11 +67,11 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) @@ -87,7 +88,7 @@ async def load_conversation(params: FunctionCallParams): with open(filename, "r") as file: params.context.set_messages(json.load(file)) logger.debug( - f"loaded conversation from {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"loaded conversation from {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) await params.llm.queue_frame(TTSSpeakFrame("Ok, I've loaded that conversation.")) except Exception as e: @@ -100,71 +101,58 @@ messages = [ "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", }, ] -tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", }, }, - { - "type": "function", - "function": { - "name": "save_conversation", - "description": "Save the current conversatione. Use this function to persist the current conversation to external storage.", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, + required=["location", "format"], +) + +save_conversation_function = FunctionSchema( + name="save_conversation", + description="Save the current conversatione. Use this function to persist the current conversation to external storage.", + properties={}, + required=[], +) + +get_filenames_function = FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], +) + +load_conversation_function = FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ + "filename": { + "type": "string", + "description": "The filename of the conversation history to load.", + } }, - { - "type": "function", - "function": { - "name": "get_saved_conversation_filenames", - "description": "Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "load_conversation", - "description": "Load a conversation history. Use this function to load a conversation history into the current session.", - "parameters": { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "The filename of the conversation history to load.", - } - }, - "required": ["filename"], - }, - }, - }, -] + required=["filename"], +) + +tools = ToolsSchema( + standard_tools=[ + weather_function, + save_conversation_function, + get_filenames_function, + load_conversation_function, + ] +) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -211,8 +199,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 92b435aa8..411a976b8 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -12,6 +12,8 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,9 +22,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService @@ -67,12 +68,12 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: # todo: extract 'system' into the first message in the list - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) @@ -89,7 +90,7 @@ async def load_conversation(params: FunctionCallParams): with open(filename, "r") as file: params.context.set_messages(json.load(file)) logger.debug( - f"loaded conversation from {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"loaded conversation from {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) await params.llm.queue_frame(TTSSpeakFrame("Ok, I've loaded that conversation.")) except Exception as e: @@ -108,59 +109,58 @@ messages = [ # {"role": "user", "content": "Tell me"}, # {"role": "user", "content": "a joke"}, ] -tools = [ - { - "name": "get_current_weather", - "description": "Get the current weather", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], + +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", }, }, - { - "name": "save_conversation", - "description": "Save the current conversation. Use this function to persist the current conversation to external storage.", - "input_schema": { - "type": "object", - "properties": {}, - "required": [], - }, + required=["location", "format"], +) + +save_conversation_function = FunctionSchema( + name="save_conversation", + description="Save the current conversation. Use this function to persist the current conversation to external storage.", + properties={}, + required=[], +) + +get_filenames_function = FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], +) + +load_conversation_function = FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ + "filename": { + "type": "string", + "description": "The filename of the conversation history to load.", + } }, - { - "name": "get_saved_conversation_filenames", - "description": "Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", - "input_schema": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - { - "name": "load_conversation", - "description": "Load a conversation history. Use this function to load a conversation history into the current session.", - "input_schema": { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "The filename of the conversation history to load.", - } - }, - "required": ["filename"], - }, - }, -] + required=["filename"], +) + +tools = ToolsSchema( + standard_tools=[ + weather_function, + save_conversation_function, + get_filenames_function, + load_conversation_function, + ] +) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -211,8 +211,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 959256cb2..8dad8148d 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -12,6 +12,8 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,9 +22,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -85,12 +86,12 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_logging(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: # todo: extract 'system' into the first message in the list - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message (the instruction to save the context) messages.pop() json.dump(messages, file, indent=2) @@ -151,78 +152,76 @@ indicate you should use the get_image tool are: # {"role": "user", "content": "Tell me"}, # {"role": "user", "content": "a joke"}, ] -tools = [ - { - "function_declarations": [ - { - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, - }, - { - "name": "save_conversation", - "description": "Save the current conversation. Use this function to persist the current conversation to external storage.", - "parameters": { - "type": "object", - "properties": { - "user_request_text": { - "type": "string", - "description": "The text of the user's request to save the conversation.", - } - }, - "required": ["user_request_text"], - }, - }, - { - "name": "get_saved_conversation_filenames", - "description": "Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", - "parameters": None, - }, - { - "name": "load_conversation", - "description": "Load a conversation history. Use this function to load a conversation history into the current session.", - "parameters": { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "The filename of the conversation history to load.", - } - }, - "required": ["filename"], - }, - }, - { - "name": "get_image", - "description": "Get and image from the camera or video stream.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question to to use when running inference on the acquired image.", - }, - }, - "required": ["question"], - }, - }, - ] + +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, }, -] + required=["location", "format"], +) + +save_conversation_function = FunctionSchema( + name="save_conversation", + description="Save the current conversation. Use this function to persist the current conversation to external storage.", + properties={ + "user_request_text": { + "type": "string", + "description": "The text of the user's request to save the conversation.", + } + }, + required=["user_request_text"], +) + +get_filenames_function = FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], +) + +load_conversation_function = FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ + "filename": { + "type": "string", + "description": "The filename of the conversation history to load.", + } + }, + required=["filename"], +) + +get_image_function = FunctionSchema( + name="get_image", + description="Get and image from the camera or video stream.", + properties={ + "question": { + "type": "string", + "description": "The question to to use when running inference on the acquired image.", + }, + }, + required=["question"], +) + +tools = ToolsSchema( + standard_tools=[ + weather_function, + save_conversation_function, + get_filenames_function, + load_conversation_function, + get_image_function, + ] +) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -266,8 +265,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("load_conversation", load_conversation) llm.register_function("get_image", get_image) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ From 620b1f785cafe383e83a441fa6cbc3e76d8cd36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 23 Sep 2025 11:35:03 -0700 Subject: [PATCH 33/35] examples: update Strands Agents with universal context and add evals --- .../07m-interruptible-aws-strands.py | 31 +++++++++++++------ scripts/evals/run-release-evals.py | 1 + .../processors/frameworks/strands_agents.py | 10 +++--- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/examples/foundational/07m-interruptible-aws-strands.py b/examples/foundational/07m-interruptible-aws-strands.py index eca2947fc..934215fbb 100644 --- a/examples/foundational/07m-interruptible-aws-strands.py +++ b/examples/foundational/07m-interruptible-aws-strands.py @@ -9,14 +9,12 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesAppendFrame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantContextAggregator, - LLMUserContextAggregator, -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.strands_agents import StrandsAgentsProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -115,19 +113,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) # Setup context aggregators for message handling - context = OpenAILLMContext() - tma_in = LLMUserContextAggregator(context=context) - tma_out = LLMAssistantContextAggregator(context=context) + context = LLMContext() + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ transport.input(), # Transport user input stt, # Speech-to-text - tma_in, # User context aggregator + context_aggregator.user(), # User responses llm, # Strands Agents processor tts, # Text-to-speech transport.output(), # Transport bot output - tma_out, # Assistant context aggregator + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -143,6 +140,20 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames( + [ + LLMMessagesAppendFrame( + messages=[ + { + "role": "user", + "content": f"Greet the user and introduce yourself.", + } + ], + run_llm=True, + ) + ] + ) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index f17a7f6fa..1079df1ee 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -83,6 +83,7 @@ TESTS_07 = [ ("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("07m-interruptible-aws-strands.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("07n-interruptible-gemini.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py index d129c2063..829c799ac 100644 --- a/src/pipecat/processors/frameworks/strands_agents.py +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -10,12 +10,12 @@ from loguru import logger from pipecat.frames.frames import ( Frame, + LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, ) from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor try: @@ -71,9 +71,11 @@ class StrandsAgentsProcessor(FrameProcessor): direction: The direction of frame flow in the pipeline. """ await super().process_frame(frame, direction) - if isinstance(frame, OpenAILLMContextFrame): - text = frame.context.messages[-1]["content"] - await self._ainvoke(str(text).strip()) + if isinstance(frame, LLMContextFrame): + messages = frame.context.get_messages() + if messages: + last_message = messages[-1] + await self._ainvoke(str(last_message["content"]).strip()) else: await self.push_frame(frame, direction) From 2571cb2e692ec7a775766a7a45366841bc3452ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 23 Sep 2025 15:28:07 -0700 Subject: [PATCH 34/35] tests: fix formatting --- tests/test_audio_buffer_processor.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_audio_buffer_processor.py b/tests/test_audio_buffer_processor.py index 24ae42265..f35c5adf0 100644 --- a/tests/test_audio_buffer_processor.py +++ b/tests/test_audio_buffer_processor.py @@ -13,7 +13,9 @@ from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor class _PassthroughResampler: - async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes: # pragma: no cover - trivial + async def resample( + self, audio: bytes, in_rate: int, out_rate: int + ) -> bytes: # pragma: no cover - trivial return audio @@ -40,7 +42,9 @@ class TestAudioBufferProcessor(unittest.IsolatedAsyncioTestCase): captured["merged"] = (audio, sample_rate, num_channels) audio_event.set() - async def on_track_audio_data(_, user: bytes, bot: bytes, sample_rate: int, num_channels: int): + async def on_track_audio_data( + _, user: bytes, bot: bytes, sample_rate: int, num_channels: int + ): captured["tracks"] = (user, bot, sample_rate, num_channels) track_event.set() @@ -80,7 +84,9 @@ class TestAudioBufferProcessor(unittest.IsolatedAsyncioTestCase): captured["merged"] = (audio, sample_rate, num_channels) audio_event.set() - async def on_track_audio_data(_, user: bytes, bot: bytes, sample_rate: int, num_channels: int): + async def on_track_audio_data( + _, user: bytes, bot: bytes, sample_rate: int, num_channels: int + ): captured["tracks"] = (user, bot, sample_rate, num_channels) track_event.set() From d7c8f8df53d1e8d190f4ba4943bc0a09e315d84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 23 Sep 2025 15:41:47 -0700 Subject: [PATCH 35/35] update CHANGELOG with AudioBufferProcessor fixes --- CHANGELOG.md | 3 +++ .../audio/audio_buffer_processor.py | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad49ae5d7..f3169f278 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an `AudioBufferProcessor` issues that was causing user audio to be + missing in stereo recordings causing bot and user overlaps. + - Fixed a `BaseOutputTransport` issue that could produce large saved `AudioBufferProcessor` files when using an audio mixer. diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index fefbafc9b..378393d9d 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -234,7 +234,7 @@ class AudioBufferProcessor(FrameProcessor): or len(self._bot_audio_buffer) >= self._buffer_size ): await self._call_on_audio_data_handler() - self._clear_primary_audio_buffers() + self._reset_primary_audio_buffers() # Process turn recording with preprocessed data. if self._enable_turn_audio: @@ -308,22 +308,25 @@ class AudioBufferProcessor(FrameProcessor): def _reset_recording(self): """Reset recording state and buffers.""" - self._reset_audio_buffers() + self._reset_all_audio_buffers() self._last_user_frame_at = time.time() self._last_bot_frame_at = time.time() - def _reset_audio_buffers(self): + def _reset_all_audio_buffers(self): """Reset all audio buffers to empty state.""" - self._user_audio_buffer = bytearray() - self._bot_audio_buffer = bytearray() - self._user_turn_audio_buffer = bytearray() - self._bot_turn_audio_buffer = bytearray() + self._reset_primary_audio_buffers() + self._reset_turn_audio_buffers() - def _clear_primary_audio_buffers(self): + def _reset_primary_audio_buffers(self): """Clear user and bot buffers while preserving turn buffers and timestamps.""" self._user_audio_buffer = bytearray() self._bot_audio_buffer = bytearray() + def _reset_turn_audio_buffers(self): + """Clear user and bot turn buffers while preserving primary buffers and timestamps.""" + self._user_turn_audio_buffer = bytearray() + self._bot_turn_audio_buffer = bytearray() + def _align_track_buffers(self): """Pad the shorter track with silence so both tracks stay in sync.""" user_len = len(self._user_audio_buffer)