diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d905d43..f3169f278 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for using universal `LLMContext` with: + + - `LLMLogObserver` + - `GatedLLMContextAggregator` (formerly `GatedOpenAILLMContextAggregator`) + - `LangchainProcessor` + - `Mem0MemoryService` + +- Added `StrandsAgentProcessor` which allows you to use the Strands Agents + framework to build your voice agents. + See https://strandsagents.com + +- Added `ElevenLabsSTTService` for speech-to-text transcription. + +- Added a peer connection monitor to the `SmallWebRTCConnection` that + automatically disconnects if the connection fails to establish within + the timeout (1 minute by default). + +- Added memory cleanup improvements to reduce memory peaks. + - Added `on_before_process_frame`, `on_after_process_frame`, `on_before_push_frame` and `on_after_push_frame`. These are synchronous events that get called before and after a frame is processed or pushed. Note that @@ -49,6 +68,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `BaseOutputTransport` methods `write_audio_frame` and `write_video_frame` now + return a boolean to indicate if the transport implementation was able to write + the given frame or not. + - Updated Silero VAD model to v6. - Updated `livekit` to 1.0.13. @@ -79,6 +102,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an `AudioBufferProcessor` issues that was causing user audio to be + missing in stereo recordings causing bot and user overlaps. + +- Fixed a `BaseOutputTransport` issue that could produce large saved + `AudioBufferProcessor` files when using an audio mixer. + +- Fixed a `PipelineRunner` issue on Windows where setting up SIGINT and SIGTERM + was raising an exception. + - Fixed an issue where multiple handlers for an event would not run in parallel. - Fixed `DailyTransport.sip_call_transfer()` to automatically use the session diff --git a/README.md b/README.md index a217318ac..0a76ff5ea 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ pipecat -[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) +[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/pipecat-ai/pipecat) # 🎙️ Pipecat: Real-Time Voice & Multimodal AI Agents @@ -79,7 +79,7 @@ You can connect to Pipecat from any platform using our official SDKs: | Category | Services | | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index d04ff6df7..3141a69d6 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -9,14 +9,11 @@ import os from dotenv import load_dotenv from loguru import logger -from pipecat.frames.frames import EndFrame +from pipecat.frames.frames import EndFrame, LLMContextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -63,7 +60,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Register an event handler so we can play the audio when the client joins @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): - await task.queue_frames([OpenAILLMContextFrame(OpenAILLMContext(messages)), EndFrame()]) + await task.queue_frames([LLMContextFrame(LLMContext(messages)), EndFrame()]) runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) diff --git a/examples/foundational/04-transports-small-webrtc.py b/examples/foundational/04-transports-small-webrtc.py index 54c63117e..997b917f9 100644 --- a/examples/foundational/04-transports-small-webrtc.py +++ b/examples/foundational/04-transports-small-webrtc.py @@ -25,7 +25,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService @@ -80,8 +81,8 @@ async def run_example(webrtc_connection: SmallWebRTCConnection): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/04a-transports-daily.py b/examples/foundational/04a-transports-daily.py index 2b8169eb5..5d2855450 100644 --- a/examples/foundational/04a-transports-daily.py +++ b/examples/foundational/04a-transports-daily.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.daily import configure from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.openai.llm import OpenAILLMService @@ -64,8 +65,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/04b-transports-livekit.py b/examples/foundational/04b-transports-livekit.py index 9c4724266..402fd79ca 100644 --- a/examples/foundational/04b-transports-livekit.py +++ b/examples/foundational/04b-transports-livekit.py @@ -18,15 +18,16 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( InterruptionFrame, - TextFrame, TranscriptionFrame, + TTSSpeakFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.livekit import configure from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService @@ -73,8 +74,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ @@ -102,7 +103,7 @@ async def main(): async def on_first_participant_joined(transport, participant_id): await asyncio.sleep(1) await task.queue_frame( - TextFrame( + TTSSpeakFrame( "Hello there! How are you doing today? Would you like to talk about the weather?" ) ) diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index c4968245a..cdcd05350 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -14,6 +14,7 @@ from loguru import logger from pipecat.frames.frames import ( DataFrame, Frame, + LLMContextFrame, LLMFullResponseStartFrame, TextFrame, ) @@ -21,10 +22,8 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.sync_parallel_pipeline import SyncParallelPipeline from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.sentence import SentenceAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments @@ -156,7 +155,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): } ] frames.append(MonthFrame(month=month)) - frames.append(OpenAILLMContextFrame(OpenAILLMContext(messages))) + frames.append(LLMContextFrame(LLMContext(messages))) task = PipelineTask( pipeline, diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index f40d2a163..48fed091e 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -15,6 +15,7 @@ from loguru import logger from pipecat.frames.frames import ( Frame, + LLMContextFrame, OutputAudioRawFrame, TextFrame, TTSAudioRawFrame, @@ -24,10 +25,8 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.sync_parallel_pipeline import SyncParallelPipeline from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.sentence import SentenceAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaHttpTTSService @@ -140,7 +139,7 @@ async def main(): ) task = PipelineTask(pipeline) - await task.queue_frame(OpenAILLMContextFrame(OpenAILLMContext(messages))) + await task.queue_frame(LLMContextFrame(LLMContext(messages))) await task.stop_when_done() await runner.run(task) diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index b48933e00..92e04e25a 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -23,7 +23,8 @@ from pipecat.metrics.metrics import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -103,8 +104,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index c5cbcff81..4f8816df6 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -24,7 +24,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -116,8 +117,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) image_sync_aggregator = ImageSyncAggregator( os.path.join(os.path.dirname(__file__), "assets", "speaking.png"), diff --git a/examples/foundational/07-interruptible-cartesia-http.py b/examples/foundational/07-interruptible-cartesia-http.py index d2e0f9c08..d121e40ba 100644 --- a/examples/foundational/07-interruptible-cartesia-http.py +++ b/examples/foundational/07-interruptible-cartesia-http.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaHttpTTSService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 5beddf7e1..81ba692c7 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -73,8 +74,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07a-interruptible-speechmatics-vad.py b/examples/foundational/07a-interruptible-speechmatics-vad.py index 09777c9b7..55514017f 100644 --- a/examples/foundational/07a-interruptible-speechmatics-vad.py +++ b/examples/foundational/07a-interruptible-speechmatics-vad.py @@ -13,10 +13,11 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMUserAggregatorParams, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.elevenlabs.tts import ElevenLabsTTSService @@ -123,8 +124,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), ) diff --git a/examples/foundational/07a-interruptible-speechmatics.py b/examples/foundational/07a-interruptible-speechmatics.py index 1d5806340..3d1e639b9 100644 --- a/examples/foundational/07a-interruptible-speechmatics.py +++ b/examples/foundational/07a-interruptible-speechmatics.py @@ -17,10 +17,11 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import ( LLMUserAggregatorParams, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.elevenlabs.tts import ElevenLabsTTSService @@ -112,8 +113,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), ) diff --git a/examples/foundational/07aa-interruptible-soniox.py b/examples/foundational/07aa-interruptible-soniox.py index 16180e77c..addfde49a 100644 --- a/examples/foundational/07aa-interruptible-soniox.py +++ b/examples/foundational/07aa-interruptible-soniox.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -73,8 +74,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ab-interruptible-inworld-http.py b/examples/foundational/07ab-interruptible-inworld-http.py index 5fd761bfb..907a358d2 100644 --- a/examples/foundational/07ab-interruptible-inworld-http.py +++ b/examples/foundational/07ab-interruptible-inworld-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -84,8 +85,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ac-interruptible-asyncai-http.py b/examples/foundational/07ac-interruptible-asyncai-http.py index d3f286393..7a92f0f7c 100644 --- a/examples/foundational/07ac-interruptible-asyncai-http.py +++ b/examples/foundational/07ac-interruptible-asyncai-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.asyncai.tts import AsyncAIHttpTTSService @@ -79,8 +80,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ac-interruptible-asyncai.py b/examples/foundational/07ac-interruptible-asyncai.py index 851fa4338..8216c317b 100644 --- a/examples/foundational/07ac-interruptible-asyncai.py +++ b/examples/foundational/07ac-interruptible-asyncai.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.asyncai.tts import AsyncAITTSService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index 2c6e653b1..aa647ff33 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -98,8 +99,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 3ef633846..32b637cf8 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -23,13 +23,8 @@ from pipecat.frames.frames import LLMMessagesUpdateFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantContextAggregator, - LLMUserContextAggregator, -) -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.langchain import LangchainProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -106,19 +101,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) lc = LangchainProcessor(history_chain) - context = OpenAILLMContext() - tma_in = LLMUserContextAggregator(context=context) - tma_out = LLMAssistantContextAggregator(context=context) + context = LLMContext() + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ transport.input(), # Transport user input stt, - tma_in, # User responses + context_aggregator.user(), # User responses lc, # Langchain tts, # TTS transport.output(), # Transport bot output - tma_out, # Assistant spoken responses + context_aggregator.assistant(), # Assistant spoken responses ] ) diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index 3f9330bbd..fc43bb134 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -71,8 +72,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index 9c4cdb204..7dd46124d 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -72,8 +73,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index 27bd212fc..cd922ddc2 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -19,10 +19,12 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.elevenlabs.stt import ElevenLabsSTTService from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -62,7 +64,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Create an HTTP session async with aiohttp.ClientSession() as session: - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = ElevenLabsSTTService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + aiohttp_session=session, + ) tts = ElevenLabsHttpTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), @@ -79,8 +84,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index ca40f6e4c..da2a8eb00 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07e-interruptible-playht-http.py b/examples/foundational/07e-interruptible-playht-http.py index be5769fbf..0026afc86 100644 --- a/examples/foundational/07e-interruptible-playht-http.py +++ b/examples/foundational/07e-interruptible-playht-http.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 6b2b8fde8..e854a61cb 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index 50a589951..f829063d3 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.azure.llm import AzureLLMService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index c73f074f7..d3e5ee373 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.openai.llm import OpenAILLMService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 1fbb716a4..313843b31 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -80,8 +81,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 675588f41..029961040 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -78,8 +79,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 3e24a7614..20ddc3f66 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -84,8 +85,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index a9da69f2c..daa944cce 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -71,8 +72,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07l-interruptible-groq.py b/examples/foundational/07l-interruptible-groq.py index 7a57bfe3e..6f90d0d8d 100644 --- a/examples/foundational/07l-interruptible-groq.py +++ b/examples/foundational/07l-interruptible-groq.py @@ -18,8 +18,9 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.groq.llm import GroqLLMService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05) ) diff --git a/examples/foundational/07m-interruptible-aws-strands.py b/examples/foundational/07m-interruptible-aws-strands.py new file mode 100644 index 000000000..934215fbb --- /dev/null +++ b/examples/foundational/07m-interruptible-aws-strands.py @@ -0,0 +1,177 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesAppendFrame, LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.frameworks.strands_agents import StrandsAgentsProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.stt import AWSTranscribeSTTService +from pipecat.services.aws.tts import AWSPollyTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +# Strands agent setup +try: + from strands import Agent, tool + from strands.models import BedrockModel +except ImportError: + logger.warning("Strands not installed. Please install with: pip install strands-agents") + Agent = None + BedrockModel = None + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +def build_agent(model_id: str, max_tokens: int): + """Create and configure a Strands agent for NAB customer service coaching. + + Args: + model_id: The AWS Bedrock model ID to use + max_tokens: Maximum tokens for the model + + Returns: + Configured Strands Agent + """ + + @tool + def check_weather(location: str) -> str: + if location.lower() == "san francisco": + return "The weather in San Francisco is sunny and 30 degrees." + elif location.lower() == "sydney": + return "The weather in Sydney is cloudy and 20 degrees." + else: + return "I'm not sure about the weather in that location." + + agent = Agent( + model=BedrockModel( + model_id=model_id, + max_tokens=max_tokens, + ), + tools=[check_weather], + system_prompt="You are a helpful assistant that can check the weather in a given location.", + ) + + return agent + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = AWSTranscribeSTTService() + + tts = AWSPollyTTSService( + region="us-west-2", # only specific regions support generative TTS + voice_id="Joanna", + params=AWSPollyTTSService.InputParams(engine="generative", rate="1.1"), + ) + + # Create Strands agent processor + try: + agent = build_agent(model_id="us.anthropic.claude-3-5-haiku-20241022-v1:0", max_tokens=8000) + llm = StrandsAgentsProcessor(agent=agent) + logger.info("Successfully created Strands agent for NAB customer service coaching") + except Exception as e: + logger.error(f"Failed to create Strands agent: {e}") + raise ValueError( + "Unable to create Strands processor. Please ensure you have properly " + "installed strands-agents and configured your AWS credentials." + ) + + # Setup context aggregators for message handling + context = LLMContext() + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # Speech-to-text + context_aggregator.user(), # User responses + llm, # Strands Agents processor + tts, # Text-to-speech + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames( + [ + LLMMessagesAppendFrame( + messages=[ + { + "role": "user", + "content": f"Greet the user and introduce yourself.", + } + ], + run_llm=True, + ) + ] + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index 754fd3b69..9343797a9 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -16,7 +16,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.aws.llm import AWSBedrockLLMService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07n-interruptible-gemini.py b/examples/foundational/07n-interruptible-gemini.py index f12693bd6..4da14f908 100644 --- a/examples/foundational/07n-interruptible-gemini.py +++ b/examples/foundational/07n-interruptible-gemini.py @@ -36,7 +36,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.llm import GoogleLLMService @@ -112,8 +113,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index f4ef03aee..203099c59 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.llm import GoogleLLMService @@ -84,8 +85,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 57e12e7fc..1b7a1daea 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.assemblyai.stt import AssemblyAISTTService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 3af964e41..3cbe7f28a 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07q-interruptible-rime-http.py b/examples/foundational/07q-interruptible-rime-http.py index 82b581d16..2e1732c6b 100644 --- a/examples/foundational/07q-interruptible-rime-http.py +++ b/examples/foundational/07q-interruptible-rime-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -80,8 +81,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 18fb459ec..9ba070151 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index ae1556a58..4f451fd53 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.nim.llm import NimLLMService @@ -71,8 +72,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 06e59b76d..40759f262 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -31,7 +31,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -244,8 +245,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) audio_collector = UserAudioCollector(context, context_aggregator.user()) pull_transcript_out_of_llm_output = TranscriptExtractor(context) fixup_context_messages = TranscriptionContextFixup(context) diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index 161e5c724..52c4ebbb1 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07v-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py index b63fc2fdf..e22ef734e 100644 --- a/examples/foundational/07v-interruptible-neuphonic-http.py +++ b/examples/foundational/07v-interruptible-neuphonic-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -79,8 +80,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07v-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py index c9265b0f0..c9bb13a70 100644 --- a/examples/foundational/07v-interruptible-neuphonic.py +++ b/examples/foundational/07v-interruptible-neuphonic.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py index dcef8dddf..dc572f652 100644 --- a/examples/foundational/07w-interruptible-fal.py +++ b/examples/foundational/07w-interruptible-fal.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07x-interruptible-local.py b/examples/foundational/07x-interruptible-local.py index 8d3ba8d17..4778d262f 100644 --- a/examples/foundational/07x-interruptible-local.py +++ b/examples/foundational/07x-interruptible-local.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -59,8 +60,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07y-interruptible-minimax.py b/examples/foundational/07y-interruptible-minimax.py index c499b8318..7995ed832 100644 --- a/examples/foundational/07y-interruptible-minimax.py +++ b/examples/foundational/07y-interruptible-minimax.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07z-interruptible-sarvam-http.py b/examples/foundational/07z-interruptible-sarvam-http.py index 7140e4c1f..29851d254 100644 --- a/examples/foundational/07z-interruptible-sarvam-http.py +++ b/examples/foundational/07z-interruptible-sarvam-http.py @@ -18,7 +18,8 @@ from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -79,8 +80,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/07z-interruptible-sarvam.py b/examples/foundational/07z-interruptible-sarvam.py index 6b027f3dd..44e0b7844 100644 --- a/examples/foundational/07z-interruptible-sarvam.py +++ b/examples/foundational/07z-interruptible-sarvam.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py index a5d1b98f6..b84e945c3 100644 --- a/examples/foundational/08-bots-arguing.py +++ b/examples/foundational/08-bots-arguing.py @@ -6,13 +6,11 @@ from typing import Tuple import aiohttp from dotenv import load_dotenv -from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, TextFrame +from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMContextFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.aggregators import SentenceAggregator -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.daily import configure from pipecat.services.azure import AzureLLMService, AzureTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService @@ -83,7 +81,7 @@ async def main(): sentence_aggregator = SentenceAggregator() pipeline = Pipeline([llm, sentence_aggregator, tts1], source_queue, sink_queue) - await source_queue.put(OpenAILLMContextFrame(OpenAILLMContext(messages))) + await source_queue.put(LLMContextFrame(LLMContext(messages))) await source_queue.put(EndFrame()) await pipeline.run_pipeline() diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index d86f40241..30a7d6ca9 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.wake_check_filter import WakeCheckFilter from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -76,8 +77,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): hey_robot_filter = WakeCheckFilter(["hey robot", "hey, robot"]) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 61e6326dd..9eae5944f 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -16,6 +16,7 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( Frame, + LLMContextFrame, LLMFullResponseEndFrame, OutputAudioRawFrame, TTSSpeakFrame, @@ -23,10 +24,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.logger import FrameLogger from pipecat.runner.types import RunnerArguments @@ -74,7 +73,7 @@ class InboundSoundEffectWrapper(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, LLMContextFrame): await self.push_frame(sounds["ding2.wav"]) # In case anything else downstream needs it await self.push_frame(frame, direction) @@ -126,8 +125,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) out_sound = OutboundSoundEffectWrapper() in_sound = InboundSoundEffectWrapper() fl = FrameLogger("LLM Out") diff --git a/examples/foundational/13k-elevenlabs-transcription.py b/examples/foundational/13k-elevenlabs-transcription.py new file mode 100644 index 000000000..bfac16c3f --- /dev/null +++ b/examples/foundational/13k-elevenlabs-transcription.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import Frame, TranscriptionFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.elevenlabs.stt import ElevenLabsSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +class TranscriptionLogger(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + print(f"Transcription: {frame.text}") + + # Push all frames through + await self.push_frame(frame, direction) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_in_enabled=True, vad_analyzer=SileroVADAnalyzer()), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, vad_analyzer=SileroVADAnalyzer() + ), + "webrtc": lambda: TransportParams(audio_in_enabled=True, vad_analyzer=SileroVADAnalyzer()), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + async with aiohttp.ClientSession() as session: + stt = ElevenLabsSTTService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + aiohttp_session=session, + ) + + tl = TranscriptionLogger() + + pipeline = Pipeline([transport.input(), stt, tl]) + + task = PipelineTask( + pipeline, + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index e9e96a989..42c728bae 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -123,8 +124,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 038d475ce..5f9a0ec06 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService @@ -118,8 +119,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "user", "content": "Say 'hello' to start the conversation."}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14aa-function-calling-aws-universal-context.py b/examples/foundational/14aa-function-calling-aws-universal-context.py deleted file mode 100644 index 1a7960811..000000000 --- a/examples/foundational/14aa-function-calling-aws-universal-context.py +++ /dev/null @@ -1,219 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.aws.llm import AWSBedrockLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.services.daily import DailyParams - -load_dotenv(override=True) - - -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def get_image(params: FunctionCallParams): - question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") - - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, - ) - - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AWSBedrockLLMService( - aws_region="us-west-2", - model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", - # Note: usually, prefer providing latency="optimized" param. - # Here we can't because AWS Bedrock doesn't support it for Claude 3.7, - # which we need for image input. - params=AWSBedrockLLMService.InputParams(temperature=0.8), - ) - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) - - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to two tools: get_weather and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? - -If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise. - """ - - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Start the conversation by introducing yourself."}, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User speech to text - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - global client_id - client_id = get_transport_client_id(transport, client) - - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 2905509ef..009f59500 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -165,8 +166,8 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo {"role": "user", "content": "Start the conversation by introducing yourself."}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 904c7caf3..9e626ef14 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -109,8 +110,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 37bc09f94..48cf95ee9 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -154,8 +155,8 @@ indicate you should use the get_image tool are: {"role": "system", "content": system_prompt}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14e-function-calling-google.py b/examples/foundational/14e-function-calling-google.py index 60b62f22b..528051d8e 100644 --- a/examples/foundational/14e-function-calling-google.py +++ b/examples/foundational/14e-function-calling-google.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -175,8 +176,8 @@ indicate you should use the get_image tool are: {"role": "user", "content": "Say hello."}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 6a9f7989a..568cf43be 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -20,8 +20,9 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -107,8 +108,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05) ) diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 0f332aa4e..a9bba2c4d 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -102,8 +103,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 757020852..ab72e2bcc 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.azure.llm import AzureLLMService @@ -110,8 +111,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 11a0b6fee..96dce71e6 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -113,8 +114,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 678792501..579a21b95 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -107,8 +108,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 99f62ab64..93dbee8b1 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -116,8 +117,8 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index d2ca39755..e7de42a43 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -116,8 +117,8 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index ebcdaf9e8..93221e1da 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.azure.tts import AzureTTSService @@ -110,8 +111,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 4c046135f..2f37768b7 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -24,7 +24,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -80,8 +81,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index 17abe1179..f9ad264eb 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -112,8 +113,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14q-function-calling-qwen.py b/examples/foundational/14q-function-calling-qwen.py index 7f09c412d..650142592 100644 --- a/examples/foundational/14q-function-calling-qwen.py +++ b/examples/foundational/14q-function-calling-qwen.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -108,8 +109,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index 95899dbe8..03aa7bb96 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.aws.llm import AWSBedrockLLMService @@ -123,8 +124,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14s-function-calling-sambanova.py b/examples/foundational/14s-function-calling-sambanova.py index eed121cdb..e425ccd29 100644 --- a/examples/foundational/14s-function-calling-sambanova.py +++ b/examples/foundational/14s-function-calling-sambanova.py @@ -20,8 +20,9 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -113,8 +114,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator( + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05) ) diff --git a/examples/foundational/14t-function-calling-direct.py b/examples/foundational/14t-function-calling-direct.py index 978513b4d..872cb2a8e 100644 --- a/examples/foundational/14t-function-calling-direct.py +++ b/examples/foundational/14t-function-calling-direct.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -109,8 +110,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14u-function-calling-ollama.py b/examples/foundational/14u-function-calling-ollama.py index c49ea2cf9..3e87c601a 100644 --- a/examples/foundational/14u-function-calling-ollama.py +++ b/examples/foundational/14u-function-calling-ollama.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -125,8 +126,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14v-function-calling-openai.py b/examples/foundational/14v-function-calling-openai.py index ba4a99117..d259e7131 100644 --- a/examples/foundational/14v-function-calling-openai.py +++ b/examples/foundational/14v-function-calling-openai.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.llm_service import FunctionCallParams @@ -131,8 +132,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14w-function-calling-mistral.py b/examples/foundational/14w-function-calling-mistral.py index f68143073..ce61085e2 100644 --- a/examples/foundational/14w-function-calling-mistral.py +++ b/examples/foundational/14w-function-calling-mistral.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -119,8 +120,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/14x-function-calling-universal-context.py b/examples/foundational/14x-function-calling-universal-context.py deleted file mode 100644 index 42c728bae..000000000 --- a/examples/foundational/14x-function-calling-universal-context.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - - -async def fetch_weather_from_api(params: FunctionCallParams): - await params.result_callback({"conditions": "nice", "temperature": "75"}) - - -async def fetch_restaurant_recommendation(params: FunctionCallParams): - await params.result_callback({"name": "The Golden Dragon"}) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) - llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) - - @llm.event_handler("on_function_calls_started") - async def on_function_calls_started(service, function_calls): - await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) - - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location", "format"], - ) - restaurant_function = FunctionSchema( - name="get_restaurant_recommendation", - description="Get a restaurant recommendation", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/14y-function-calling-google-universal-context.py b/examples/foundational/14y-function-calling-google-universal-context.py deleted file mode 100644 index 528051d8e..000000000 --- a/examples/foundational/14y-function-calling-google-universal-context.py +++ /dev/null @@ -1,234 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.google.llm import GoogleLLMService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams - -load_dotenv(override=True) - - -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def fetch_restaurant_recommendation(params: FunctionCallParams): - await params.result_callback({"name": "The Golden Dragon"}) - - -async def get_image(params: FunctionCallParams): - question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") - - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, - ) - - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) - llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) - - @llm.event_handler("on_function_calls_started") - async def on_function_calls_started(service, function_calls): - await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) - - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location", "format"], - ) - restaurant_function = FunctionSchema( - name="get_restaurant_recommendation", - description="Get a restaurant recommendation", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function, restaurant_function]) - - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to three tools: get_weather, get_restaurant_recommendation, and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? -""" - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Say hello."}, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - global client_id - client_id = get_transport_client_id(transport, client) - - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/14z-function-calling-anthropic-universal-context.py b/examples/foundational/14z-function-calling-anthropic-universal-context.py deleted file mode 100644 index 253ebf4d0..000000000 --- a/examples/foundational/14z-function-calling-anthropic-universal-context.py +++ /dev/null @@ -1,216 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import asyncio -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.services.daily import DailyParams - -load_dotenv(override=True) - - -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def get_image(params: FunctionCallParams): - question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") - - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, - ) - - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - model="claude-3-7-sonnet-latest", - params=AnthropicLLMService.InputParams(enable_prompt_caching=True), - ) - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) - - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to two tools: get_weather and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? - -If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise. - """ - - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Start the conversation by introducing yourself."}, - ] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User speech to text - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - global client_id - client_id = get_transport_client_id(transport, client) - - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 598f007af..afa51289c 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -9,8 +9,9 @@ import os from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,7 +21,8 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -120,25 +122,19 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) llm.register_function("switch_voice", tts.switch_voice) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "switch_voice", - "description": "Switch your voice only when the user asks you to", - "parameters": { - "type": "object", - "properties": { - "voice": { - "type": "string", - "description": "The voice the user wants you to use", - }, - }, - "required": ["voice"], - }, + switch_voice_function = FunctionSchema( + name="switch_voice", + description="Switch your voice only when the user asks you to", + properties={ + "voice": { + "type": "string", + "description": "The voice the user wants you to use", }, - ) - ] + }, + required=["voice"], + ) + tools = ToolsSchema(standard_tools=[switch_voice_function]) + messages = [ { "role": "system", @@ -146,8 +142,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index af8357065..7e70b0f2d 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -10,8 +10,9 @@ import os from deepgram import LiveOptions from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -21,7 +22,8 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -111,25 +113,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) llm.register_function("switch_language", tts.switch_language) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "switch_language", - "description": "Switch to another language when the user asks you to", - "parameters": { - "type": "object", - "properties": { - "language": { - "type": "string", - "description": "The language the user wants you to speak", - }, - }, - "required": ["language"], - }, + switch_language_function = FunctionSchema( + name="switch_language", + description="Switch to another language when the user asks you to", + properties={ + "language": { + "type": "string", + "description": "The language the user wants you to speak", }, - ) - ] + }, + required=["language"], + ) + tools = ToolsSchema(standard_tools=[switch_language_function]) messages = [ { "role": "system", @@ -137,8 +132,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index cbf2de9dc..89316e63c 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 5c76f3d48..cc36d891d 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import EndFrame, LLMMessagesAppendFrame, LLMRunFrame, from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.user_idle_processor import UserIdleProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) async def handle_user_idle(user_idle: UserIdleProcessor, retry_count: int) -> bool: if retry_count == 1: diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 9cae10573..93c1fa438 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -12,6 +12,8 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,9 +22,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -66,11 +67,11 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) @@ -87,7 +88,7 @@ async def load_conversation(params: FunctionCallParams): with open(filename, "r") as file: params.context.set_messages(json.load(file)) logger.debug( - f"loaded conversation from {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"loaded conversation from {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) await params.llm.queue_frame(TTSSpeakFrame("Ok, I've loaded that conversation.")) except Exception as e: @@ -100,71 +101,58 @@ messages = [ "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", }, ] -tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", }, }, - { - "type": "function", - "function": { - "name": "save_conversation", - "description": "Save the current conversatione. Use this function to persist the current conversation to external storage.", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, + required=["location", "format"], +) + +save_conversation_function = FunctionSchema( + name="save_conversation", + description="Save the current conversatione. Use this function to persist the current conversation to external storage.", + properties={}, + required=[], +) + +get_filenames_function = FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], +) + +load_conversation_function = FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ + "filename": { + "type": "string", + "description": "The filename of the conversation history to load.", + } }, - { - "type": "function", - "function": { - "name": "get_saved_conversation_filenames", - "description": "Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "load_conversation", - "description": "Load a conversation history. Use this function to load a conversation history into the current session.", - "parameters": { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "The filename of the conversation history to load.", - } - }, - "required": ["filename"], - }, - }, - }, -] + required=["filename"], +) + +tools = ToolsSchema( + standard_tools=[ + weather_function, + save_conversation_function, + get_filenames_function, + load_conversation_function, + ] +) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -211,8 +199,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 92b435aa8..411a976b8 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -12,6 +12,8 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,9 +22,8 @@ from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService @@ -67,12 +68,12 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: # todo: extract 'system' into the first message in the list - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) @@ -89,7 +90,7 @@ async def load_conversation(params: FunctionCallParams): with open(filename, "r") as file: params.context.set_messages(json.load(file)) logger.debug( - f"loaded conversation from {filename}\n{json.dumps(params.context.messages, indent=4)}" + f"loaded conversation from {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) await params.llm.queue_frame(TTSSpeakFrame("Ok, I've loaded that conversation.")) except Exception as e: @@ -108,59 +109,58 @@ messages = [ # {"role": "user", "content": "Tell me"}, # {"role": "user", "content": "a joke"}, ] -tools = [ - { - "name": "get_current_weather", - "description": "Get the current weather", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], + +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", }, }, - { - "name": "save_conversation", - "description": "Save the current conversation. Use this function to persist the current conversation to external storage.", - "input_schema": { - "type": "object", - "properties": {}, - "required": [], - }, + required=["location", "format"], +) + +save_conversation_function = FunctionSchema( + name="save_conversation", + description="Save the current conversation. Use this function to persist the current conversation to external storage.", + properties={}, + required=[], +) + +get_filenames_function = FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], +) + +load_conversation_function = FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ + "filename": { + "type": "string", + "description": "The filename of the conversation history to load.", + } }, - { - "name": "get_saved_conversation_filenames", - "description": "Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", - "input_schema": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - { - "name": "load_conversation", - "description": "Load a conversation history. Use this function to load a conversation history into the current session.", - "input_schema": { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "The filename of the conversation history to load.", - } - }, - "required": ["filename"], - }, - }, -] + required=["filename"], +) + +tools = ToolsSchema( + standard_tools=[ + weather_function, + save_conversation_function, + get_filenames_function, + load_conversation_function, + ] +) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -211,8 +211,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 959256cb2..8dad8148d 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -12,6 +12,8 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,9 +22,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -85,12 +86,12 @@ async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_logging(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages(), indent=4)}" ) try: with open(filename, "w") as file: # todo: extract 'system' into the first message in the list - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last message (the instruction to save the context) messages.pop() json.dump(messages, file, indent=2) @@ -151,78 +152,76 @@ indicate you should use the get_image tool are: # {"role": "user", "content": "Tell me"}, # {"role": "user", "content": "a joke"}, ] -tools = [ - { - "function_declarations": [ - { - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, - }, - { - "name": "save_conversation", - "description": "Save the current conversation. Use this function to persist the current conversation to external storage.", - "parameters": { - "type": "object", - "properties": { - "user_request_text": { - "type": "string", - "description": "The text of the user's request to save the conversation.", - } - }, - "required": ["user_request_text"], - }, - }, - { - "name": "get_saved_conversation_filenames", - "description": "Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", - "parameters": None, - }, - { - "name": "load_conversation", - "description": "Load a conversation history. Use this function to load a conversation history into the current session.", - "parameters": { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "The filename of the conversation history to load.", - } - }, - "required": ["filename"], - }, - }, - { - "name": "get_image", - "description": "Get and image from the camera or video stream.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question to to use when running inference on the acquired image.", - }, - }, - "required": ["question"], - }, - }, - ] + +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, }, -] + required=["location", "format"], +) + +save_conversation_function = FunctionSchema( + name="save_conversation", + description="Save the current conversation. Use this function to persist the current conversation to external storage.", + properties={ + "user_request_text": { + "type": "string", + "description": "The text of the user's request to save the conversation.", + } + }, + required=["user_request_text"], +) + +get_filenames_function = FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], +) + +load_conversation_function = FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ + "filename": { + "type": "string", + "description": "The filename of the conversation history to load.", + } + }, + required=["filename"], +) + +get_image_function = FunctionSchema( + name="get_image", + description="Get and image from the camera or video stream.", + properties={ + "question": { + "type": "string", + "description": "The question to to use when running inference on the acquired image.", + }, + }, + required=["question"], +) + +tools = ToolsSchema( + standard_tools=[ + weather_function, + save_conversation_function, + get_filenames_function, + load_conversation_function, + get_image_function, + ] +) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -266,8 +265,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("load_conversation", load_conversation) llm.register_function("get_image", get_image) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/21-tavus-transport.py b/examples/foundational/21-tavus-transport.py index 406c41928..4aabaa23e 100644 --- a/examples/foundational/21-tavus-transport.py +++ b/examples/foundational/21-tavus-transport.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService @@ -64,8 +65,8 @@ async def main(): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/21a-tavus-video-service.py b/examples/foundational/21a-tavus-video-service.py index fd4c82e54..0aa2f7e17 100644 --- a/examples/foundational/21a-tavus-video-service.py +++ b/examples/foundational/21a-tavus-video-service.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -83,8 +84,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 4907aa137..98ad23c0b 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -16,8 +16,9 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.gated_openai_llm_context import GatedOpenAILLMContextAggregator -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.gated_llm_context import GatedLLMContextAggregator +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.null_filter import NullFilter from pipecat.processors.filters.wake_notifier_filter import WakeNotifierFilter from pipecat.processors.user_idle_processor import UserIdleProcessor @@ -50,8 +51,8 @@ class TurnDetectionLLM(Pipeline): }, ] - statement_context = OpenAILLMContext(statement_messages) - statement_context_aggregator = statement_llm.create_context_aggregator(statement_context) + statement_context = LLMContext(statement_messages) + statement_context_aggregator = LLMContextAggregatorPair(statement_context) # We have instructed the LLM to return 'YES' if it thinks the user # completed a sentence. So, if it's 'YES' we will return true in this @@ -72,9 +73,7 @@ class TurnDetectionLLM(Pipeline): # This processor keeps the last context and will let it through once the # notifier is woken up. We start with the gate open because we send an # initial context frame to start the conversation. - gated_context_aggregator = GatedOpenAILLMContextAggregator( - notifier=notifier, start_open=True - ) + gated_context_aggregator = GatedLLMContextAggregator(notifier=notifier, start_open=True) # Notify if the user hasn't said anything. async def user_idle_notifier(frame): @@ -147,8 +146,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm_main.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) # LLM + turn detection (with an extra LLM as a judge) llm = TurnDetectionLLM(llm_main, context_aggregator) diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index a6747f3d6..b34947c2b 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -20,7 +20,8 @@ from pipecat.frames.frames import LLMRunFrame, MixerEnableFrame, MixerUpdateSett from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -93,8 +94,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index 30bf59303..6c4804c37 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -21,7 +21,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -114,8 +115,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 36827b8f3..9479632b5 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/28-transcription-processor.py b/examples/foundational/28-transcription-processor.py index b2a9f435e..6aa3168ac 100644 --- a/examples/foundational/28-transcription-processor.py +++ b/examples/foundational/28-transcription-processor.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage, Transcripti from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -137,8 +138,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) # Create transcript processor and handler transcript = TranscriptProcessor() diff --git a/examples/foundational/29-turn-tracking-observer.py b/examples/foundational/29-turn-tracking-observer.py index 5b6d8a4d5..83faa71dc 100644 --- a/examples/foundational/29-turn-tracking-observer.py +++ b/examples/foundational/29-turn-tracking-observer.py @@ -19,7 +19,8 @@ from pipecat.observers.loggers.user_bot_latency_log_observer import UserBotLaten from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -75,8 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index b5d7e51b1..54318884e 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -29,9 +29,8 @@ from pipecat.observers.loggers.llm_log_observer import LLMLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -124,8 +123,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 9912dc818..f782e7c36 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -21,7 +21,8 @@ from pipecat.observers.base_observer import BaseObserver, FramePushed from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -115,7 +116,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tools=tools, ) - context = OpenAILLMContext( + context = LLMContext( [ { "role": "user", @@ -123,7 +124,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): } ], ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/33-gemini-rag.py b/examples/foundational/33-gemini-rag.py index b9293ae23..e2e88e390 100644 --- a/examples/foundational/33-gemini-rag.py +++ b/examples/foundational/33-gemini-rag.py @@ -55,6 +55,8 @@ from dotenv import load_dotenv from google import genai from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -63,7 +65,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -121,11 +124,7 @@ async def query_knowledge_base(params: FunctionCallParams): # for our case, the first two messages are the instructions and the user message # so we remove them. - conversation_turns = params.context.messages[2:] - # convert to standard messages - messages = [] - for turn in conversation_turns: - messages.extend(params.context.to_standard_messages(turn)) + conversation_turns = params.context.get_messages()[2:] def _is_tool_call(turn): if turn.get("role", None) == "tool": @@ -135,7 +134,7 @@ async def query_knowledge_base(params: FunctionCallParams): return False # filter out tool calls - messages = [turn for turn in messages if not _is_tool_call(turn)] + messages = [turn for turn in conversation_turns if not _is_tool_call(turn)] # use the last 3 turns as the conversation history/context messages = messages[-3:] messages_json = json.dumps(messages, ensure_ascii=False, indent=2) @@ -199,25 +198,20 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): api_key=os.getenv("GOOGLE_API_KEY"), ) llm.register_function("query_knowledge_base", query_knowledge_base) - tools = [ - { - "function_declarations": [ - { - "name": "query_knowledge_base", - "description": "Query the knowledge base for the answer to the question.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question to query the knowledge base with.", - }, - }, - }, - }, - ], + + query_function = FunctionSchema( + name="query_knowledge_base", + description="Query the knowledge base for the answer to the question.", + properties={ + "question": { + "type": "string", + "description": "The question to query the knowledge base with.", + }, }, - ] + required=["question"], + ) + tools = ToolsSchema(standard_tools=[query_function]) + system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. @@ -230,8 +224,8 @@ Your response will be turned into speech so use only simple words and punctuatio {"role": "user", "content": "Greet the user."}, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index 3f03ec25e..cd5250dfc 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -58,7 +58,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -133,8 +134,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 9b33c36ec..7ed9eb268 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -52,7 +52,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -194,8 +195,8 @@ Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue."" }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) # Create pipeline pipeline = Pipeline( diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py index cc0c9f29f..4cf20b750 100644 --- a/examples/foundational/36-user-email-gathering.py +++ b/examples/foundational/36-user-email-gathering.py @@ -9,8 +9,9 @@ import os from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,14 +20,14 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.services.rime.tts import RimeHttpTTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -90,26 +91,20 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # sent to the same callback with an additional function_name parameter. llm.register_function("store_user_emails", store_user_emails) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "store_user_emails", - "description": "Store user emails when confirmed", - "parameters": { - "type": "object", - "properties": { - "emails": { - "type": "array", - "description": "The list of user emails", - "items": {"type": "string"}, - }, - }, - "required": ["emails"], - }, + store_emails_function = FunctionSchema( + name="store_user_emails", + description="Store user emails when confirmed", + properties={ + "emails": { + "type": "array", + "description": "The list of user emails", + "items": {"type": "string"}, }, - ) - ] + }, + required=["emails"], + ) + tools = ToolsSchema(standard_tools=[store_emails_function]) + messages = [ { "role": "system", @@ -120,8 +115,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/37-mem0.py b/examples/foundational/37-mem0.py index 57be73df4..436f0a452 100644 --- a/examples/foundational/37-mem0.py +++ b/examples/foundational/37-mem0.py @@ -55,7 +55,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -244,8 +245,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set up conversation context and management # The context_aggregator will automatically collect conversation context - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) pipeline = Pipeline( diff --git a/examples/foundational/38-smart-turn-fal.py b/examples/foundational/38-smart-turn-fal.py index 6de739b04..03792c11d 100644 --- a/examples/foundational/38-smart-turn-fal.py +++ b/examples/foundational/38-smart-turn-fal.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -81,8 +82,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/38a-smart-turn-local-coreml.py b/examples/foundational/38a-smart-turn-local-coreml.py index b7bf1c14b..fc11b4137 100644 --- a/examples/foundational/38a-smart-turn-local-coreml.py +++ b/examples/foundational/38a-smart-turn-local-coreml.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -97,8 +98,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/38b-smart-turn-local.py b/examples/foundational/38b-smart-turn-local.py index 81952b2c3..a99b870ef 100644 --- a/examples/foundational/38b-smart-turn-local.py +++ b/examples/foundational/38b-smart-turn-local.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -74,8 +75,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index 2e709304d..c51de3284 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -29,7 +29,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -157,8 +158,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py index c4e5d1df7..1777f9ce5 100644 --- a/examples/foundational/39a-mcp-run-sse.py +++ b/examples/foundational/39a-mcp-run-sse.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService @@ -93,8 +94,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39b-multiple-mcp.py index 9faae1b2b..36ec13d60 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39b-multiple-mcp.py @@ -32,7 +32,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -169,8 +170,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): all_standard_tools = run_tools.standard_tools + tools.standard_tools all_tools = ToolsSchema(standard_tools=all_standard_tools) - context = OpenAILLMContext(messages, all_tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, all_tools) + context_aggregator = LLMContextAggregatorPair(context) mcp_image_processor = UrlToImageProcessor(aiohttp_session=session) pipeline = Pipeline( diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39c-mcp-run-http.py index 3365a9fe2..38234de1c 100644 --- a/examples/foundational/39c-mcp-run-http.py +++ b/examples/foundational/39c-mcp-run-http.py @@ -19,7 +19,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -97,8 +98,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/41a-text-only-webrtc.py b/examples/foundational/41a-text-only-webrtc.py index 95b14c5db..f8e5414b0 100644 --- a/examples/foundational/41a-text-only-webrtc.py +++ b/examples/foundational/41a-text-only-webrtc.py @@ -16,7 +16,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import ( ActionResult, RTVIAction, @@ -28,8 +29,7 @@ from pipecat.processors.frameworks.rtvi import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.openai import OpenAIContextAggregatorPair -from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.llm import OpenAIContextAggregatorPair, OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams load_dotenv(override=True) @@ -91,8 +91,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) action_llm_append_to_messages = create_action_llm_append_to_messages(context_aggregator) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) diff --git a/examples/foundational/41b-text-and-audio-webrtc.py b/examples/foundational/41b-text-and-audio-webrtc.py index 338987e7e..af604fc84 100644 --- a/examples/foundational/41b-text-and-audio-webrtc.py +++ b/examples/foundational/41b-text-and-audio-webrtc.py @@ -17,7 +17,8 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import ( ActionResult, RTVIAction, @@ -31,8 +32,7 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.openai import OpenAIContextAggregatorPair -from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.llm import OpenAIContextAggregatorPair, OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams load_dotenv(override=True) @@ -107,8 +107,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) action_llm_append_to_messages = create_action_llm_append_to_messages(context_aggregator) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) diff --git a/examples/foundational/42-interruption-config.py b/examples/foundational/42-interruption-config.py index 9c5fa0cfc..c3c899b89 100644 --- a/examples/foundational/42-interruption-config.py +++ b/examples/foundational/42-interruption-config.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/43a-heygen-video-service.py b/examples/foundational/43a-heygen-video-service.py index 81aac51ff..9f25a0cfb 100644 --- a/examples/foundational/43a-heygen-video-service.py +++ b/examples/foundational/43a-heygen-video-service.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -86,8 +87,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 2e718b156..080bd1ca6 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import EndTaskFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -78,8 +79,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/45-before-and-after-events.py b/examples/foundational/45-before-and-after-events.py index c84053bbd..fd7bdfa5b 100644 --- a/examples/foundational/45-before-and-after-events.py +++ b/examples/foundational/45-before-and-after-events.py @@ -18,7 +18,8 @@ from pipecat.frames.frames import DataFrame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -85,8 +86,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py new file mode 100644 index 000000000..bfb718ff2 --- /dev/null +++ b/examples/foundational/46-video-processing.py @@ -0,0 +1,175 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +import os + +import cv2 +import numpy as np +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import Frame, InputImageRawFrame, LLMRunFrame, OutputImageRawFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.daily.transport import DailyParams, DailyTransport + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_10ms_chunks=2, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_10ms_chunks=2, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +class EdgeDetectionProcessor(FrameProcessor): + def __init__(self, video_out_width, video_out_height: int): + super().__init__() + self._video_out_width = video_out_width + self._video_out_height = video_out_height + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # Send back the user's camera video with edge detection applied + if isinstance(frame, InputImageRawFrame) and frame.transport_source == "camera": + # Convert bytes to NumPy array + img = np.frombuffer(frame.image, dtype=np.uint8).reshape( + (frame.size[1], frame.size[0], 3) + ) + + # perform edge detection only on camera frames + img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR) + + # convert the size if needed + desired_size = (self._video_out_width, self._video_out_height) + if frame.size != desired_size: + resized_image = cv2.resize(img, desired_size) + out_frame = OutputImageRawFrame(resized_image.tobytes(), desired_size, frame.format) + await self.push_frame(out_frame) + else: + out_frame = OutputImageRawFrame( + image=img.tobytes(), size=frame.size, format=frame.format + ) + await self.push_frame(out_frame) + else: + await self.push_frame(frame, direction) + + +SYSTEM_INSTRUCTION = f""" +"You are Gemini Chatbot, a friendly, helpful robot. + +Your goal is to demonstrate your capabilities in a succinct way. + +Your output will be converted to audio so don't include special characters in your answers. + +Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most. +""" + + +async def run_bot(pipecat_transport): + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck + transcribe_user_audio=True, + system_instruction=SYSTEM_INSTRUCTION, + ) + + messages = [ + { + "role": "user", + "content": "Start by greeting the user warmly and introducing yourself.", + } + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # RTVI events for Pipecat client UI + rtvi = RTVIProcessor() + + pipeline = Pipeline( + [ + pipecat_transport.input(), + context_aggregator.user(), + rtvi, + llm, # LLM + EdgeDetectionProcessor( + pipecat_transport._params.video_out_width, + pipecat_transport._params.video_out_height, + ), # Sending the video back to the user + pipecat_transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + observers=[RTVIObserver(rtvi)], + ) + + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + logger.info("Pipecat client ready.") + await rtvi.set_bot_ready() + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @pipecat_transport.event_handler("on_client_connected") + async def on_client_connected(transport, participant): + logger.info("Pipecat Client connected") + if isinstance(transport, DailyTransport): + await pipecat_transport.capture_participant_video(participant["id"], framerate=30) + else: + await pipecat_transport.capture_participant_video("camera") + + @pipecat_transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info("Pipecat Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False, force_gc=True) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/quickstart/bot.py b/examples/quickstart/bot.py index ee97175a6..f353ecf6f 100644 --- a/examples/quickstart/bot.py +++ b/examples/quickstart/bot.py @@ -43,7 +43,8 @@ logger.info("Loading pipeline components...") from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -77,8 +78,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) diff --git a/pyproject.toml b/pyproject.toml index c26b7744c..d0e727ba1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-ope livekit = [ "livekit~=1.0.13", "livekit-api~=1.0.5", "tenacity>=8.2.3,<10.0.0" ] lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] -mcp = [ "mcp[cli]~=1.9.4" ] +mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] mistral = [] mlx-whisper = [ "mlx-whisper~=0.4.2" ] @@ -95,13 +95,14 @@ sambanova = [] sarvam = [ "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk~=2.23.1" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] -local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1, <2" ] +local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] remote-smart-turn = [] -silero = [ "onnxruntime>=1.20.1, <2" ] +silero = [ "onnxruntime>=1.20.1,<2" ] simli = [ "simli-ai~=0.1.10"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.0" ] speechmatics = [ "speechmatics-rt>=0.4.0" ] +strands = [ "strands-agents>=1.9.1,<2" ] tavus=[] together = [] tracing = [ "opentelemetry-sdk>=1.33.0", "opentelemetry-api>=1.33.0", "opentelemetry-instrumentation>=0.54b0" ] diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 0506136ff..1079df1ee 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -83,6 +83,7 @@ TESTS_07 = [ ("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("07m-interruptible-aws-strands.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("07n-interruptible-gemini.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), @@ -135,25 +136,6 @@ TESTS_14 = [ ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14x-function-calling-universal-context.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ( - "14y-function-calling-google-universal-context.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), - ( - "14z-function-calling-anthropic-universal-context.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), - ( - "14aa-function-calling-aws-universal-context.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), # Currently not working. # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), diff --git a/src/pipecat/observers/loggers/llm_log_observer.py b/src/pipecat/observers/loggers/llm_log_observer.py index 53a0ac484..a88f3ca2b 100644 --- a/src/pipecat/observers/loggers/llm_log_observer.py +++ b/src/pipecat/observers/loggers/llm_log_observer.py @@ -11,6 +11,7 @@ from loguru import logger from pipecat.frames.frames import ( FunctionCallInProgressFrame, FunctionCallResultFrame, + LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, @@ -79,10 +80,13 @@ class LLMLogObserver(BaseObserver): f"🧠 {arrow} {dst} LLM MESSAGES FRAME: {frame.messages} at {time_sec:.2f}s" ) # Log OpenAILLMContextFrame (input) - elif isinstance(frame, OpenAILLMContextFrame): - logger.debug( - f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {frame.context.messages} at {time_sec:.2f}s" + elif isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + messages = ( + frame.context.messages + if isinstance(frame, OpenAILLMContextFrame) + else frame.context.get_messages() ) + logger.debug(f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {messages} at {time_sec:.2f}s") # Log function call result (input) elif isinstance(frame, FunctionCallResultFrame): logger.debug( diff --git a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py index 505a0a807..22e2cc358 100644 --- a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py +++ b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py @@ -61,17 +61,29 @@ class UserBotLatencyLogObserver(BaseObserver): elif isinstance(data.frame, UserStoppedSpeakingFrame): self._user_stopped_time = time.time() elif isinstance(data.frame, (EndFrame, CancelFrame)): - if self._latencies: - avg_latency = mean(self._latencies) - min_latency = min(self._latencies) - max_latency = max(self._latencies) - logger.info( - f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING - Avg: {avg_latency:.3f}s, Min: {min_latency:.3f}s, Max: {max_latency:.3f}s" - ) + self._log_summary() elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time: latency = time.time() - self._user_stopped_time self._user_stopped_time = 0 self._latencies.append(latency) - logger.debug( - f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s" - ) + self._log_latency(latency) + + def _log_summary(self): + if not self._latencies: + return + avg_latency = mean(self._latencies) + min_latency = min(self._latencies) + max_latency = max(self._latencies) + logger.info( + f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING - Avg: {avg_latency:.3f}s, Min: {min_latency:.3f}s, Max: {max_latency:.3f}s" + ) + + def _log_latency(self, latency: float): + """Log the latency. + + Args: + latency: The latency to log. + """ + logger.debug( + f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s" + ) diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index e8c325521..9d82fcd88 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -106,13 +106,21 @@ class PipelineRunner(BaseObject): def _setup_sigint(self): """Set up signal handlers for graceful shutdown.""" - loop = asyncio.get_running_loop() - loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler()) + try: + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler()) + except NotImplementedError: + # Windows fallback + signal.signal(signal.SIGINT, lambda s, f: self._sig_handler()) def _setup_sigterm(self): """Set up signal handlers for graceful shutdown.""" - loop = asyncio.get_running_loop() - loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler()) + try: + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler()) + except NotImplementedError: + # Windows fallback + signal.signal(signal.SIGTERM, lambda s, f: self._sig_handler()) def _sig_handler(self): """Handle interrupt signals by cancelling all tasks.""" diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 57e4621ce..35d1971a5 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -777,7 +777,6 @@ class PipelineTask(BasePipelineTask): """ running = True last_frame_time = 0 - frame_buffer = deque(maxlen=10) # Store last 10 frames while running: try: @@ -785,9 +784,6 @@ class PipelineTask(BasePipelineTask): self._idle_queue.get(), timeout=self._idle_timeout_secs ) - if not isinstance(frame, InputAudioRawFrame): - frame_buffer.append(frame) - if isinstance(frame, StartFrame) or isinstance(frame, self._idle_timeout_frames): # If we find a StartFrame or one of the frames that prevents a # time out we update the time. @@ -798,7 +794,7 @@ class PipelineTask(BasePipelineTask): # valid frames. diff_time = time.time() - last_frame_time if diff_time >= self._idle_timeout_secs: - running = await self._idle_timeout_detected(frame_buffer) + running = await self._idle_timeout_detected() # Reset `last_frame_time` so we don't trigger another # immediate idle timeout if we are not cancelling. For # example, we might want to force the bot to say goodbye @@ -808,14 +804,11 @@ class PipelineTask(BasePipelineTask): self._idle_queue.task_done() except asyncio.TimeoutError: - running = await self._idle_timeout_detected(frame_buffer) + running = await self._idle_timeout_detected() - async def _idle_timeout_detected(self, last_frames: Deque[Frame]) -> bool: + async def _idle_timeout_detected(self) -> bool: """Handle idle timeout detection and optional cancellation. - Args: - last_frames: Recent frames received before timeout for debugging. - Returns: Whether the pipeline task should continue running. """ @@ -823,10 +816,7 @@ class PipelineTask(BasePipelineTask): if self._cancelled: return True - logger.warning("Idle timeout detected. Last 10 frames received:") - for i, frame in enumerate(last_frames, 1): - logger.warning(f"Frame {i}: {frame}") - + logger.warning("Idle timeout detected.") await self._call_event_handler("on_idle_timeout") if self._cancel_on_idle_timeout: logger.warning(f"Idle pipeline detected, cancelling pipeline task...") diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_llm_context.py similarity index 81% rename from src/pipecat/processors/aggregators/gated_openai_llm_context.py rename to src/pipecat/processors/aggregators/gated_llm_context.py index 56423403d..6535bffbb 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_llm_context.py @@ -4,20 +4,20 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Gated OpenAI LLM context aggregator for controlled message flow.""" +"""Gated LLM context aggregator for controlled message flow.""" -from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame +from pipecat.frames.frames import CancelFrame, EndFrame, Frame, LLMContextFrame, StartFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.sync.base_notifier import BaseNotifier -class GatedOpenAILLMContextAggregator(FrameProcessor): - """Aggregator that gates OpenAI LLM context frames until notified. +class GatedLLMContextAggregator(FrameProcessor): + """Aggregator that gates LLM context frames until notified. - This aggregator captures OpenAI LLM context frames and holds them until - a notifier signals that they can be released. This is useful for controlling - the flow of context frames based on external conditions or timing. + This aggregator captures LLM context frames and holds them until a notifier + signals that they can be released. This is useful for controlling the flow + of context frames based on external conditions or timing. """ def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs): @@ -35,7 +35,7 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): self._gate_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames, gating OpenAI LLM context frames. + """Process incoming frames, gating LLM context frames. Args: frame: The frame to process. @@ -49,7 +49,7 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): if isinstance(frame, (EndFrame, CancelFrame)): await self._stop() await self.push_frame(frame) - elif isinstance(frame, OpenAILLMContextFrame): + elif isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): if self._start_open: self._start_open = False await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/aggregators/gated_open_ai_llm_context.py b/src/pipecat/processors/aggregators/gated_open_ai_llm_context.py new file mode 100644 index 000000000..42e38928f --- /dev/null +++ b/src/pipecat/processors/aggregators/gated_open_ai_llm_context.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Gated OpenAI LLM context aggregator for controlled message flow.""" + +from pipecat.processors.aggregators.gated_llm_context import GatedLLMContextAggregator + +# Alias for backward compatibility with the previous name +GatedOpenAILLMContextAggregator = GatedLLMContextAggregator diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 276051919..378393d9d 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -229,9 +229,12 @@ class AudioBufferProcessor(FrameProcessor): # Save time of frame so we can compute silence. self._last_bot_frame_at = time.time() - if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size: + if self._buffer_size > 0 and ( + len(self._user_audio_buffer) >= self._buffer_size + or len(self._bot_audio_buffer) >= self._buffer_size + ): await self._call_on_audio_data_handler() - self._reset_recording() + self._reset_primary_audio_buffers() # Process turn recording with preprocessed data. if self._enable_turn_audio: @@ -272,9 +275,15 @@ class AudioBufferProcessor(FrameProcessor): async def _call_on_audio_data_handler(self): """Call the audio data event handlers with buffered audio.""" - if not self.has_audio() or not self._recording: + if not self._recording: return + if len(self._user_audio_buffer) == 0 and len(self._bot_audio_buffer) == 0: + return + + self._align_track_buffers() + flush_time = time.time() + # Call original handler with merged audio merged_audio = self.merge_audio_buffers() await self._call_event_handler( @@ -290,23 +299,49 @@ class AudioBufferProcessor(FrameProcessor): self._num_channels, ) + self._last_user_frame_at = flush_time + self._last_bot_frame_at = flush_time + def _buffer_has_audio(self, buffer: bytearray) -> bool: """Check if a buffer contains audio data.""" return buffer is not None and len(buffer) > 0 def _reset_recording(self): """Reset recording state and buffers.""" - self._reset_audio_buffers() + self._reset_all_audio_buffers() self._last_user_frame_at = time.time() self._last_bot_frame_at = time.time() - def _reset_audio_buffers(self): + def _reset_all_audio_buffers(self): """Reset all audio buffers to empty state.""" + self._reset_primary_audio_buffers() + self._reset_turn_audio_buffers() + + def _reset_primary_audio_buffers(self): + """Clear user and bot buffers while preserving turn buffers and timestamps.""" self._user_audio_buffer = bytearray() self._bot_audio_buffer = bytearray() + + def _reset_turn_audio_buffers(self): + """Clear user and bot turn buffers while preserving primary buffers and timestamps.""" self._user_turn_audio_buffer = bytearray() self._bot_turn_audio_buffer = bytearray() + def _align_track_buffers(self): + """Pad the shorter track with silence so both tracks stay in sync.""" + user_len = len(self._user_audio_buffer) + bot_len = len(self._bot_audio_buffer) + if user_len == bot_len: + return + + target_len = max(user_len, bot_len) + if user_len < target_len: + self._user_audio_buffer.extend(b"\x00" * (target_len - user_len)) + self._last_user_frame_at = max(self._last_user_frame_at, self._last_bot_frame_at) + if bot_len < target_len: + self._bot_audio_buffer.extend(b"\x00" * (target_len - bot_len)) + self._last_bot_frame_at = max(self._last_bot_frame_at, self._last_user_frame_at) + async def _resample_input_audio(self, frame: InputAudioRawFrame) -> bytes: """Resample audio frame to the target sample rate.""" return await self._input_resampler.resample( diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index b67e25105..97a6ce343 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -12,6 +12,7 @@ from loguru import logger from pipecat.frames.frames import ( Frame, + LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, TextFrame, @@ -64,11 +65,16 @@ class LangchainProcessor(FrameProcessor): """ await super().process_frame(frame, direction) - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): # Messages are accumulated on the context as a list of messages. # The last one by the human is the one we want to send to the LLM. logger.debug(f"Got transcription frame {frame}") - text: str = frame.context.messages[-1]["content"] + messages = ( + frame.context.messages + if isinstance(frame, OpenAILLMContextFrame) + else frame.context.get_messages() + ) + text: str = messages[-1]["content"] await self._ainvoke(text.strip()) else: diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py new file mode 100644 index 000000000..829c799ac --- /dev/null +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -0,0 +1,169 @@ +"""Strands Agent integration for Pipecat. + +This module provides integration with Strands Agents for handling conversational AI +interactions. It supports both single agent and multi-agent graphs. +""" + +from typing import Optional + +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, +) +from pipecat.metrics.metrics import LLMTokenUsage +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + +try: + from strands import Agent + from strands.multiagent.graph import Graph +except ModuleNotFoundError as e: + logger.exception("In order to use Strands Agents, you need to `pip install strands-agents`.") + raise Exception(f"Missing module: {e}") + + +class StrandsAgentsProcessor(FrameProcessor): + """Processor that integrates Strands Agents with Pipecat's frame pipeline. + + This processor takes LLM message frames, extracts the latest user message, + and processes it through either a single Strands Agent or a multi-agent Graph. + The response is streamed back as text frames with appropriate response markers. + + Supports both single agent streaming and graph-based multi-agent workflows. + """ + + def __init__( + self, + agent: Optional[Agent] = None, + graph: Optional[Graph] = None, + graph_exit_node: Optional[str] = None, + ): + """Initialize the Strands Agents processor. + + Args: + agent: The Strands Agent to use for single-agent processing. + graph: The Strands multi-agent Graph to use for graph-based processing. + graph_exit_node: The exit node name when using graph-based processing. + + Raises: + AssertionError: If neither agent nor graph is provided, or if graph is + provided without a graph_exit_node. + """ + super().__init__() + self.agent = agent + self.graph = graph + self.graph_exit_node = graph_exit_node + + assert self.agent or self.graph, "Either agent or graph must be provided" + + if self.graph: + assert self.graph_exit_node, "graph_exit_node must be provided if graph is provided" + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle LLM message frames. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + if isinstance(frame, LLMContextFrame): + messages = frame.context.get_messages() + if messages: + last_message = messages[-1] + await self._ainvoke(str(last_message["content"]).strip()) + else: + await self.push_frame(frame, direction) + + async def _ainvoke(self, text: str): + """Invoke the Strands agent with the provided text and stream results as Pipecat frames. + + Args: + text: The user input text to process through the agent or graph. + """ + logger.debug(f"Invoking Strands agent with: {text}") + ttfb_tracking = True + try: + await self.push_frame(LLMFullResponseStartFrame()) + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + if self.graph: + # Graph does not stream; await full result then emit assistant text + graph_result = await self.graph.invoke_async(text) + if ttfb_tracking: + await self.stop_ttfb_metrics() + ttfb_tracking = False + try: + node_result = graph_result.results[self.graph_exit_node] + logger.debug(f"Node result: {node_result}") + for agent_result in node_result.get_agent_results(): + # Push to TTS service + message = getattr(agent_result, "message", None) + if isinstance(message, dict) and "content" in message: + for block in message["content"]: + if isinstance(block, dict) and "text" in block: + await self.push_frame(LLMTextFrame(str(block["text"]))) + # Update usage metrics + await self._report_usage_metrics( + agent_result.metrics.accumulated_usage.get("inputTokens", 0), + agent_result.metrics.accumulated_usage.get("outputTokens", 0), + agent_result.metrics.accumulated_usage.get("totalTokens", 0), + ) + except Exception as parse_err: + logger.warning(f"Failed to extract messages from GraphResult: {parse_err}") + else: + # Agent supports streaming events via async iterator + async for event in self.agent.stream_async(text): + # Push to TTS service + if isinstance(event, dict) and "data" in event: + await self.push_frame(LLMTextFrame(str(event["data"]))) + if ttfb_tracking: + await self.stop_ttfb_metrics() + ttfb_tracking = False + + # Update usage metrics + if ( + isinstance(event, dict) + and "event" in event + and "metadata" in event["event"] + ): + if "usage" in event["event"]["metadata"]: + usage = event["event"]["metadata"]["usage"] + await self._report_usage_metrics( + usage.get("inputTokens", 0), + usage.get("outputTokens", 0), + usage.get("totalTokens", 0), + ) + except GeneratorExit: + logger.warning(f"{self} generator was closed prematurely") + except Exception as e: + logger.exception(f"{self} an unknown error occurred: {e}") + finally: + if ttfb_tracking: + await self.stop_ttfb_metrics() + ttfb_tracking = False + await self.stop_processing_metrics() + await self.push_frame(LLMFullResponseEndFrame()) + + def can_generate_metrics(self) -> bool: + """Check if this service can generate performance metrics. + + Returns: + True as this service supports metrics generation. + """ + return True + + async def _report_usage_metrics( + self, prompt_tokens: int, completion_tokens: int, total_tokens: int + ): + tokens = LLMTokenUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + await self.start_llm_usage_metrics(tokens) diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py new file mode 100644 index 000000000..291bad414 --- /dev/null +++ b/src/pipecat/services/elevenlabs/stt.py @@ -0,0 +1,339 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""ElevenLabs speech-to-text service implementation. + +This module provides integration with ElevenLabs' Speech-to-Text API for transcription +using segmented audio processing. The service uploads audio files and receives +transcription results directly. +""" + +import io +from typing import AsyncGenerator, Optional + +import aiohttp +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.stt_service import SegmentedSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + + +def language_to_elevenlabs_language(language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs language code. + + Source: + https://elevenlabs.io/docs/capabilities/speech-to-text + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding ElevenLabs language code, or None if not supported. + """ + BASE_LANGUAGES = { + Language.AF: "afr", # Afrikaans + Language.AM: "amh", # Amharic + Language.AR: "ara", # Arabic + Language.HY: "hye", # Armenian + Language.AS: "asm", # Assamese + Language.AST: "ast", # Asturian + Language.AZ: "aze", # Azerbaijani + Language.BE: "bel", # Belarusian + Language.BN: "ben", # Bengali + Language.BS: "bos", # Bosnian + Language.BG: "bul", # Bulgarian + Language.MY: "mya", # Burmese + Language.YUE: "yue", # Cantonese + Language.CA: "cat", # Catalan + Language.CEB: "ceb", # Cebuano + Language.NY: "nya", # Chichewa + Language.HR: "hrv", # Croatian + Language.CS: "ces", # Czech + Language.DA: "dan", # Danish + Language.NL: "nld", # Dutch + Language.EN: "eng", # English + Language.ET: "est", # Estonian + Language.FIL: "fil", # Filipino + Language.FI: "fin", # Finnish + Language.FR: "fra", # French + Language.FF: "ful", # Fulah + Language.GL: "glg", # Galician + Language.LG: "lug", # Ganda + Language.KA: "kat", # Georgian + Language.DE: "deu", # German + Language.EL: "ell", # Greek + Language.GU: "guj", # Gujarati + Language.HA: "hau", # Hausa + Language.HE: "heb", # Hebrew + Language.HI: "hin", # Hindi + Language.HU: "hun", # Hungarian + Language.IS: "isl", # Icelandic + Language.IG: "ibo", # Igbo + Language.ID: "ind", # Indonesian + Language.GA: "gle", # Irish + Language.IT: "ita", # Italian + Language.JA: "jpn", # Japanese + Language.JV: "jav", # Javanese + Language.KEA: "kea", # Kabuverdianu + Language.KN: "kan", # Kannada + Language.KK: "kaz", # Kazakh + Language.KM: "khm", # Khmer + Language.KO: "kor", # Korean + Language.KU: "kur", # Kurdish + Language.KY: "kir", # Kyrgyz + Language.LO: "lao", # Lao + Language.LV: "lav", # Latvian + Language.LN: "lin", # Lingala + Language.LT: "lit", # Lithuanian + Language.LUO: "luo", # Luo + Language.LB: "ltz", # Luxembourgish + Language.MK: "mkd", # Macedonian + Language.MS: "msa", # Malay + Language.ML: "mal", # Malayalam + Language.MT: "mlt", # Maltese + Language.ZH: "zho", # Mandarin Chinese + Language.MI: "mri", # Māori + Language.MR: "mar", # Marathi + Language.MN: "mon", # Mongolian + Language.NE: "nep", # Nepali + Language.NSO: "nso", # Northern Sotho + Language.NO: "nor", # Norwegian + Language.OC: "oci", # Occitan + Language.OR: "ori", # Odia + Language.PS: "pus", # Pashto + Language.FA: "fas", # Persian + Language.PL: "pol", # Polish + Language.PT: "por", # Portuguese + Language.PA: "pan", # Punjabi + Language.RO: "ron", # Romanian + Language.RU: "rus", # Russian + Language.SR: "srp", # Serbian + Language.SN: "sna", # Shona + Language.SD: "snd", # Sindhi + Language.SK: "slk", # Slovak + Language.SL: "slv", # Slovenian + Language.SO: "som", # Somali + Language.ES: "spa", # Spanish + Language.SW: "swa", # Swahili + Language.SV: "swe", # Swedish + Language.TA: "tam", # Tamil + Language.TG: "tgk", # Tajik + Language.TE: "tel", # Telugu + Language.TH: "tha", # Thai + Language.TR: "tur", # Turkish + Language.UK: "ukr", # Ukrainian + Language.UMB: "umb", # Umbundu + Language.UR: "urd", # Urdu + Language.UZ: "uzb", # Uzbek + Language.VI: "vie", # Vietnamese + Language.CY: "cym", # Welsh + Language.WO: "wol", # Wolof + Language.XH: "xho", # Xhosa + Language.ZU: "zul", # Zulu + } + + result = BASE_LANGUAGES.get(language) + + # If not found in base languages, try to find the base language from a variant + if not result: + lang_str = str(language.value) + base_code = lang_str.split("-")[0].lower() + result = base_code if base_code in BASE_LANGUAGES.values() else None + + return result + + +class ElevenLabsSTTService(SegmentedSTTService): + """Speech-to-text service using ElevenLabs' file-based API. + + This service uses ElevenLabs' Speech-to-Text API to perform transcription on audio + segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. + The service uploads audio files to ElevenLabs and receives transcription results directly. + """ + + class InputParams(BaseModel): + """Configuration parameters for ElevenLabs STT API. + + Parameters: + language: Target language for transcription. + tag_audio_events: Whether to include audio events like (laughter), (coughing), in the transcription. + """ + + language: Optional[Language] = None + tag_audio_events: bool = True + + def __init__( + self, + *, + api_key: str, + aiohttp_session: aiohttp.ClientSession, + base_url: str = "https://api.elevenlabs.io", + model: str = "scribe_v1", + sample_rate: Optional[int] = None, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the ElevenLabs STT service. + + Args: + api_key: ElevenLabs API key for authentication. + aiohttp_session: aiohttp ClientSession for HTTP requests. + base_url: Base URL for ElevenLabs API. + model: Model ID for transcription. Defaults to "scribe_v1". + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. + params: Configuration parameters for the STT service. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ + super().__init__( + sample_rate=sample_rate, + **kwargs, + ) + + params = params or ElevenLabsSTTService.InputParams() + + self._api_key = api_key + self._base_url = base_url + self._session = aiohttp_session + self._model_id = model + self._tag_audio_events = params.tag_audio_events + + self._settings = { + "language": self.language_to_service_language(params.language) + if params.language + else "eng", + } + + def can_generate_metrics(self) -> bool: + """Check if the service can generate processing metrics. + + Returns: + True, as ElevenLabs STT service supports metrics generation. + """ + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs service-specific language code. + + Args: + language: The language to convert. + + Returns: + The ElevenLabs-specific language code, or None if not supported. + """ + return language_to_elevenlabs_language(language) + + async def set_language(self, language: Language): + """Set the transcription language. + + Args: + language: The language to use for speech-to-text transcription. + """ + logger.info(f"Switching STT language to: [{language}]") + self._settings["language"] = self.language_to_service_language(language) + + async def set_model(self, model: str): + """Set the STT model. + + Args: + model: The model name to use for transcription. + + Note: + ElevenLabs STT API does not currently support model selection. + This method is provided for interface compatibility. + """ + await super().set_model(model) + logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model") + + async def _transcribe_audio(self, audio_data: bytes) -> dict: + """Upload audio data to ElevenLabs and get transcription result. + + Args: + audio_data: Raw audio bytes in WAV format. + + Returns: + The transcription result data. + + Raises: + Exception: If transcription fails or returns an error. + """ + url = f"{self._base_url}/v1/speech-to-text" + headers = {"xi-api-key": self._api_key} + + # Create form data with the audio file + data = aiohttp.FormData() + data.add_field( + "file", + io.BytesIO(audio_data), + filename="audio.wav", + content_type="audio/x-wav", + ) + + # Add required model_id, language_code, and tag_audio_events + data.add_field("model_id", self._model_id) + data.add_field("language_code", self._settings["language"]) + data.add_field("tag_audio_events", str(self._tag_audio_events).lower()) + + async with self._session.post(url, data=data, headers=headers) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"ElevenLabs transcription error: {error_text}") + raise Exception(f"Transcription failed with status {response.status}: {error_text}") + + result = await response.json() + return result + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[str] = None + ): + """Handle a transcription result with tracing.""" + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe an audio segment using ElevenLabs' STT API. + + Args: + audio: Raw audio bytes in WAV format (already converted by base class). + + Yields: + Frame: TranscriptionFrame containing the transcribed text, or ErrorFrame on failure. + + Note: + The audio is already in WAV format from the SegmentedSTTService. + Only non-empty transcriptions are yielded. + """ + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + # Upload audio and get transcription result directly + result = await self._transcribe_audio(audio) + + # Extract transcription text + text = result.get("text", "").strip() + if text: + # Use the language_code returned by the API + detected_language = result.get("language_code", "eng") + + await self._handle_transcription(text, True, detected_language) + logger.debug(f"Transcription: [{text}]") + + yield TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + detected_language, + result=result, + ) + + except Exception as e: + logger.error(f"ElevenLabs STT error: {e}") + yield ErrorFrame(f"ElevenLabs STT error: {str(e)}") diff --git a/src/pipecat/services/mem0/memory.py b/src/pipecat/services/mem0/memory.py index b134f5162..a80398728 100644 --- a/src/pipecat/services/mem0/memory.py +++ b/src/pipecat/services/mem0/memory.py @@ -16,7 +16,8 @@ from typing import Any, Dict, List, Optional from loguru import logger from pydantic import BaseModel, Field -from pipecat.frames.frames import ErrorFrame, Frame, LLMMessagesFrame +from pipecat.frames.frames import ErrorFrame, Frame, LLMContextFrame, LLMMessagesFrame +from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, @@ -180,11 +181,11 @@ class Mem0MemoryService(FrameProcessor): logger.error(f"Error retrieving memories from Mem0: {e}") return [] - def _enhance_context_with_memories(self, context: OpenAILLMContext, query: str): + def _enhance_context_with_memories(self, context: LLMContext | OpenAILLMContext, query: str): """Enhance the LLM context with relevant memories. Args: - context: The OpenAILLMContext to enhance with memory information. + context: The LLM context to enhance with memory information. query: The query to search for relevant memories. """ # Skip if this is the same query we just processed @@ -222,11 +223,11 @@ class Mem0MemoryService(FrameProcessor): context = None messages = None - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): context = frame.context elif isinstance(frame, LLMMessagesFrame): messages = frame.messages - context = OpenAILLMContext.from_messages(messages) + context = LLMContext(messages) if context: try: diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 6f6376c02..47127b77a 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -281,8 +281,10 @@ class BaseOpenAILLMService(LLMService): # base64 encode any images for message in messages: if message.get("mime_type") == "image/jpeg": - encoded_image = base64.b64encode(message["data"].getvalue()).decode("utf-8") - text = message["content"] + # Avoid .getvalue() which makes a full copy of BytesIO + raw_bytes = message["data"].read() + encoded_image = base64.b64encode(raw_bytes).decode("utf-8") + text = message.get("content", "") message["content"] = [ {"type": "text", "text": text}, { @@ -290,6 +292,7 @@ class BaseOpenAILLMService(LLMService): "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}, }, ] + # Explicit cleanup del message["data"] del message["mime_type"] diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index 182a89321..5ee37bf68 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -68,6 +68,9 @@ class Language(StrEnum): AS = "as" AS_IN = "as-IN" + # Asturian + AST = "ast" + # Azerbaijani AZ = "az" AZ_AZ = "az-AZ" @@ -101,6 +104,9 @@ class Language(StrEnum): CA = "ca" CA_ES = "ca-ES" + # Cebuano + CEB = "ceb" + # Mandarin Chinese CMN = "cmn" CMN_CN = "cmn-CN" @@ -185,6 +191,9 @@ class Language(StrEnum): FA = "fa" FA_IR = "fa-IR" + # Fulah + FF = "ff" + # Finnish FI = "fi" FI_FI = "fi-FI" @@ -251,6 +260,9 @@ class Language(StrEnum): ID = "id" ID_ID = "id-ID" + # Igbo + IG = "ig" + # Icelandic IS = "is" IS_IS = "is-IS" @@ -279,6 +291,9 @@ class Language(StrEnum): KA = "ka" KA_GE = "ka-GE" + # Kabuverdianu + KEA = "kea" + # Kazakh KK = "kk" KK_KZ = "kk-KZ" @@ -295,6 +310,13 @@ class Language(StrEnum): KO = "ko" KO_KR = "ko-KR" + # Kurdish + KU = "ku" + + # Kyrgyz + KY = "ky" + KY_KG = "ky-KG" + # Latin LA = "la" @@ -312,6 +334,12 @@ class Language(StrEnum): LT = "lt" LT_LT = "lt-LT" + # Ganda + LG = "lg" + + # Luo + LUO = "luo" + # Latvian LV = "lv" LV_LV = "lv-LV" @@ -366,6 +394,12 @@ class Language(StrEnum): NL_BE = "nl-BE" NL_NL = "nl-NL" + # Northern Sotho + NSO = "nso" + + # Chichewa + NY = "ny" + # Occitan OC = "oc" @@ -484,6 +518,9 @@ class Language(StrEnum): UK = "uk" UK_UA = "uk-UA" + # Umbundu + UMB = "umb" + # Urdu UR = "ur" UR_IN = "ur-IN" @@ -497,6 +534,9 @@ class Language(StrEnum): VI = "vi" VI_VN = "vi-VN" + # Wolof + WO = "wo" + # Wu Chinese WUU = "wuu" WUU_CN = "wuu-CN" @@ -507,7 +547,7 @@ class Language(StrEnum): # Yoruba YO = "yo" - # Yue Chinese + # Yue Chinese (Cantonese) YUE = "yue" YUE_CN = "yue-CN" diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 0316df194..9d1b2b3bb 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -202,21 +202,27 @@ class BaseOutputTransport(FrameProcessor): """ pass - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the transport. Args: frame: The output video frame to write. - """ - pass - async def write_audio_frame(self, frame: OutputAudioRawFrame): + Returns: + True if the video frame was written successfully, False otherwise. + """ + return False + + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the transport. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ - pass + return False async def write_dtmf(self, frame: OutputDTMFFrame | OutputDTMFUrgentFrame): """Write a DTMF tone using the transport's preferred method. @@ -659,6 +665,7 @@ class BaseOutputTransport(FrameProcessor): self._audio_queue.get(), timeout=vad_stop_secs ) yield frame + self._audio_queue.task_done() except asyncio.TimeoutError: # Notify the bot stopped speaking upstream if necessary. await self._bot_stopped_speaking() @@ -673,6 +680,7 @@ class BaseOutputTransport(FrameProcessor): frame.audio = await self._mixer.mix(frame.audio) last_frame_time = time.time() yield frame + self._audio_queue.task_done() except asyncio.QueueEmpty: # Notify the bot stopped speaking upstream if necessary. diff_time = time.time() - last_frame_time @@ -738,12 +746,22 @@ class BaseOutputTransport(FrameProcessor): # Handle frame. await self._handle_frame(frame) - # Also, push frame downstream in case anyone else needs it. - await self._transport.push_frame(frame) + # If we are not able to write to the transport we shouldn't + # pushb downstream. + push_downstream = True - # Send audio. - if isinstance(frame, OutputAudioRawFrame): - await self._transport.write_audio_frame(frame) + # Try to send audio to the transport. + try: + if isinstance(frame, OutputAudioRawFrame): + push_downstream = await self._transport.write_audio_frame(frame) + except Exception as e: + logger.error(f"{self} Error writing {frame} to transport: {e}") + push_downstream = False + + # If we were able to send to the transport, push the frame + # downstream in case anyone else needs it. + if push_downstream: + await self._transport.push_frame(frame) # # Video handling diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index bf0036950..edb25531f 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -506,11 +506,14 @@ class DailyTransportClient(EventHandler): self._custom_audio_tracks[destination] = await self.add_custom_audio_track(destination) self._client.update_publishing({"customAudio": {destination: True}}) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the appropriate audio track. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ future = self._get_event_loop().create_future() @@ -526,18 +529,24 @@ class DailyTransportClient(EventHandler): audio_source.write_frames(frame.audio, completion=completion_callback(future)) else: logger.warning(f"{self} unable to write audio frames to destination [{destination}]") - future.set_result(None) + future.set_result(0) - await future + num_frames = await future + return num_frames > 0 - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the camera device. Args: frame: The image frame to write. + + Returns: + True if the video frame was written successfully, False otherwise. """ if not frame.transport_destination and self._camera: self._camera.write_frame(frame.image) + return True + return False async def setup(self, setup: FrameProcessorSetup): """Setup the client with task manager and event queues. @@ -1835,24 +1844,33 @@ class DailyOutputTransport(BaseOutputTransport): Args: destination: The destination identifier to register. + + Returns: + True if the audio frame was written successfully, False otherwise. """ await self._client.register_audio_destination(destination) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the Daily call. Args: frame: The audio frame to write. - """ - await self._client.write_audio_frame(frame) - async def write_video_frame(self, frame: OutputImageRawFrame): + Returns: + True if the audio frame was written successfully, False otherwise. + """ + return await self._client.write_audio_frame(frame) + + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the Daily call. Args: frame: The video frame to write. + + Returns: + True if the video frame was written successfully, False otherwise. """ - await self._client.write_video_frame(frame) + return await self._client.write_video_frame(frame) def _supports_native_dtmf(self) -> bool: """Daily supports native DTMF via telephone events. @@ -1974,9 +1992,6 @@ class DailyTransport(BaseTransport): self._register_event_handler("on_recording_stopped") self._register_event_handler("on_recording_error") self._register_event_handler("on_before_leave", sync=True) - # Deprecated - self._register_event_handler("on_joined") - self._register_event_handler("on_left") # # BaseTransport diff --git a/src/pipecat/transports/livekit/transport.py b/src/pipecat/transports/livekit/transport.py index 227a9f4f9..334d48ecb 100644 --- a/src/pipecat/transports/livekit/transport.py +++ b/src/pipecat/transports/livekit/transport.py @@ -329,19 +329,21 @@ class LiveKitTransportClient: except Exception as e: logger.error(f"Error sending DTMF tone {digit}: {e}") - async def publish_audio(self, audio_frame: rtc.AudioFrame): + async def publish_audio(self, audio_frame: rtc.AudioFrame) -> bool: """Publish an audio frame to the room. Args: audio_frame: The LiveKit audio frame to publish. """ if not self._connected or not self._audio_source: - return + return False try: await self._audio_source.capture_frame(audio_frame) + return True except Exception as e: logger.error(f"Error publishing audio: {e}") + return False def get_participants(self) -> List[str]: """Get list of participant IDs in the room. @@ -849,14 +851,17 @@ class LiveKitOutputTransport(BaseOutputTransport): else: await self._client.send_data(message.encode()) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the LiveKit room. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ livekit_audio = self._convert_pipecat_audio_to_livekit(frame.audio) - await self._client.publish_audio(livekit_audio) + return await self._client.publish_audio(livekit_audio) def _supports_native_dtmf(self) -> bool: """LiveKit supports native DTMF via telephone events. diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index b38d1dd8e..7f809b5d4 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -172,16 +172,21 @@ class LocalAudioOutputTransport(BaseOutputTransport): self._out_stream.close() self._out_stream = None - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the output stream. Args: frame: The audio frame to write to the output device. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._out_stream: await self.get_event_loop().run_in_executor( self._executor, self._out_stream.write, frame.audio ) + return True + return False class LocalAudioTransport(BaseTransport): diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index c687370ce..6cceb2cf1 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -191,24 +191,33 @@ class TkOutputTransport(BaseOutputTransport): self._out_stream.close() self._out_stream = None - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the output stream. Args: frame: The audio frame to write to the output device. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._out_stream: await self.get_event_loop().run_in_executor( self._executor, self._out_stream.write, frame.audio ) + return True + return False - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the Tkinter display. Args: frame: The video frame to display in the Tkinter window. + + Returns: + True if the video frame was written successfully, False otherwise. """ self.get_event_loop().call_soon(self._write_frame_to_tk, frame) + return True def _write_frame_to_tk(self, frame: OutputImageRawFrame): """Write frame data to the Tkinter image label.""" diff --git a/src/pipecat/transports/smallwebrtc/connection.py b/src/pipecat/transports/smallwebrtc/connection.py index decd8ca58..45b1cffb2 100644 --- a/src/pipecat/transports/smallwebrtc/connection.py +++ b/src/pipecat/transports/smallwebrtc/connection.py @@ -206,11 +206,16 @@ class SmallWebRTCConnection(BaseObject): for real-time audio/video communication. """ - def __init__(self, ice_servers: Optional[Union[List[str], List[IceServer]]] = None): + def __init__( + self, + ice_servers: Optional[Union[List[str], List[IceServer]]] = None, + connection_timeout_secs: int = 60, + ): """Initialize the WebRTC connection. Args: ice_servers: List of ICE servers as URLs or IceServer objects. + connection_timeout_secs: Timeout in seconds for connecting to the peer. Raises: TypeError: If ice_servers contains mixed types or unsupported types. @@ -231,6 +236,7 @@ class SmallWebRTCConnection(BaseObject): VIDEO_TRANSCEIVER_INDEX: self.video_input_track, SCREEN_VIDEO_TRANSCEIVER_INDEX: self.screen_video_input_track, } + self.connection_timeout_secs = connection_timeout_secs self._initialize() @@ -279,6 +285,7 @@ class SmallWebRTCConnection(BaseObject): self._last_received_time = None self._message_queue = [] self._pending_app_messages = [] + self._connecting_timeout_task = None def _setup_listeners(self): """Set up event listeners for the peer connection.""" @@ -499,6 +506,7 @@ class SmallWebRTCConnection(BaseObject): self._message_queue.clear() self._pending_app_messages.clear() self._track_map = {} + self._cancel_monitoring_connecting_state() def get_answer(self): """Get the SDP answer for the current connection. @@ -516,9 +524,45 @@ class SmallWebRTCConnection(BaseObject): "pc_id": self._pc_id, } + def _monitoring_connecting_state(self) -> None: + """Start monitoring the peer connection while it is in the *connecting* state. + + This method schedules a timeout task that will automatically close the + connection if it remains in the connecting state for more than the specified + timeout, default to 60 seconds. + """ + logger.debug("Monitoring connecting state") + + async def timeout_handler(): + # We will close the connection in case we have remained in the connecting state for over 1 minute + await asyncio.sleep(self.connection_timeout_secs) + logger.warning("Timeout establishing the connection to the remote peer. Closing.") + + await self._close() + + # Create and store the timeout task + self._connecting_timeout_task = asyncio.create_task(timeout_handler()) + + def _cancel_monitoring_connecting_state(self) -> None: + """Cancel the ongoing connecting-state timeout task, if any. + + This method should be called once the connection has either succeeded or + transitioned out of the connecting state. If the timeout task is still + pending, it will be canceled and the reference cleared. + """ + if self._connecting_timeout_task and not self._connecting_timeout_task.done(): + logger.debug("Cancelling the connecting timeout task") + self._connecting_timeout_task.cancel() + self._connecting_timeout_task = None + async def _handle_new_connection_state(self): """Handle changes in the peer connection state.""" state = self._pc.connectionState + if state == "connecting": + self._monitoring_connecting_state() + else: + self._cancel_monitoring_connecting_state() + if state == "connected" and not self._connect_invoked: # We are going to wait until the pipeline is ready before triggering the event return diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index 2abe2abf5..4a9ba9341 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -309,7 +309,7 @@ class SmallWebRTCClient: # self._webrtc_connection.ask_to_renegotiate() frame = None except MediaStreamError: - logger.warning("Received an unexpected media stream error while reading the audio.") + logger.warning("Received an unexpected media stream error while reading the video.") frame = None if frame is None or not isinstance(frame, VideoFrame): @@ -321,15 +321,21 @@ class SmallWebRTCClient: # Convert frame to NumPy array in its native format frame_array = frame.to_ndarray(format=format_name) frame_rgb = self._convert_frame(frame_array, format_name) + del frame_array # free intermediate array immediately + image_bytes = frame_rgb.tobytes() + del frame_rgb # free RGB array immediately image_frame = UserImageRawFrame( user_id=self._webrtc_connection.pc_id, - image=frame_rgb.tobytes(), + image=image_bytes, size=(frame.width, frame.height), format="RGB", ) image_frame.transport_source = video_source + del frame # free original VideoFrame + del image_bytes # reference kept in image_frame + yield image_frame async def read_audio_frame(self): @@ -364,40 +370,62 @@ class SmallWebRTCClient: resampled_frames = self._pipecat_resampler.resample(frame) for resampled_frame in resampled_frames: # 16-bit PCM bytes - pcm_bytes = resampled_frame.to_ndarray().astype(np.int16).tobytes() + pcm_array = resampled_frame.to_ndarray().astype(np.int16) + pcm_bytes = pcm_array.tobytes() + del pcm_array # free NumPy array immediately + audio_frame = InputAudioRawFrame( audio=pcm_bytes, sample_rate=resampled_frame.sample_rate, num_channels=self._audio_in_channels, ) + del pcm_bytes # reference kept in audio_frame + yield audio_frame else: # 16-bit PCM bytes - pcm_bytes = frame.to_ndarray().astype(np.int16).tobytes() + pcm_array = frame.to_ndarray().astype(np.int16) + pcm_bytes = pcm_array.tobytes() + del pcm_array # free NumPy array immediately + audio_frame = InputAudioRawFrame( audio=pcm_bytes, sample_rate=frame.sample_rate, num_channels=self._audio_in_channels, ) + del pcm_bytes # reference kept in audio_frame + yield audio_frame - async def write_audio_frame(self, frame: OutputAudioRawFrame): + del frame # free original AudioFrame + + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebRTC connection. Args: frame: The audio frame to transmit. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._can_send() and self._audio_output_track: await self._audio_output_track.add_audio_bytes(frame.audio) + return True + return False - async def write_video_frame(self, frame: OutputImageRawFrame): + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the WebRTC connection. Args: frame: The video frame to transmit. + + Returns: + True if the video frame was written successfully, False otherwise. """ if self._can_send() and self._video_output_track: self._video_output_track.add_video_frame(frame) + return True + return False async def setup(self, _params: TransportParams, frame): """Set up the client with transport parameters. @@ -800,21 +828,27 @@ class SmallWebRTCOutputTransport(BaseOutputTransport): """ await self._client.send_message(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebRTC connection. Args: frame: The output audio frame to transmit. - """ - await self._client.write_audio_frame(frame) - async def write_video_frame(self, frame: OutputImageRawFrame): + Returns: + True if the audio frame was written successfully, False otherwise. + """ + return await self._client.write_audio_frame(frame) + + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: """Write a video frame to the WebRTC connection. Args: frame: The output video frame to transmit. + + Returns: + True if the video frame was written successfully, False otherwise. """ - await self._client.write_video_frame(frame) + return await self._client.write_video_frame(frame) class SmallWebRTCTransport(BaseTransport): diff --git a/src/pipecat/transports/tavus/transport.py b/src/pipecat/transports/tavus/transport.py index 23e14f97b..b199de09c 100644 --- a/src/pipecat/transports/tavus/transport.py +++ b/src/pipecat/transports/tavus/transport.py @@ -395,15 +395,18 @@ class TavusTransportClient: participant_settings=participant_settings, profile_settings=profile_settings ) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the transport. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if not self._client: - return - await self._client.write_audio_frame(frame) + return False + return await self._client.write_audio_frame(frame) async def register_audio_destination(self, destination: str): """Register an audio destination for output. @@ -625,15 +628,18 @@ class TavusOutputTransport(BaseOutputTransport): """Handle interruption events by sending interrupt message.""" await self._client.send_interrupt_message() - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the Tavus transport. Args: frame: The audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ # This is the custom track destination expected by Tavus frame.transport_destination = self._transport_destination - await self._client.write_audio_frame(frame) + return await self._client.write_audio_frame(frame) async def register_audio_destination(self, destination: str): """Register an audio destination. diff --git a/src/pipecat/transports/websocket/client.py b/src/pipecat/transports/websocket/client.py index d141b52f3..c18c0d22a 100644 --- a/src/pipecat/transports/websocket/client.py +++ b/src/pipecat/transports/websocket/client.py @@ -150,17 +150,39 @@ class WebsocketClientSession: await self._websocket.close() self._websocket = None - async def send(self, message: websockets.Data): + async def send(self, message: websockets.Data) -> bool: """Send a message through the WebSocket connection. Args: message: The message data to send. """ + result = False try: if self._websocket: await self._websocket.send(message) + result = True except Exception as e: logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})") + finally: + return result + + @property + def is_connected(self) -> bool: + """Check if the WebSocket is currently connected. + + Returns: + True if the WebSocket is in connected state. + """ + return self._websocket.state == websockets.State.OPEN if self._websocket else False + + @property + def is_closing(self) -> bool: + """Check if the WebSocket is currently closing. + + Returns: + True if the WebSocket is in the process of closing. + """ + return self._websocket.state == websockets.State.CLOSING if self._websocket else False async def _client_task_handler(self): """Handle incoming messages from the WebSocket connection.""" @@ -371,12 +393,18 @@ class WebsocketClientOutputTransport(BaseOutputTransport): """ await self._write_frame(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebSocket with optional WAV header. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ + if self._session.is_closing or not self._session.is_connected: + return False + frame = OutputAudioRawFrame( audio=frame.audio, sample_rate=self.sample_rate, @@ -402,10 +430,16 @@ class WebsocketClientOutputTransport(BaseOutputTransport): # Simulate audio playback with a sleep. await self._write_audio_sleep() + return True + async def _write_frame(self, frame: Frame): """Write a frame to the WebSocket after serialization.""" + if self._session.is_closing or not self._session.is_connected: + return + if not self._params.serializer: return + payload = await self._params.serializer.serialize(frame) if payload: await self._session.send(payload) diff --git a/src/pipecat/transports/websocket/fastapi.py b/src/pipecat/transports/websocket/fastapi.py index cfa68f5cb..72654204a 100644 --- a/src/pipecat/transports/websocket/fastapi.py +++ b/src/pipecat/transports/websocket/fastapi.py @@ -410,14 +410,17 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): """ await self._write_frame(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebSocket with timing simulation. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if self._client.is_closing or not self._client.is_connected: - return + return False frame = OutputAudioRawFrame( audio=frame.audio, @@ -444,6 +447,8 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): # Simulate audio playback with a sleep. await self._write_audio_sleep() + return True + async def _write_frame(self, frame: Frame): """Serialize and send a frame through the WebSocket.""" if self._client.is_closing or not self._client.is_connected: diff --git a/src/pipecat/transports/websocket/server.py b/src/pipecat/transports/websocket/server.py index 67631ab04..095407422 100644 --- a/src/pipecat/transports/websocket/server.py +++ b/src/pipecat/transports/websocket/server.py @@ -346,14 +346,17 @@ class WebsocketServerOutputTransport(BaseOutputTransport): """ await self._write_frame(frame) - async def write_audio_frame(self, frame: OutputAudioRawFrame): + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the WebSocket client with timing control. Args: frame: The output audio frame to write. + + Returns: + True if the audio frame was written successfully, False otherwise. """ if not self._websocket: - return + return False frame = OutputAudioRawFrame( audio=frame.audio, @@ -380,6 +383,8 @@ class WebsocketServerOutputTransport(BaseOutputTransport): # Simulate audio playback with a sleep. await self._write_audio_sleep() + return True + async def _write_frame(self, frame: Frame): """Serialize and send a frame to the WebSocket client.""" if not self._params.serializer: diff --git a/src/pipecat/utils/base_object.py b/src/pipecat/utils/base_object.py index a1c75f733..f0744f3b8 100644 --- a/src/pipecat/utils/base_object.py +++ b/src/pipecat/utils/base_object.py @@ -139,7 +139,7 @@ class BaseObject(ABC): name=event_name, handlers=[], is_sync=sync ) else: - logger.warning(f"Event handler {event_name} not registered") + logger.warning(f"Event handler {event_name} already registered") async def _call_event_handler(self, event_name: str, *args, **kwargs): """Call all registered handlers for the specified event. diff --git a/tests/test_audio_buffer_processor.py b/tests/test_audio_buffer_processor.py new file mode 100644 index 000000000..f35c5adf0 --- /dev/null +++ b/tests/test_audio_buffer_processor.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import struct +import unittest + +from pipecat.frames.frames import InputAudioRawFrame, OutputAudioRawFrame, StartFrame +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor + + +class _PassthroughResampler: + async def resample( + self, audio: bytes, in_rate: int, out_rate: int + ) -> bytes: # pragma: no cover - trivial + return audio + + +class TestAudioBufferProcessor(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.processor = AudioBufferProcessor(sample_rate=16000, num_channels=2, buffer_size=4) + self.processor._input_resampler = _PassthroughResampler() + self.processor._output_resampler = _PassthroughResampler() + self.processor._update_sample_rate(StartFrame(audio_out_sample_rate=16000)) + await self.processor.start_recording() + + async def asyncTearDown(self): + if getattr(self.processor, "_recording", False): + await self.processor.stop_recording() + await self.processor.cleanup() + + async def test_flush_user_audio_pads_bot_track(self): + user_audio = struct.pack("