diff --git a/README.md b/README.md index e5f4b270a..ea25d492c 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/pipecat-ai/pipecat) +[![](https://getmanta.ai/api/badges?text=Manta%20Graph&link=manta)](https://getmanta.ai/pipecat) # πŸŽ™οΈ Pipecat: Real-Time Voice & Multimodal AI Agents @@ -43,6 +44,10 @@ Looking to build structured conversations? Check out [Pipecat Flows](https://git Want to build beautiful and engaging experiences? Checkout the [Voice UI Kit](https://github.com/pipecat-ai/voice-ui-kit), a collection of components, hooks and templates for building voice AI applications quickly. +### πŸ› οΈ Create and deploy projects + +Create a new project in under a minute with the [Pipecat CLI](https://github.com/pipecat-ai/pipecat-cli). Then use the CLI to monitor and deploy your agent to production. + ### πŸ” Debugging Looking for help debugging your pipeline and processors? Check out [Whisker](https://github.com/pipecat-ai/whisker), a real-time Pipecat debugger. @@ -51,6 +56,10 @@ Looking for help debugging your pipeline and processors? Check out [Whisker](htt Love terminal applications? Check out [Tail](https://github.com/pipecat-ai/tail), a terminal dashboard for Pipecat. +### πŸ“ΊοΈ Pipecat TV Channel + +Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.youtube.com/playlist?list=PLzU2zoMTQIHjqC3v4q2XVSR3hGSzwKFwH) channel. + ## 🎬 See it in action

@@ -58,24 +67,24 @@ Love terminal applications? Check out [Tail](https://github.com/pipecat-ai/tail)
  - +

## 🧩 Available services -| Category | Services | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | +| Category | Services | +| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | -| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | -| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) | -| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | -| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | -| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | -| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | +| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | +| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) | +| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | +| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | +| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | πŸ“š [View full services documentation β†’](https://docs.pipecat.ai/server/services/supported-services) diff --git a/docs/api/conf.py b/docs/api/conf.py index a3062b612..8946a34d3 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -50,6 +50,7 @@ autodoc_mock_imports = [ # Krisp - has build issues on some platforms "pipecat_ai_krisp", "krisp", + "krisp_audio", # System-specific GUI libraries "_tkinter", "tkinter", diff --git a/env.example b/env.example index 9c9ee5d8c..2865772ea 100644 --- a/env.example +++ b/env.example @@ -4,6 +4,9 @@ AICOUSTICS_LICENSE_KEY=... # Anthropic ANTHROPIC_API_KEY=... +# Assembly AI +ASSEMBLYAI_API_KEY=... + # Async ASYNCAI_API_KEY=... ASYNCAI_VOICE_ID=... @@ -21,12 +24,19 @@ AZURE_CHATGPT_API_KEY=... AZURE_CHATGPT_ENDPOINT=https://... AZURE_CHATGPT_MODEL=... +AZURE_REALTIME_API_KEY=... +AZURE_REALTIME_BASE_URL=... + AZURE_DALLE_API_KEY=... AZURE_DALLE_ENDPOINT=https://... AZURE_DALLE_MODEL=... # Cartesia CARTESIA_API_KEY=... +CARTESIA_VOICE_ID=... + +# Cerebras +CEREBRAS_API_KEY=... # Daily DAILY_API_KEY=... @@ -35,42 +45,75 @@ DAILY_SAMPLE_ROOM_URL=https://... # Deepgram DEEPGRAM_API_KEY=... +# DeepSeek +DEEPSEEK_API_KEY=... + # ElevenLabs ELEVENLABS_API_KEY=... ELEVENLABS_VOICE_ID=... -# Neuphonic -NEUPHONIC_API_KEY=... - # Fal FAL_KEY=... # Fireworks FIREWORKS_API_KEY=... +# Fish Audio +FISH_API_KEY=... + # Gladia GLADIA_API_KEY=... GLADIA_REGION=... # Google GOOGLE_API_KEY=... -GOOGLE_CLOUD_PROJECT_ID=... -GOOGLE_TEST_CREDENTIALS=... GOOGLE_VERTEX_TEST_CREDENTIALS=... +GOOGLE_CLOUD_PROJECT_ID=... +GOOGLE_CLOUD_LOCATION=... +GOOGLE_TEST_CREDENTIALS=... + +# Grok +GROK_API_KEY=... + +# Groq +GROQ_API_KEY=... + +# Heygen +HEYGEN_API_KEY=... # Hume HUME_API_KEY=... +HUME_VOICE_ID=... + +# Inworld +INWORLD_API_KEY=... + +# Krisp +KRISP_MODEL_PATH=... + +# Krisp Viva +KRISP_VIVA_MODEL_PATH=... + +# LiveKit +LIVEKIT_API_KEY=... +LIVEKIT_API_SECRET=... # LMNT LMNT_API_KEY=... LMNT_VOICE_ID=... -# Perplexity -PERPLEXITY_API_KEY=... +# MiniMax +MINIMAX_API_KEY=... +MINIMAX_GROUP_ID=... -# PlayHT -PLAYHT_USER_ID=... -PLAYHT_API_KEY=... +# Mistral +MISTRAL_API_KEY=... + +# Neuphonic +NEUPHONIC_API_KEY=... + +# NVIDIA +NVIDIA_API_KEY=... # OpenAI OPENAI_API_KEY=... @@ -78,83 +121,73 @@ OPENAI_API_KEY=... # OpenPipe OPENPIPE_API_KEY=... -# Tavus -TAVUS_API_KEY=... -TAVUS_REPLICA_ID=... -TAVUS_PERSONA_ID=... +# OpenRouter +OPENROUTER_API_KEY=... + +# Perplexity +PERPLEXITY_API_KEY=... + +# Picovoice Koala +KOALA_ACCESS_KEY=... + +# Piper +PIPER_BASE_URL=... + +# PlayHT +PLAYHT_USER_ID=... +PLAYHT_API_KEY=... + +# Plivo +PLIVO_AUTH_ID=... +PLIVO_AUTH_TOKEN=... + +# Qwen +QWEN_API_KEY=... + +# Rime +RIME_API_KEY=... +RIME_VOICE_ID=... + +# SambaNova +SAMBANOVA_API_KEY=... + +# Sarvam AI +SARVAM_API_KEY=... + +# Sentry +SENTRY_DSN=... # Simli SIMLI_API_KEY=... SIMLI_FACE_ID=... -# Krisp -KRISP_MODEL_PATH=... - -# DeepSeek -DEEPSEEK_API_KEY=... - -# Groq -GROQ_API_KEY=... - -# Grok -GROK_API_KEY=... - -# Inworld -INWORLD_API_KEY=... - -# Together.ai -TOGETHER_API_KEY=... - -# Cerebras -CEREBRAS_API_KEY=... - -# Fish Audio -FISH_API_KEY=... - -# Assembly AI -ASSEMBLYAI_API_KEY=... - -# OpenRouter -OPENROUTER_API_KEY=... - -# Piper -PIPER_BASE_URL=... - # Smart turn LOCAL_SMART_TURN_MODEL_PATH=... FAL_SMART_TURN_API_KEY=... +# Soniox +SONIOX_API_KEY=... + +# Speechmatics +SPEECHMATICS_API_KEY=... + +# Tavus +TAVUS_API_KEY=... +TAVUS_REPLICA_ID=... + +# Telnyx +TELNYX_API_KEY=... +TELNYX_ACCOUNT_SID=... + +# Together.ai +TOGETHER_API_KEY=... + # Twilio TWILIO_ACCOUNT_SID=... TWILIO_AUTH_TOKEN=... -# MiniMax -MINIMAX_API_KEY=... -MINIMAX_GROUP_ID=... - -# Sarvam AI -SARVAM_API_KEY=... - -# Soniox -SONIOX_API_KEY= - -# Speechmatics -SPEECHMATICS_API_KEY=... - -# SambaNova -SAMBANOVA_API_KEY=... - -# Sentry -SENTRY_DSN=... - -# Heygen -HEYGEN_API_KEY=... - -# Mistral -MISTRAL_API_KEY=... - -# NVIDIA -NVIDIA_API_KEY=... - -# Qwen -QWEN_API_KEY=... +# WhatsApp +WHATSAPP_TOKEN=... +WHATSAPP_WEBHOOK_VERIFICATION_TOKEN=... +WHATSAPP_PHONE_NUMBER_ID=... +WHATSAPP_APP_SECRET=... \ No newline at end of file diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 81ba692c7..1e7bd5718 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -21,8 +21,8 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.stt import CartesiaSTTService from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -58,7 +58,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index cd922ddc2..8a144dab3 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -23,7 +23,6 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.stt import ElevenLabsSTTService from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService from pipecat.services.openai.llm import OpenAILLMService diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index 9343797a9..2d3bb1dac 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -67,8 +67,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = AWSBedrockLLMService( aws_region="us-west-2", - model="us.anthropic.claude-3-5-haiku-20241022-v1:0", - params=AWSBedrockLLMService.InputParams(temperature=0.8, latency="optimized"), + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8), ) messages = [ diff --git a/examples/foundational/07n-interruptible-gemini-image.py b/examples/foundational/07n-interruptible-gemini-image.py new file mode 100644 index 000000000..61b8e650a --- /dev/null +++ b/examples/foundational/07n-interruptible-gemini-image.py @@ -0,0 +1,151 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +""" +A conversational AI bot using Gemini for both LLM, STT and TTS. + +This example demonstrates how to use Gemini's image generation capabilities. + +Features showcased: +- Gemini LLM for conversation and image generation +- Google TTS and STT + +Run with: + python examples/foundational/07n-interruptible-gemini-image.py + +Make sure to set your environment variables: + export GOOGLE_API_KEY=your_api_key_here +""" + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.google.stt import GoogleSTTService +from pipecat.services.google.tts import GoogleTTSService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = GoogleSTTService( + params=GoogleSTTService.InputParams(languages=Language.EN_US), + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + ) + + tts = GoogleTTSService( + voice_id="en-US-Chirp3-HD-Charon", + params=GoogleTTSService.InputParams(language=Language.EN_US), + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + ) + + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + model="gemini-2.5-flash-image", + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # Gemini TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation with a styled introduction + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/07p-interruptible-krisp-viva.py b/examples/foundational/07p-interruptible-krisp-viva.py new file mode 100644 index 000000000..c7ca15b40 --- /dev/null +++ b/examples/foundational/07p-interruptible-krisp-viva.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepgram.tts import DeepgramTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=KrispVivaFilter(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=KrispVivaFilter(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=KrispVivaFilter(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py deleted file mode 100644 index b84e945c3..000000000 --- a/examples/foundational/08-bots-arguing.py +++ /dev/null @@ -1,147 +0,0 @@ -import asyncio -import logging -import os -from typing import Tuple - -import aiohttp -from dotenv import load_dotenv - -from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMContextFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.processors.aggregators import SentenceAggregator -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.daily import configure -from pipecat.services.azure import AzureLLMService, AzureTTSService -from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.fal import FalImageGenService -from pipecat.transports.daily.transport import DailyTransport - -load_dotenv(override=True) - -logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") -logger = logging.getLogger("pipecat") -logger.setLevel(logging.DEBUG) - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Respond bot", - duration_minutes=10, - mic_enabled=True, - mic_sample_rate=16000, - camera_enabled=True, - camera_width=1024, - camera_height=1024, - ) - - llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), - model=os.getenv("AZURE_CHATGPT_MODEL"), - ) - tts1 = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), - ) - tts2 = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id="jBpfuIE2acCO8z3wKNLl", - ) - dalle = FalImageGenService( - params=FalImageGenService.InputParams(image_size="1024x1024"), - aiohttp_session=session, - key=os.getenv("FAL_KEY"), - ) - - bot1_messages = [ - { - "role": "system", - "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long.", - }, - ] - bot2_messages = [ - { - "role": "system", - "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich.", - }, - ] - - async def get_text_and_audio(messages) -> Tuple[str, bytearray]: - """This function streams text from the LLM and uses the TTS service to convert - that text to speech as it's received. - """ - source_queue = asyncio.Queue() - sink_queue = asyncio.Queue() - sentence_aggregator = SentenceAggregator() - pipeline = Pipeline([llm, sentence_aggregator, tts1], source_queue, sink_queue) - - await source_queue.put(LLMContextFrame(LLMContext(messages))) - await source_queue.put(EndFrame()) - await pipeline.run_pipeline() - - message = "" - all_audio = bytearray() - while sink_queue.qsize(): - frame = sink_queue.get_nowait() - if isinstance(frame, TextFrame): - message += frame.text - elif isinstance(frame, AudioFrame): - all_audio.extend(frame.audio) - - return (message, all_audio) - - async def get_bot1_statement(): - message, audio = await get_text_and_audio(bot1_messages) - - bot1_messages.append({"role": "assistant", "content": message}) - bot2_messages.append({"role": "user", "content": message}) - - return audio - - async def get_bot2_statement(): - message, audio = await get_text_and_audio(bot2_messages) - - bot2_messages.append({"role": "assistant", "content": message}) - bot1_messages.append({"role": "user", "content": message}) - - return audio - - async def argue(): - for i in range(100): - print(f"In iteration {i}") - - bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed" - - (audio1, image_data1) = await asyncio.gather( - get_bot1_statement(), dalle.run_image_gen(bot1_description) - ) - await transport.send_queue.put( - [ - ImageFrame(image_data1[1], image_data1[2]), - AudioFrame(audio1), - ] - ) - - bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed" - - (audio2, image_data2) = await asyncio.gather( - get_bot2_statement(), dalle.run_image_gen(bot2_description) - ) - await transport.send_queue.put( - [ - ImageFrame(image_data2[1], image_data2[2]), - AudioFrame(audio2), - ] - ) - - await asyncio.gather(transport.run(), argue()) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/08-custom-frame-processor.py b/examples/foundational/08-custom-frame-processor.py new file mode 100644 index 000000000..20da4f876 --- /dev/null +++ b/examples/foundational/08-custom-frame-processor.py @@ -0,0 +1,170 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import io +import os +import re + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import ( + Frame, + LLMRunFrame, + MetricsFrame, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +def format_metrics(metrics, indent=0): + lines = [] + tab = "\t" * indent + + for metric in metrics: + lines.append(tab + type(metric).__name__) + for field, value in vars(metric).items(): + if hasattr(value, "__dict__") and not isinstance( + value, (str, int, float, bool, type(None)) + ): + lines.append(f"{tab}\t{field}={type(value).__name__}") + for k, v in vars(value).items(): + lines.append(f"{tab}\t\t{k}={repr(v)}") + else: + lines.append(f"{tab}\t{field}={repr(value)}") + + return "\n".join(lines) + + +class MetricsFrameLogger(FrameProcessor): + """MetricsFrameLogger formats and logs all MetericsFrames""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, MetricsFrame): + logger.info(f"{frame.name}\n {format_metrics(frame.data)}") + await self.push_frame(frame, direction) + + # ALWAYS push all frames + else: + # SUPER IMPORTANT: always push every frame! + await self.push_frame(frame, direction) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + metrics_frame_processor = MetricsFrameLogger() + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + metrics_frame_processor, # pretty print metrics frames + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/13f-cartesia-transcription.py b/examples/foundational/13f-cartesia-transcription.py index 913dce797..d685d0ba8 100644 --- a/examples/foundational/13f-cartesia-transcription.py +++ b/examples/foundational/13f-cartesia-transcription.py @@ -48,10 +48,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService( - api_key=os.getenv("CARTESIA_API_KEY"), - base_url=os.getenv("CARTESIA_BASE_URL"), - ) + stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) tl = TranscriptionLogger() diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index f9ad264eb..cba5eee60 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -76,9 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleVertexLLMService( credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - params=GoogleVertexLLMService.InputParams( - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - ), + project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), + location=os.getenv("GOOGLE_CLOUD_LOCATION"), ) # You can aslo register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index 03aa7bb96..15f7e37a0 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -79,8 +79,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = AWSBedrockLLMService( aws_region="us-west-2", - model="us.anthropic.claude-3-5-haiku-20241022-v1:0", - params=AWSBedrockLLMService.InputParams(temperature=0.8, latency="optimized"), + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8), ) # You can also register a function_name of None to get all functions diff --git a/examples/foundational/14x-function-calling-openpipe.py b/examples/foundational/14x-function-calling-openpipe.py new file mode 100644 index 000000000..3f2537bb7 --- /dev/null +++ b/examples/foundational/14x-function-calling-openpipe.py @@ -0,0 +1,182 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os +import time + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.services.openpipe.llm import OpenPipeLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +async def fetch_weather_from_api(params: FunctionCallParams): + await params.result_callback({"conditions": "nice", "temperature": "75"}) + + +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + timestamp = int(time.time()) + llm = OpenPipeLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + openpipe_api_key=os.getenv("OPENPIPE_API_KEY"), + tags={"conversation_id": f"pipecat-{timestamp}"}, + ) + + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 0f9309f3d..f182d7c8c 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -24,14 +24,15 @@ from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioNoiseReduction, InputAudioTranscription, - OpenAIRealtimeLLMService, SemanticTurnDetection, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index a39826b81..c4b0fc02a 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -21,13 +21,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport +from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( - AzureRealtimeLLMService, +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioTranscription, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py index b5d1d73e1..bb63a4814 100644 --- a/examples/foundational/19b-openai-realtime-text.py +++ b/examples/foundational/19b-openai-realtime-text.py @@ -22,16 +22,17 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioNoiseReduction, InputAudioTranscription, - OpenAIRealtimeLLMService, SemanticTurnDetection, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 52c7d8f49..629a17c67 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -25,13 +25,14 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioTranscription, - OpenAIRealtimeLLMService, SessionProperties, TurnDetection, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 411a976b8..e8822bbc6 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -72,7 +72,6 @@ async def save_conversation(params: FunctionCallParams): ) try: with open(filename, "w") as file: - # todo: extract 'system' into the first message in the list messages = params.context.get_messages() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 8dad8148d..b32c2fd5b 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -90,7 +90,6 @@ async def save_conversation(params: FunctionCallParams): ) try: with open(filename, "w") as file: - # todo: extract 'system' into the first message in the list messages = params.context.get_messages() # remove the last message (the instruction to save the context) messages.pop() diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index bd3d9d545..0bc3a0d4e 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -20,10 +20,12 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.aws_nova_sonic.aws import AWSNovaSonicLLMService +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -75,7 +77,7 @@ async def save_conversation(params: FunctionCallParams): filename = f"{BASE_FILENAME}{timestamp}.json" try: with open(filename, "w") as file: - messages = params.context.get_messages_for_persistent_storage() + messages = params.context.get_messages() # remove the last few messages. in reverse order, they are: # - the in progress save tool call # - the invocation of the save tool call @@ -223,13 +225,13 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext( + context = LLMContext( messages=[ {"role": "system", "content": f"{system_instruction}"}, ], tools=tools, ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-live.py similarity index 97% rename from examples/foundational/26-gemini-multimodal-live.py rename to examples/foundational/26-gemini-live.py index b446a9b8c..dd5dfcffc 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-live.py @@ -17,7 +17,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): Respond to what the user said in a creative and helpful way. """ - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-live-transcription.py similarity index 97% rename from examples/foundational/26a-gemini-multimodal-live-transcription.py rename to examples/foundational/26a-gemini-live-transcription.py index ad3cd06ee..de277156b 100644 --- a/examples/foundational/26a-gemini-multimodal-live-transcription.py +++ b/examples/foundational/26a-gemini-live-transcription.py @@ -20,7 +20,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -65,7 +65,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-live-function-calling.py similarity index 95% rename from examples/foundational/26b-gemini-multimodal-live-function-calling.py rename to examples/foundational/26b-gemini-live-function-calling.py index f14713a5c..65d159bb0 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-live-function-calling.py @@ -22,7 +22,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -122,12 +122,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): required=["location"], ) search_tool = {"google_search": {}} + # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears + # you cannot use the "google_search" tool alongside other tools. + # See https://github.com/googleapis/python-genai/issues/941. tools = ToolsSchema( standard_tools=[weather_function, restaurant_function], custom_tools={AdapterType.GEMINI: [search_tool]}, ) - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, tools=tools, diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-live-video.py similarity index 96% rename from examples/foundational/26c-gemini-multimodal-live-video.py rename to examples/foundational/26c-gemini-live-video.py index a28eaaacf..7b765075e 100644 --- a/examples/foundational/26c-gemini-multimodal-live-video.py +++ b/examples/foundational/26c-gemini-live-video.py @@ -24,7 +24,7 @@ from pipecat.runner.utils import ( maybe_capture_participant_camera, maybe_capture_participant_screen, ) -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -58,7 +58,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-live-text.py similarity index 91% rename from examples/foundational/26d-gemini-multimodal-live-text.py rename to examples/foundational/26d-gemini-live-text.py index 667887b03..062c0231b 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-live-text.py @@ -20,9 +20,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.gemini_multimodal_live.gemini import ( - GeminiMultimodalLiveLLMService, - GeminiMultimodalModalities, +from pipecat.services.google.gemini_live.llm import ( + GeminiLiveLLMService, + GeminiModalities, InputParams, ) from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -80,11 +80,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = GeminiMultimodalLiveLLMService( + # KNOWN ISSUE: If using GeminiLiveVertexLLMService, you cannot specify a + # modality other than AUDIO (at least not if using the service's default + # model, which is a native audio model: + # https://cloud.google.com/vertex-ai/generative-ai/docs/live-api/tools#native-audio). + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=SYSTEM_INSTRUCTION, tools=[{"google_search": {}}, {"code_execution": {}}], - params=InputParams(modalities=GeminiMultimodalModalities.TEXT), + params=InputParams(modalities=GeminiModalities.TEXT), ) # Optionally, you can set the response modalities via a function diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-live-google-search.py similarity index 97% rename from examples/foundational/26e-gemini-multimodal-google-search.py rename to examples/foundational/26e-gemini-live-google-search.py index 31fad54fe..178fdd282 100644 --- a/examples/foundational/26e-gemini-multimodal-google-search.py +++ b/examples/foundational/26e-gemini-live-google-search.py @@ -19,7 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") # Initialize the Gemini Multimodal Live model - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck system_instruction=system_instruction, diff --git a/examples/foundational/26f-gemini-multimodal-live-files-api.py b/examples/foundational/26f-gemini-live-files-api.py similarity index 98% rename from examples/foundational/26f-gemini-multimodal-live-files-api.py rename to examples/foundational/26f-gemini-live-files-api.py index b01c21803..eeda16f52 100644 --- a/examples/foundational/26f-gemini-multimodal-live-files-api.py +++ b/examples/foundational/26f-gemini-live-files-api.py @@ -19,9 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import ( - GeminiMultimodalLiveLLMService, -) +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -110,7 +108,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ # Initialize Gemini service with File API support - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py b/examples/foundational/26g-gemini-live-groundingMetadata.py similarity index 97% rename from examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py rename to examples/foundational/26g-gemini-live-groundingMetadata.py index 6cfeed51b..bea1756b2 100644 --- a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py +++ b/examples/foundational/26g-gemini-live-groundingMetadata.py @@ -9,13 +9,13 @@ from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import Frame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.services.google.frames import LLMSearchResponseFrame +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -105,7 +105,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): custom_tools={AdapterType.GEMINI: [{"google_search": {}}, {"code_execution": {}}]}, ) - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=SYSTEM_INSTRUCTION, voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/foundational/26h-gemini-live-vertex-function-calling.py b/examples/foundational/26h-gemini-live-vertex-function-calling.py new file mode 100644 index 000000000..c0344a052 --- /dev/null +++ b/examples/foundational/26h-gemini-live-vertex-function-calling.py @@ -0,0 +1,191 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os +from datetime import datetime + +from dotenv import load_dotenv +from google.genai.types import HttpOptions +from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService +from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +async def fetch_weather_from_api(params: FunctionCallParams): + temperature = 75 if params.arguments["format"] == "fahrenheit" else 24 + await params.result_callback( + { + "conditions": "nice", + "temperature": temperature, + "format": params.arguments["format"], + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + } + ) + + +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + +system_instruction = """ +You are a helpful assistant who can answer questions and use tools. + +You have three tools available to you: +1. get_current_weather: Use this tool to get the current weather in a specific location. +2. get_restaurant_recommendation: Use this tool to get a restaurant recommendation in a specific location. +""" + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears + # you cannot use the "google_search" tool alongside other tools. + # See https://github.com/googleapis/python-genai/issues/941. + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) + + llm = GeminiLiveVertexLLMService( + credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), + project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), + location=os.getenv("GOOGLE_CLOUD_LOCATION"), + system_instruction=system_instruction, + voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck + tools=tools, + ) + + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + context = OpenAILLMContext( + [{"role": "user", "content": "Say hello."}], + ) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/26i-gemini-live-graceful-end.py b/examples/foundational/26i-gemini-live-graceful-end.py new file mode 100644 index 000000000..e51bbb032 --- /dev/null +++ b/examples/foundational/26i-gemini-live-graceful-end.py @@ -0,0 +1,204 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os +from datetime import datetime + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import EndTaskFrame, LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +async def fetch_weather_from_api(params: FunctionCallParams): + temperature = 75 if params.arguments["format"] == "fahrenheit" else 24 + await params.result_callback( + { + "conditions": "nice", + "temperature": temperature, + "format": params.arguments["format"], + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + } + ) + + +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + +async def end_conversation(params: FunctionCallParams): + await params.result_callback({"success": True}) + await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + + +system_instruction = """ +You are a helpful assistant who can answer questions and use tools. + +You have three tools available to you: +1. get_current_weather: Use this tool to get the current weather in a specific location. +2. get_restaurant_recommendation: Use this tool to get a restaurant recommendation in a specific location. +3. end_conversation: Use this tool to gracefully end the conversation. + +After you've responded to the user three times, do two things, in order: +1. Politely let them know that that's all the time you have today and say goodbye. +2. Call the end_conversation tool to gracefully end the conversation. +""" + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + end_conversation_function = FunctionSchema( + name="end_conversation", + description="Gracefully end the conversation", + properties={}, + required=[], + ) + search_tool = {"google_search": {}} + tools = ToolsSchema( + standard_tools=[weather_function, restaurant_function, end_conversation_function], + custom_tools={AdapterType.GEMINI: [search_tool]}, + ) + + llm = GeminiLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=system_instruction, + tools=tools, + ) + + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + llm.register_function("end_conversation", end_conversation) + + context = OpenAILLMContext( + [{"role": "user", "content": "Say hello."}], + ) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 9479632b5..348cf117b 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -9,7 +9,6 @@ import os from dotenv import load_dotenv from loguru import logger -from simli import SimliConfig from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 @@ -66,11 +65,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="a167e0f3-df7e-4d52-a9c3-f949145efdab", + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", ) simli_ai = SimliVideoService( - SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID")), + api_key=os.getenv("SIMLI_API_KEY"), + face_id="cace3ef7-a4c4-425d-a8cf-a5358eb0c427", ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") diff --git a/examples/foundational/40-aws-nova-sonic.py b/examples/foundational/40-aws-nova-sonic.py index de7bbf638..e5e36e404 100644 --- a/examples/foundational/40-aws-nova-sonic.py +++ b/examples/foundational/40-aws-nova-sonic.py @@ -18,10 +18,11 @@ from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.aws_nova_sonic import AWSNovaSonicLLMService +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -119,9 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_current_weather", fetch_weather_from_api) # Set up context and context management. - # AWSNovaSonicService will adapt OpenAI LLM context objects with standard message format to - # what's expected by Nova Sonic. - context = OpenAILLMContext( + context = LLMContext( messages=[ {"role": "system", "content": f"{system_instruction}"}, { @@ -131,7 +130,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ], tools=tools, ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = LLMContextAggregatorPair(context) # Build the pipeline pipeline = Pipeline( diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py index bfb718ff2..62ea5debe 100644 --- a/examples/foundational/46-video-processing.py +++ b/examples/foundational/46-video-processing.py @@ -20,7 +20,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import TransportParams from pipecat.transports.daily.transport import DailyParams, DailyTransport @@ -94,7 +94,7 @@ Respond to what the user said in a creative and helpful way. Keep your responses async def run_bot(pipecat_transport): - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck transcribe_user_audio=True, diff --git a/examples/foundational/47-sentry-metrics.py b/examples/foundational/47-sentry-metrics.py new file mode 100644 index 000000000..ae7b7a59d --- /dev/null +++ b/examples/foundational/47-sentry-metrics.py @@ -0,0 +1,142 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import sentry_sdk +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.metrics.sentry import SentryMetrics +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + # Initialize Sentry + sentry_sdk.init( + dsn=os.getenv("SENTRY_DSN"), + traces_sample_rate=1.0, + ) + + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + metrics=SentryMetrics(), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + metrics=SentryMetrics(), + ) + + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + metrics=SentryMetrics(), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/48-service-switcher.py b/examples/foundational/48-service-switcher.py new file mode 100644 index 000000000..d0e15d2d3 --- /dev/null +++ b/examples/foundational/48-service-switcher.py @@ -0,0 +1,153 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, ManuallySwitchServiceFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.service_switcher import ServiceSwitcher, ServiceSwitcherStrategyManual +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.stt import CartesiaSTTService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepgram.tts import DeepgramTTSService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt_cartesia = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) + stt_deepgram = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt_switcher = ServiceSwitcher( + services=[stt_cartesia, stt_deepgram], strategy_type=ServiceSwitcherStrategyManual + ) + + tts_cartesia = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", + ) + tts_deepgram = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts_switcher = ServiceSwitcher( + services=[tts_cartesia, tts_deepgram], strategy_type=ServiceSwitcherStrategyManual + ) + + llm_openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm_google = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + llm_switcher = ServiceSwitcher( + services=[llm_openai, llm_google], strategy_type=ServiceSwitcherStrategyManual + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt_switcher, + context_aggregator.user(), # User responses + llm_switcher, # LLM + tts_switcher, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + await asyncio.sleep(15) + print(f"Switching to {stt_deepgram}") + await task.queue_frames([ManuallySwitchServiceFrame(service=stt_deepgram)]) + await asyncio.sleep(15) + print(f"Switching to {llm_google}") + await task.queue_frames([ManuallySwitchServiceFrame(service=llm_google)]) + await asyncio.sleep(15) + print(f"Switching to {tts_deepgram}") + await task.queue_frames([ManuallySwitchServiceFrame(service=tts_deepgram)]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/README.md b/examples/foundational/README.md index 9a6c26005..c1ead3ece 100644 --- a/examples/foundational/README.md +++ b/examples/foundational/README.md @@ -105,7 +105,7 @@ uv run 07-interruptible.py -t twilio -x NGROK_HOST_NAME ### Vision & Multimodal - **[12a-describe-video-gemini-flash.py](./12a-describe-video-gemini-flash.py)**: Bot describes user's video (Video input, Multimodal LLMs) -- **[26c-gemini-multimodal-live-video.py](./26c-gemini-multimodal-live-video.py)**: Gemini with video input (Streaming video, Function calls) +- **[26c-gemini-live-video.py](./26c-gemini-live-video.py)**: Gemini with video input (Streaming video, Function calls) ### Voice & Language diff --git a/examples/foundational/assets/moondream.png b/examples/foundational/assets/moondream.png new file mode 100644 index 000000000..17ba5401f Binary files /dev/null and b/examples/foundational/assets/moondream.png differ diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md index cf7c2de1a..91a3fd888 100644 --- a/examples/quickstart/README.md +++ b/examples/quickstart/README.md @@ -73,13 +73,13 @@ Transform your local bot into a production-ready service. Pipecat Cloud handles 1. [Sign up for Pipecat Cloud](https://pipecat.daily.co/sign-up). -2. Install the Pipecat Cloud CLI: +2. Install the Pipecat CLI: ```bash - uv add pipecatcloud + uv tool install pipecat-ai-cli ``` -> πŸ’‘ Tip: You can run the `pipecatcloud` CLI using the `pcc` alias. +> πŸ’‘ Tip: You can run the `pipecat` CLI using the `pc` alias. 3. Set up Docker for building your bot image: @@ -113,12 +113,22 @@ secret_set = "quickstart-secrets" > πŸ’‘ Tip: [Set up `image_credentials`](https://docs.pipecat.ai/deployment/pipecat-cloud/fundamentals/secrets#image-pull-secrets) in your TOML file for authenticated image pulls +### Log in to Pipecat Cloud + +To start using the CLI, authenticate to Pipecat Cloud: + +```bash +pipecat cloud auth login +``` + +You'll be presented with a link that you can click to authenticate your client. + ### Configure secrets Upload your API keys to Pipecat Cloud's secure storage: ```bash -uv run pcc secrets set quickstart-secrets --file .env +pipecat cloud secrets set quickstart-secrets --file .env ``` This creates a secret set called `quickstart-secrets` (matching your TOML file) and uploads all your API keys from `.env`. @@ -128,13 +138,13 @@ This creates a secret set called `quickstart-secrets` (matching your TOML file) Build your Docker image and push to Docker Hub: ```bash -uv run pcc docker build-push +pipecat cloud docker build-push ``` Deploy to Pipecat Cloud: ```bash -uv run pcc deploy +pipecat cloud deploy ``` ### Connect to your agent diff --git a/examples/quickstart/pcc-deploy.toml b/examples/quickstart/pcc-deploy.toml index 28413327f..ff77c45dc 100644 --- a/examples/quickstart/pcc-deploy.toml +++ b/examples/quickstart/pcc-deploy.toml @@ -1,6 +1,11 @@ agent_name = "quickstart" image = "your_username/quickstart:0.1" secret_set = "quickstart-secrets" +agent_profile = "agent-1x" + +# RECOMMENDED: Set an image pull secret: +# https://docs.pipecat.ai/deployment/pipecat-cloud/fundamentals/secrets#image-pull-secrets +# image_credentials = "your_image_pull_secret" [scaling] min_agents = 1 diff --git a/examples/quickstart/pyproject.toml b/examples/quickstart/pyproject.toml index 5d9df3eb4..863e350d4 100644 --- a/examples/quickstart/pyproject.toml +++ b/examples/quickstart/pyproject.toml @@ -4,13 +4,14 @@ version = "0.1.0" description = "Quickstart example for building voice AI bots with Pipecat" requires-python = ">=3.10" dependencies = [ - "pipecat-ai[webrtc,daily,silero,deepgram,openai,cartesia,local-smart-turn-v3,runner]>=0.0.86", - "pipecatcloud>=0.2.4" + "pipecat-ai[webrtc,daily,silero,deepgram,openai,cartesia,local-smart-turn-v3,runner]", + "pipecat-ai-cli" ] [dependency-groups] dev = [ - "ruff~=0.12.1", + "pyright>=1.1.404,<2", + "ruff>=0.12.11,<1", ] [tool.ruff] diff --git a/pyproject.toml b/pyproject.toml index 0adbf281e..5760361b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "pyloudnorm~=0.1.1", "resampy~=0.4.3", "soxr~=0.5.0", - "openai>=1.74.0,<=1.99.1", + "openai>=1.74.0,<3", # Pinning numba to resolve package dependencies "numba==0.61.2", "wait_for2>=0.4.1; python_version<'3.12'", @@ -50,19 +50,19 @@ anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] -aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.0.2; python_version>='3.12'" ] +aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.19.9" ] +daily = [ "daily-python~=0.21.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] fireworks = [] fish = [ "ormsgpack~=1.7.0", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] -google = [ "google-cloud-speech~=2.32.0", "google-cloud-texttospeech~=2.26.0", "google-genai~=1.24.0", "pipecat-ai[websockets-base]" ] +google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.41.0,<2", "pipecat-ai[websockets-base]" ] grok = [] groq = [ "groq~=0.23.0" ] gstreamer = [ "pygobject~=3.50.0" ] @@ -84,7 +84,7 @@ nim = [] neuphonic = [ "pipecat-ai[websockets-base]" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "pipecat-ai[websockets-base]" ] -openpipe = [ "openpipe~=4.50.0" ] +openpipe = [ "openpipe>=4.50.0,<6" ] openrouter = [] perplexity = [] playht = [ "pipecat-ai[websockets-base]" ] @@ -102,7 +102,7 @@ silero = [ "onnxruntime>=1.20.1,<2" ] simli = [ "simli-ai~=0.1.10"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.0" ] -speechmatics = [ "speechmatics-rt>=0.4.0" ] +speechmatics = [ "speechmatics-rt>=0.5.0" ] strands = [ "strands-agents>=1.9.1,<2" ] tavus=[] together = [] diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 3fc55d983..14f9dee52 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -75,8 +75,6 @@ TESTS_07 = [ EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST, ), - ("07e-interruptible-playht.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07e-interruptible-playht-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07f-interruptible-azure.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07g-interruptible-openai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ("07h-interruptible-openpipe.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), @@ -138,6 +136,7 @@ TESTS_14 = [ ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14x-function-calling-openpipe.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # Currently not working. # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), @@ -149,7 +148,10 @@ TESTS_15 = [ ] TESTS_19 = [ + ("19-openai-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # OpenAI Realtime not released on Azure yet + # ("19a-azure-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19b-openai-realtime-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19b-openai-realtime-beta-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), @@ -162,18 +164,18 @@ TESTS_21 = [ TESTS_26 = [ ("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ( - "26a-gemini-multimodal-live-transcription.py", + "26a-gemini-live-transcription.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST, ), ( - "26b-gemini-multimodal-live-function-calling.py", + "26b-gemini-live-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST, ), - ("26c-gemini-multimodal-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("26c-gemini-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ( "26e-gemini-multimodal-google-search.py", PROMPT_ONLINE_SEARCH, @@ -181,7 +183,13 @@ TESTS_26 = [ BOT_SPEAKS_FIRST, ), # Currently not working. - # ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("26d-gemini-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ( + "26h-gemini-live-vertex-function-calling.py", + PROMPT_WEATHER, + EVAL_WEATHER, + BOT_SPEAKS_FIRST, + ), ] TESTS_27 = [ diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index adfe81005..a106b4de4 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -110,7 +110,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): system = NOT_GIVEN messages = [] - # first, map messages using self._from_universal_context_message(m) + # First, map messages using self._from_universal_context_message(m) try: messages = [self._from_universal_context_message(m) for m in universal_context_messages] except Exception as e: diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py index 64319d266..60f12798b 100644 --- a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -6,13 +6,47 @@ """AWS Nova Sonic LLM adapter for Pipecat.""" +import copy import json -from typing import Any, Dict, List, TypedDict +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, List, Optional, TypedDict + +from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage + + +class Role(Enum): + """Roles supported in AWS Nova Sonic conversations. + + Parameters: + SYSTEM: System-level messages (not used in conversation history). + USER: Messages sent by the user. + ASSISTANT: Messages sent by the assistant. + TOOL: Messages sent by tools (not used in conversation history). + """ + + SYSTEM = "SYSTEM" + USER = "USER" + ASSISTANT = "ASSISTANT" + TOOL = "TOOL" + + +@dataclass +class AWSNovaSonicConversationHistoryMessage: + """A single message in AWS Nova Sonic conversation history. + + Parameters: + role: The role of the message sender (USER or ASSISTANT only). + text: The text content of the message. + """ + + role: Role # only USER and ASSISTANT + text: str class AWSNovaSonicLLMInvocationParams(TypedDict): @@ -21,7 +55,9 @@ class AWSNovaSonicLLMInvocationParams(TypedDict): This is a placeholder until support for universal LLMContext machinery is added for AWS Nova Sonic. """ - pass + system_instruction: Optional[str] + messages: List[AWSNovaSonicConversationHistoryMessage] + tools: List[Dict[str, Any]] class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): @@ -34,7 +70,7 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): @property def id_for_llm_specific_messages(self) -> str: """Get the identifier used in LLMSpecificMessage instances for AWS Nova Sonic.""" - raise NotImplementedError("Universal LLMContext is not yet supported for AWS Nova Sonic.") + return "aws-nova-sonic" def get_llm_invocation_params(self, context: LLMContext) -> AWSNovaSonicLLMInvocationParams: """Get AWS Nova Sonic-specific LLM invocation parameters from a universal LLM context. @@ -47,7 +83,13 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): Returns: Dictionary of parameters for invoking AWS Nova Sonic's LLM API. """ - raise NotImplementedError("Universal LLMContext is not yet supported for AWS Nova Sonic.") + messages = self._from_universal_context_messages(self.get_messages(context)) + return { + "system_instruction": messages.system_instruction, + "messages": messages.messages, + # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) + "tools": self.from_standard_tools(context.tools) or [], + } def get_messages_for_logging(self, context) -> List[Dict[str, Any]]: """Get messages from a universal LLM context in a format ready for logging about AWS Nova Sonic. @@ -62,7 +104,75 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): Returns: List of messages in a format ready for logging about AWS Nova Sonic. """ - raise NotImplementedError("Universal LLMContext is not yet supported for AWS Nova Sonic.") + return self._from_universal_context_messages(self.get_messages(context)).messages + + @dataclass + class ConvertedMessages: + """Container for Google-formatted messages converted from universal context.""" + + messages: List[AWSNovaSonicConversationHistoryMessage] + system_instruction: Optional[str] = None + + def _from_universal_context_messages( + self, universal_context_messages: List[LLMContextMessage] + ) -> ConvertedMessages: + system_instruction = None + messages = [] + + # Bail if there are no messages + if not universal_context_messages: + return self.ConvertedMessages() + + universal_context_messages = copy.deepcopy(universal_context_messages) + + # If we have a "system" message as our first message, let's pull that out into "instruction" + if universal_context_messages[0].get("role") == "system": + system = universal_context_messages.pop(0) + content = system.get("content") + if isinstance(content, str): + system_instruction = content + elif isinstance(content, list): + system_instruction = content[0].get("text") + if system_instruction: + self._system_instruction = system_instruction + + # Process remaining messages to fill out conversation history. + # Nova Sonic supports "user" and "assistant" messages in history. + for universal_context_message in universal_context_messages: + message = self._from_universal_context_message(universal_context_message) + if message: + messages.append(message) + + return self.ConvertedMessages(messages=messages, system_instruction=system_instruction) + + def _from_universal_context_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + """Convert standard message format to Nova Sonic format. + + Args: + message: Standard message dictionary to convert. + + Returns: + Nova Sonic conversation history message, or None if not convertible. + """ + role = message.get("role") + if message.get("role") == "user" or message.get("role") == "assistant": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + # There won't be content if this is an assistant tool call entry. + # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation + # history + if content: + return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova + # Sonic conversation history @staticmethod def _to_aws_nova_sonic_function_format(function: FunctionSchema) -> Dict[str, Any]: diff --git a/src/pipecat/adapters/services/bedrock_adapter.py b/src/pipecat/adapters/services/bedrock_adapter.py index 681dfb3dc..852ea17a4 100644 --- a/src/pipecat/adapters/services/bedrock_adapter.py +++ b/src/pipecat/adapters/services/bedrock_adapter.py @@ -107,7 +107,7 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): system = None messages = [] - # first, map messages using self._from_universal_context_message(m) + # First, map messages using self._from_universal_context_message(m) try: messages = [self._from_universal_context_message(m) for m in universal_context_messages] except Exception as e: diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 63a86e6d2..26f037127 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -8,8 +8,8 @@ import base64 import json -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, TypedDict +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, TypedDict from loguru import logger from openai import NotGiven @@ -87,9 +87,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Includes both converted standard tools and any custom Gemini-specific tools. """ functions_schema = tools_schema.standard_tools - formatted_standard_tools = [ - {"function_declarations": [func.to_default_dict() for func in functions_schema]} - ] + formatted_standard_tools = ( + [{"function_declarations": [func.to_default_dict() for func in functions_schema]}] + if functions_schema + else [] + ) custom_gemini_tools = [] if tools_schema.custom_tools: custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, []) @@ -131,6 +133,28 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): messages: List[Content] system_instruction: Optional[str] = None + @dataclass + class MessageConversionResult: + """Result of converting a single universal context message to Google format. + + Either content (a Google Content object) or a system instruction string + is guaranteed to be set. + + Also returns a tool call ID to name mapping for any tool calls + discovered in the message. + """ + + content: Optional[Content] = None + system_instruction: Optional[str] = None + tool_call_id_to_name_mapping: Dict[str, str] = field(default_factory=dict) + + @dataclass + class MessageConversionParams: + """Parameters for converting a single universal context message to Google format.""" + + already_have_system_instruction: bool + tool_call_id_to_name_mapping: Dict[str, str] + def _from_universal_context_messages( self, universal_context_messages: List[LLMContextMessage] ) -> ConvertedMessages: @@ -154,24 +178,26 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): """ system_instruction = None messages = [] + tool_call_id_to_name_mapping = {} # Process each message, preserving Google-formatted messages and converting others for message in universal_context_messages: - if isinstance(message, LLMSpecificMessage): - # Assume that LLMSpecificMessage wraps a message in Google format - messages.append(message.message) - continue - - # Convert standard format to Google format - converted = self._from_standard_message( - message, already_have_system_instruction=bool(system_instruction) + result = self._from_universal_context_message( + message, + params=self.MessageConversionParams( + already_have_system_instruction=bool(system_instruction), + tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, + ), ) - if isinstance(converted, Content): - # Regular (non-system) message - messages.append(converted) - else: - # System instruction - system_instruction = converted + # Each result is either a Content or a system instruction + if result.content: + messages.append(result.content) + elif result.system_instruction: + system_instruction = result.system_instruction + + # Merge tool call ID to name mapping + if result.tool_call_id_to_name_mapping: + tool_call_id_to_name_mapping.update(result.tool_call_id_to_name_mapping) # Check if we only have function-related messages (no regular text) has_regular_messages = any( @@ -191,9 +217,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): return self.ConvertedMessages(messages=messages, system_instruction=system_instruction) + def _from_universal_context_message( + self, message: LLMContextMessage, *, params: MessageConversionParams + ) -> MessageConversionResult: + if isinstance(message, LLMSpecificMessage): + return self.MessageConversionResult(content=message.message) + return self._from_standard_message(message, params=params) + def _from_standard_message( - self, message: LLMStandardMessage, already_have_system_instruction: bool - ) -> Content | str: + self, message: LLMStandardMessage, *, params: MessageConversionParams + ) -> MessageConversionResult: """Convert standard universal context message to Google Content object. Handles conversion of text, images, and function calls to Google's @@ -203,10 +236,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Args: message: Message in standard universal context format. already_have_system_instruction: Whether we already have a system instruction + params: Parameters for conversion. Returns: - Content object with role and parts, or a plain string for system - messages. + MessageConversionResult containing either a Content object or a + system instruction string. Examples: Standard text message:: @@ -240,38 +274,48 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Converts to Google Content with:: Content( - role="model", + role="user", parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))] ) """ role = message["role"] content = message.get("content", []) + if role == "system": - if already_have_system_instruction: + if params.already_have_system_instruction: role = "user" # Convert system message to user role if we already have a system instruction else: - # System instructions are returned as plain text + system_instruction: str = None if isinstance(content, str): - return content + system_instruction = content elif isinstance(content, list): # If content is a list, we assume it's a list of text parts, per the standard - return " ".join(part["text"] for part in content if part.get("type") == "text") + system_instruction = " ".join( + part["text"] for part in content if part.get("type") == "text" + ) + if system_instruction: + return self.MessageConversionResult(system_instruction=system_instruction) elif role == "assistant": role = "model" parts = [] + tool_call_id_to_name_mapping = {} + if message.get("tool_calls"): for tc in message["tool_calls"]: + id = tc["id"] + name = tc["function"]["name"] + tool_call_id_to_name_mapping[id] = name parts.append( Part( function_call=FunctionCall( - name=tc["function"]["name"], + name=name, args=json.loads(tc["function"]["arguments"]), ) ) ) elif role == "tool": - role = "model" + role = "user" try: response = json.loads(message["content"]) if isinstance(response, dict): @@ -282,12 +326,17 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # Response might not be JSON-deserializable. # This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string. response_dict = {"value": message["content"]} + + # Get function name from mapping using tool_call_id, or fallback + tool_call_id = message.get("tool_call_id") + function_name = "tool_call_result" # Default fallback + if tool_call_id and tool_call_id in params.tool_call_id_to_name_mapping: + function_name = params.tool_call_id_to_name_mapping[tool_call_id] + parts.append( - Part( - function_response=FunctionResponse( - name="tool_call_result", # seems to work to hard-code the same name every time - response=response_dict, - ) + Part.from_function_response( + name=function_name, + response=response_dict, ) ) elif isinstance(content, str): @@ -310,4 +359,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): audio_bytes = base64.b64decode(input_audio["data"]) parts.append(Part(inline_data=Blob(mime_type="audio/wav", data=audio_bytes))) - return Content(role=role, parts=parts) + return self.MessageConversionResult( + content=Content(role=role, parts=parts), + tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, + ) diff --git a/src/pipecat/audio/filters/krisp_viva_filter.py b/src/pipecat/audio/filters/krisp_viva_filter.py new file mode 100644 index 000000000..ddb489168 --- /dev/null +++ b/src/pipecat/audio/filters/krisp_viva_filter.py @@ -0,0 +1,193 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Krisp noise reduction audio filter for Pipecat. + +This module provides an audio filter implementation using Krisp VIVA SDK. +""" + +import os + +import numpy as np +from loguru import logger + +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame + +try: + import krisp_audio +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the Krisp filter, you need to install krisp_audio.") + raise Exception(f"Missing module: {e}") + + +def _log_callback(log_message, log_level): + logger.info(f"[{log_level}] {log_message}") + + +class KrispVivaFilter(BaseAudioFilter): + """Audio filter using the Krisp VIVA SDK. + + Provides real-time noise reduction for audio streams using Krisp's + proprietary noise suppression algorithms. This filter requires a + valid Krisp model file to operate. + + Supported sample rates: + - 8000 Hz + - 16000 Hz + - 24000 Hz + - 32000 Hz + - 44100 Hz + - 48000 Hz + """ + + # Initialize Krisp Audio SDK globally + krisp_audio.globalInit("", _log_callback, krisp_audio.LogLevel.Off) + SDK_VERSION = krisp_audio.getVersion() + logger.debug( + f"Krisp Audio Python SDK Version: {SDK_VERSION.major}." + f"{SDK_VERSION.minor}.{SDK_VERSION.patch}" + ) + + SAMPLE_RATES = { + 8000: krisp_audio.SamplingRate.Sr8000Hz, + 16000: krisp_audio.SamplingRate.Sr16000Hz, + 24000: krisp_audio.SamplingRate.Sr24000Hz, + 32000: krisp_audio.SamplingRate.Sr32000Hz, + 44100: krisp_audio.SamplingRate.Sr44100Hz, + 48000: krisp_audio.SamplingRate.Sr48000Hz, + } + + FRAME_SIZE_MS = 10 # Krisp requires audio frames of 10ms duration for processing. + + def __init__(self, model_path: str = None, noise_suppression_level: int = 100) -> None: + """Initialize the Krisp noise reduction filter. + + Args: + model_path: Path to the Krisp model file (.kef extension). + If None, uses KRISP_VIVA_MODEL_PATH environment variable. + noise_suppression_level: Noise suppression level. + + Raises: + ValueError: If model_path is not provided and KRISP_VIVA_MODEL_PATH is not set. + Exception: If model file doesn't have .kef extension. + FileNotFoundError: If model file doesn't exist. + """ + super().__init__() + + # Set model path, checking environment if not specified + self._model_path = model_path or os.getenv("KRISP_VIVA_MODEL_PATH") + if not self._model_path: + logger.error("Model path is not provided and KRISP_VIVA_MODEL_PATH is not set.") + raise ValueError("Model path for KrispAudioProcessor must be provided.") + + if not self._model_path.endswith(".kef"): + raise Exception("Model is expected with .kef extension") + + if not os.path.isfile(self._model_path): + raise FileNotFoundError(f"Model file not found: {self._model_path}") + + self._filtering = True + self._session = None + self._samples_per_frame = None + self._noise_suppression_level = noise_suppression_level + + # Audio buffer to accumulate samples for complete frames + self._audio_buffer = bytearray() + + def _int_to_sample_rate(self, sample_rate): + """Convert integer sample rate to krisp_audio SamplingRate enum. + + Args: + sample_rate: Sample rate as integer + + Returns: + krisp_audio.SamplingRate enum value + + Raises: + ValueError: If sample rate is not supported + """ + if sample_rate not in self.SAMPLE_RATES: + raise ValueError("Unsupported sample rate") + return self.SAMPLE_RATES[sample_rate] + + async def start(self, sample_rate: int): + """Initialize the Krisp processor with the transport's sample rate. + + Args: + sample_rate: The sample rate of the input transport in Hz. + """ + model_info = krisp_audio.ModelInfo() + model_info.path = self._model_path + + nc_cfg = krisp_audio.NcSessionConfig() + nc_cfg.inputSampleRate = self._int_to_sample_rate(sample_rate) + nc_cfg.inputFrameDuration = krisp_audio.FrameDuration.Fd10ms + nc_cfg.outputSampleRate = nc_cfg.inputSampleRate + nc_cfg.modelInfo = model_info + + self._samples_per_frame = int((sample_rate * self.FRAME_SIZE_MS) / 1000) + self._session = krisp_audio.NcInt16.create(nc_cfg) + + async def stop(self): + """Clean up the Krisp processor when stopping.""" + self._session = None + + async def process_frame(self, frame: FilterControlFrame): + """Process control frames to enable/disable filtering. + + Args: + frame: The control frame containing filter commands. + """ + if isinstance(frame, FilterEnableFrame): + self._filtering = frame.enable + + async def filter(self, audio: bytes) -> bytes: + """Apply Krisp noise reduction to audio data. + + Args: + audio: Raw audio data as bytes to be filtered. + + Returns: + Noise-reduced audio data as bytes. + """ + if not self._filtering: + return audio + + # Add incoming audio to our buffer + self._audio_buffer.extend(audio) + + # Calculate how many complete frames we can process + total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample + num_complete_frames = total_samples // self._samples_per_frame + + if num_complete_frames == 0: + # Not enough samples for a complete frame yet, return empty + return b"" + + # Calculate how many bytes we need for complete frames + complete_samples_count = num_complete_frames * self._samples_per_frame + bytes_to_process = complete_samples_count * 2 # 2 bytes per sample + + # Extract the bytes we can process + audio_to_process = bytes(self._audio_buffer[:bytes_to_process]) + + # Remove processed bytes from buffer, keep the remainder + self._audio_buffer = self._audio_buffer[bytes_to_process:] + + # Process the complete frames + samples = np.frombuffer(audio_to_process, dtype=np.int16) + frames = samples.reshape(-1, self._samples_per_frame) + processed_samples = np.empty_like(samples) + + for i, frame in enumerate(frames): + cleaned_frame = self._session.process(frame, self._noise_suppression_level) + processed_samples[i * self._samples_per_frame : (i + 1) * self._samples_per_frame] = ( + cleaned_frame + ) + + return processed_samples.tobytes() diff --git a/src/pipecat/pipeline/llm_switcher.py b/src/pipecat/pipeline/llm_switcher.py index a15e0a3c2..50d919263 100644 --- a/src/pipecat/pipeline/llm_switcher.py +++ b/src/pipecat/pipeline/llm_switcher.py @@ -14,20 +14,41 @@ from pipecat.services.llm_service import LLMService class LLMSwitcher(ServiceSwitcher[StrategyType]): - """A pipeline that switches between different LLMs at runtime.""" + """A pipeline that switches between different LLMs at runtime. + + Example:: + + llm_switcher = LLMSwitcher( + llms=[openai_llm, anthropic_llm], + strategy_type=ServiceSwitcherStrategyManual + ) + """ def __init__(self, llms: List[LLMService], strategy_type: Type[StrategyType]): - """Initialize the service switcher with a list of LLMs and a switching strategy.""" + """Initialize the service switcher with a list of LLMs and a switching strategy. + + Args: + llms: List of LLM services to switch between. + strategy_type: The strategy class to use for switching between LLMs. + """ super().__init__(llms, strategy_type) @property def llms(self) -> List[LLMService]: - """Get the list of LLMs managed by this switcher.""" + """Get the list of LLMs managed by this switcher. + + Returns: + List of LLM services managed by this switcher. + """ return self.services @property def active_llm(self) -> Optional[LLMService]: - """Get the currently active LLM, if any.""" + """Get the currently active LLM. + + Returns: + The currently active LLM service, or None if no LLM is active. + """ return self.strategy.active_service async def run_inference(self, context: LLMContext) -> Optional[str]: diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 9d82fcd88..c7e246681 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -70,11 +70,15 @@ class PipelineRunner(BaseObject): """ logger.debug(f"Runner {self} started running {task}") self._tasks[task.name] = task - params = PipelineTaskParams(loop=self._loop) + + # PipelineTask handles asyncio.CancelledError to shutdown the pipeline + # properly and re-raises it in case there's more cleanup to do. try: + params = PipelineTaskParams(loop=self._loop) await task.run(params) except asyncio.CancelledError: - await self._cancel() + pass + del self._tasks[task.name] # Cleanup base object. diff --git a/src/pipecat/pipeline/service_switcher.py b/src/pipecat/pipeline/service_switcher.py index eea55e68d..8895d663c 100644 --- a/src/pipecat/pipeline/service_switcher.py +++ b/src/pipecat/pipeline/service_switcher.py @@ -21,10 +21,22 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class ServiceSwitcherStrategy: - """Base class for service switching strategies.""" + """Base class for service switching strategies. + + Note: + Strategy classes are instantiated internally by ServiceSwitcher. + Developers should pass the strategy class (not an instance) to ServiceSwitcher. + """ def __init__(self, services: List[FrameProcessor]): - """Initialize the service switcher strategy with a list of services.""" + """Initialize the service switcher strategy with a list of services. + + Note: + This is called internally by ServiceSwitcher. Do not instantiate directly. + + Args: + services: List of frame processors to switch between. + """ self.services = services self.active_service: Optional[FrameProcessor] = None @@ -46,10 +58,24 @@ class ServiceSwitcherStrategyManual(ServiceSwitcherStrategy): This strategy allows the user to manually select which service is active. The initial active service is the first one in the list. + + Example:: + + stt_switcher = ServiceSwitcher( + services=[stt_1, stt_2], + strategy_type=ServiceSwitcherStrategyManual + ) """ def __init__(self, services: List[FrameProcessor]): - """Initialize the manual service switcher strategy with a list of services.""" + """Initialize the manual service switcher strategy with a list of services. + + Note: + This is called internally by ServiceSwitcher. Do not instantiate directly. + + Args: + services: List of frame processors to switch between. + """ super().__init__(services) self.active_service = services[0] if services else None @@ -85,7 +111,12 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): """A pipeline that switches between different services at runtime.""" def __init__(self, services: List[FrameProcessor], strategy_type: Type[StrategyType]): - """Initialize the service switcher with a list of services and a switching strategy.""" + """Initialize the service switcher with a list of services and a switching strategy. + + Args: + services: List of frame processors to switch between. + strategy_type: The strategy class to use for switching between services. + """ strategy = strategy_type(services) super().__init__(*self._make_pipeline_definitions(services, strategy)) self.services = services @@ -100,14 +131,20 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]): active_service: FrameProcessor, direction: FrameDirection, ): - """Initialize the service switcher filter with a strategy and direction.""" + """Initialize the service switcher filter with a strategy and direction. + + Args: + wrapped_service: The service that this filter wraps. + active_service: The currently active service. + direction: The direction of frame flow to filter. + """ + self._wrapped_service = wrapped_service + self._active_service = active_service async def filter(_: Frame) -> bool: return self._wrapped_service == self._active_service - super().__init__(filter, direction) - self._wrapped_service = wrapped_service - self._active_service = active_service + super().__init__(filter, direction, filter_system_frames=True) async def process_frame(self, frame, direction): """Process a frame through the filter, handling special internal filter-updating frames.""" diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 9ce3baf7f..a511db12a 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -138,6 +138,8 @@ class PipelineTask(BasePipelineTask): Use this event for cleanup, logging, or post-processing tasks. Users can inspect the frame if they need to handle specific cases. + - on_pipeline_error: Called when an error occurs with ErrorFrame + Example:: @task.event_handler("on_frame_reached_upstream") @@ -148,9 +150,17 @@ class PipelineTask(BasePipelineTask): async def on_pipeline_idle_timeout(task): ... + @task.event_handler("on_pipeline_started") + async def on_pipeline_started(task, frame): + ... + @task.event_handler("on_pipeline_finished") async def on_pipeline_finished(task, frame): ... + + @task.event_handler("on_pipeline_error") + async def on_pipeline_error(task, frame): + ... """ def __init__( @@ -259,6 +269,9 @@ class PipelineTask(BasePipelineTask): # StopFrame) has been received at the end of the pipeline. self._pipeline_end_event = asyncio.Event() + # This event is set when the pipeline truly finishes. + self._pipeline_finished_event = asyncio.Event() + # This is the final pipeline. It is composed of a source processor, # followed by the user pipeline, and ending with a sink processor. The # source allows us to receive and react to upstream frames, and the sink @@ -288,6 +301,7 @@ class PipelineTask(BasePipelineTask): self._register_event_handler("on_pipeline_ended") self._register_event_handler("on_pipeline_cancelled") self._register_event_handler("on_pipeline_finished") + self._register_event_handler("on_pipeline_error") @property def params(self) -> PipelineParams: @@ -390,11 +404,7 @@ class PipelineTask(BasePipelineTask): await self.queue_frame(EndFrame()) async def cancel(self): - """Immediately stop the running pipeline. - - Cancels all running tasks and stops frame processing without - waiting for completion. - """ + """Request the running pipeline to cancel.""" if not self._finished: await self._cancel() @@ -406,51 +416,38 @@ class PipelineTask(BasePipelineTask): """ if self.has_finished(): return - cleanup_pipeline = True + + # Setup processors. + await self._setup(params) + + # Create all main tasks and wait for the main push task. This is the + # task that pushes frames to the very beginning of our pipeline (i.e. to + # our controlled source processor). + await self._create_tasks() + try: - # Setup processors. - await self._setup(params) - - # Create all main tasks and wait of the main push task. This is the - # task that pushes frames to the very beginning of our pipeline (our - # controlled source processor). - push_task = await self._create_tasks() - await push_task - - # We have already cleaned up the pipeline inside the task. - cleanup_pipeline = False - - # Pipeline has finished nicely. - self._finished = True + # Wait for pipeline to finish. + await self._wait_for_pipeline_finished() except asyncio.CancelledError: - # Raise exception back to the pipeline runner so it can cancel this - # task properly. + logger.debug(f"Pipeline task {self} got cancelled from outside...") + # We have been cancelled from outside, let's just cancel everything. + await self._cancel() + # Wait again for pipeline to finish. This time we have really + # cancelled, so it should really finish. + await self._wait_for_pipeline_finished() + # Re-raise in case there's more cleanup to do. raise finally: # We can reach this point for different reasons: # - # 1. The task has finished properly (e.g. `EndFrame`). - # 2. By calling `PipelineTask.cancel()`. - # 3. By asyncio task cancellation. - # - # Case (1) will execute the code below without issues because - # `self._finished` is true. - # - # Case (2) will execute the code below without issues because - # `self._cancelled` is true. - # - # Case (3) will raise the exception above (because we are cancelling - # the asyncio task). This will be then captured by the - # `PipelineRunner` which will call `PipelineTask.cancel()` and - # therefore becoming case (2). - if self._finished or self._cancelled: - logger.debug(f"Pipeline task {self} is finishing cleanup...") - await self._cancel_tasks() - await self._cleanup(cleanup_pipeline) - if self._check_dangling_tasks: - self._print_dangling_tasks() - self._finished = True - logger.debug(f"Pipeline task {self} has finished") + # 1. The pipeline task has finished (try case). + # 2. By an asyncio task cancellation (except case). + logger.debug(f"Pipeline task {self} is finishing...") + await self._cancel_tasks() + if self._check_dangling_tasks: + self._print_dangling_tasks() + self._finished = True + logger.debug(f"Pipeline task {self} has finished") async def queue_frame(self, frame: Frame): """Queue a single frame to be pushed down the pipeline. @@ -478,19 +475,7 @@ class PipelineTask(BasePipelineTask): if not self._cancelled: logger.debug(f"Cancelling pipeline task {self}") self._cancelled = True - cancel_frame = CancelFrame() - # Make sure everything is cleaned up downstream. This is sent - # out-of-band from the main streaming task which is what we want since - # we want to cancel right away. - await self._pipeline.queue_frame(cancel_frame) - # Wait for CancelFrame to make it through the pipeline. - await self._wait_for_pipeline_end(cancel_frame) - # Only cancel the push task, we don't want to be able to process any - # other frame after cancel. Everything else will be cancelled in - # run(). - if self._process_push_task: - await self._task_manager.cancel_task(self._process_push_task) - self._process_push_task = None + await self.queue_frame(CancelFrame()) async def _create_tasks(self): """Create and start all pipeline processing tasks.""" @@ -592,6 +577,17 @@ class PipelineTask(BasePipelineTask): self._pipeline_end_event.clear() + # We are really done. + self._pipeline_finished_event.set() + + async def _wait_for_pipeline_finished(self): + await self._pipeline_finished_event.wait() + self._pipeline_finished_event.clear() + # Make sure we wait for the main task to complete. + if self._process_push_task: + await self._process_push_task + self._process_push_task = None + async def _setup(self, params: PipelineTaskParams): """Set up the pipeline task and all processors.""" mgr_params = TaskManagerParams(loop=params.loop) @@ -694,12 +690,11 @@ class PipelineTask(BasePipelineTask): logger.debug(f"{self}: received interruption task frame {frame}") await self._pipeline.queue_frame(InterruptionFrame()) elif isinstance(frame, ErrorFrame): + await self._call_event_handler("on_pipeline_error", frame) if frame.fatal: logger.error(f"A fatal error occurred: {frame}") # Cancel all tasks downstream. await self.queue_frame(CancelFrame()) - # Tell the task we should stop. - await self.queue_frame(StopTaskFrame()) else: logger.warning(f"{self}: Something went wrong: {frame}") diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index 7bf83480d..98ff7c91e 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -189,7 +189,7 @@ class TaskObserver(BaseObserver): if isinstance(data, FramePushed): if on_push_frame_deprecated: await observer.on_push_frame( - data.src, data.dst, data.frame, data.direction, data.timestamp + data.source, data.destination, data.frame, data.direction, data.timestamp ) else: await observer.on_push_frame(data) diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 8b677cf02..913566909 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -15,9 +15,10 @@ service-specific adapter. """ import base64 +import copy import io from dataclasses import dataclass -from typing import Any, List, Optional, TypeAlias, Union +from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union from loguru import logger from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN @@ -31,6 +32,9 @@ from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.frames.frames import AudioRawFrame +if TYPE_CHECKING: + from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext + # "Re-export" types from OpenAI that we're using as universal context types. # NOTE: if universal message types need to someday diverge from OpenAI's, we # should consider managing our own definitions. But we should do so carefully, @@ -65,6 +69,26 @@ class LLMContext: and content formatting. """ + @staticmethod + def from_openai_context(openai_context: "OpenAILLMContext") -> "LLMContext": + """Create a universal LLM context from an OpenAI-specific context. + + NOTE: this should only be used internally, for facilitating migration + from OpenAILLMContext to LLMContext. New user code should use + LLMContext directly. + + Args: + openai_context: The OpenAI LLM context to convert. + + Returns: + New LLMContext instance with converted messages and settings. + """ + return LLMContext( + messages=openai_context.get_messages(), + tools=openai_context.tools, + tool_choice=openai_context.tool_choice, + ) + def __init__( self, messages: Optional[List[LLMContextMessage]] = None, @@ -82,6 +106,19 @@ class LLMContext: self._tools: ToolsSchema | NotGiven = LLMContext._normalize_and_validate_tools(tools) self._tool_choice: LLMContextToolChoice | NotGiven = tool_choice + @property + def messages(self) -> List[LLMContextMessage]: + """Get the current messages list. + + NOTE: This is equivalent to calling `get_messages()` with no filter. If + you want to filter out LLM-specific messages that don't pertain to your + LLM, use `get_messages()` directly. + + Returns: + List of conversation messages. + """ + return self.get_messages() + def get_messages(self, llm_specific_filter: Optional[str] = None) -> List[LLMContextMessage]: """Get the current messages list. @@ -89,7 +126,8 @@ class LLMContext: llm_specific_filter: Optional filter to return LLM-specific messages for the given LLM, in addition to the standard messages. If messages end up being filtered, an error will be - logged. + logged; this is intended to catch accidental use of + incompatible LLM-specific messages. Returns: List of conversation messages. diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index e663b81f4..556f2bc87 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -12,7 +12,7 @@ allowing for flexible frame filtering logic in processing pipelines. from typing import Awaitable, Callable -from pipecat.frames.frames import EndFrame, Frame, SystemFrame +from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -28,6 +28,7 @@ class FunctionFilter(FrameProcessor): self, filter: Callable[[Frame], Awaitable[bool]], direction: FrameDirection = FrameDirection.DOWNSTREAM, + filter_system_frames: bool = False, ): """Initialize the function filter. @@ -36,22 +37,32 @@ class FunctionFilter(FrameProcessor): frame should pass through, False otherwise. direction: The direction to apply filtering. Only frames moving in this direction will be filtered. Defaults to DOWNSTREAM. + filter_system_frames: Whether to filter system frames. Defaults to False. """ super().__init__() self._filter = filter self._direction = direction + self._filter_system_frames = filter_system_frames # # Frame processor # - # Ignore system frames, end frames and frames that are not following the - # direction of this gate def _should_passthrough_frame(self, frame, direction): """Check if a frame should pass through without filtering.""" - # Ignore system frames, end frames and frames that are not following the - # direction of this gate - return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction + # Always passthrough frames in the wrong direction + if direction != self._direction: + return True + + # Always passthrough lifecycle frames + if isinstance(frame, (StartFrame, EndFrame, CancelFrame)): + return True + + # If not filtering system frames, passthrough all other system frames + if not self._filter_system_frames and isinstance(frame, SystemFrame): + return True + + return False async def process_frame(self, frame: Frame, direction: FrameDirection): """Process a frame through the filter. diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 0b55082aa..1ca3333b5 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -877,6 +877,8 @@ class FrameProcessor(BaseObject): """ while True: + (frame, direction, callback) = await self.__input_queue.get() + if self.__should_block_system_frames and self.__input_event: logger.trace(f"{self}: system frame processing paused") await self.__input_event.wait() @@ -884,8 +886,6 @@ class FrameProcessor(BaseObject): self.__should_block_system_frames = False logger.trace(f"{self}: system frame processing resumed") - (frame, direction, callback) = await self.__input_queue.get() - if isinstance(frame, SystemFrame): await self.__process_frame(frame, direction, callback) elif self.__process_queue: @@ -900,6 +900,8 @@ class FrameProcessor(BaseObject): async def __process_frame_task_handler(self): """Handle non-system frames from the process queue.""" while True: + (frame, direction, callback) = await self.__process_queue.get() + if self.__should_block_frames and self.__process_event: logger.trace(f"{self}: frame processing paused") await self.__process_event.wait() @@ -907,8 +909,6 @@ class FrameProcessor(BaseObject): self.__should_block_frames = False logger.trace(f"{self}: frame processing resumed") - (frame, direction, callback) = await self.__process_queue.get() - await self.__process_frame(frame, direction, callback) self.__process_queue.task_done() diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 393cb7115..08d127ef6 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -1018,6 +1018,7 @@ class RTVIObserver(BaseObserver): if ( isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)) + and (direction == FrameDirection.DOWNSTREAM) and self._params.user_speaking_enabled ): await self._handle_interruptions(frame) diff --git a/src/pipecat/runner/daily.py b/src/pipecat/runner/daily.py index f52ccbc2b..568d54381 100644 --- a/src/pipecat/runner/daily.py +++ b/src/pipecat/runner/daily.py @@ -76,12 +76,14 @@ class DailyRoomConfig(BaseModel): async def configure( aiohttp_session: aiohttp.ClientSession, *, + api_key: Optional[str] = None, room_exp_duration: Optional[float] = 2.0, token_exp_duration: Optional[float] = 2.0, sip_caller_phone: Optional[str] = None, sip_enable_video: Optional[bool] = False, sip_num_endpoints: Optional[int] = 1, sip_codecs: Optional[Dict[str, List[str]]] = None, + room_properties: Optional[DailyRoomProperties] = None, ) -> DailyRoomConfig: """Configure Daily room URL and token with optional SIP capabilities. @@ -91,6 +93,7 @@ async def configure( Args: aiohttp_session: HTTP session for making API requests. + api_key: Daily API key. room_exp_duration: Room expiration time in hours. token_exp_duration: Token expiration time in hours. sip_caller_phone: Phone number or identifier for SIP display name. @@ -99,6 +102,10 @@ async def configure( sip_num_endpoints: Number of allowed SIP endpoints. sip_codecs: Codecs to support for audio and video. If None, uses Daily defaults. Example: {"audio": ["OPUS"], "video": ["H264"]} + room_properties: Optional DailyRoomProperties to use instead of building from + individual parameters. When provided, this overrides room_exp_duration and + SIP-related parameters. If not provided, properties are built from the + individual parameters as before. Returns: DailyRoomConfig: Object with room_url, token, and optional sip_endpoint. @@ -115,18 +122,48 @@ async def configure( # SIP-enabled room sip_config = await configure(session, sip_caller_phone="+15551234567") print(f"SIP endpoint: {sip_config.sip_endpoint}") + + # Custom room properties with recording enabled + custom_props = DailyRoomProperties( + enable_recording="cloud", + max_participants=2, + ) + config = await configure(session, room_properties=custom_props) """ # Check for required API key - api_key = os.getenv("DAILY_API_KEY") + api_key = api_key or os.getenv("DAILY_API_KEY") if not api_key: raise Exception( "DAILY_API_KEY environment variable is required. " "Get your API key from https://dashboard.daily.co/developers" ) + # Warn if both room_properties and individual parameters are provided + if room_properties is not None: + individual_params_provided = any( + [ + room_exp_duration != 2.0, + token_exp_duration != 2.0, + sip_caller_phone is not None, + sip_enable_video is not False, + sip_num_endpoints != 1, + sip_codecs is not None, + ] + ) + if individual_params_provided: + logger.warning( + "Both room_properties and individual parameters (room_exp_duration, token_exp_duration, " + "sip_*) were provided. The room_properties will be used and individual parameters " + "will be ignored." + ) + # Determine if SIP mode is enabled sip_enabled = sip_caller_phone is not None + # If room_properties is provided, check if it has SIP configuration + if room_properties and room_properties.sip: + sip_enabled = True + daily_rest_helper = DailyRESTHelper( daily_api_key=api_key, daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), @@ -150,27 +187,29 @@ async def configure( room_name = f"{room_prefix}-{uuid.uuid4().hex[:8]}" logger.info(f"Creating new Daily room: {room_name}") - # Calculate expiration time - expiration_time = time.time() + (room_exp_duration * 60 * 60) + # Use provided room_properties or build from parameters + if room_properties is None: + # Calculate expiration time + expiration_time = time.time() + (room_exp_duration * 60 * 60) - # Create room properties - room_properties = DailyRoomProperties( - exp=expiration_time, - eject_at_room_exp=True, - ) - - # Add SIP configuration if enabled - if sip_enabled: - sip_params = DailyRoomSipParams( - display_name=sip_caller_phone, - video=sip_enable_video, - sip_mode="dial-in", - num_endpoints=sip_num_endpoints, - codecs=sip_codecs, + # Create room properties + room_properties = DailyRoomProperties( + exp=expiration_time, + eject_at_room_exp=True, ) - room_properties.sip = sip_params - room_properties.enable_dialout = True # Enable outbound calls if needed - room_properties.start_video_off = not sip_enable_video # Voice-only by default + + # Add SIP configuration if enabled + if sip_enabled: + sip_params = DailyRoomSipParams( + display_name=sip_caller_phone, + video=sip_enable_video, + sip_mode="dial-in", + num_endpoints=sip_num_endpoints, + codecs=sip_codecs, + ) + room_properties.sip = sip_params + room_properties.enable_dialout = True # Enable outbound calls if needed + room_properties.start_video_off = not sip_enable_video # Voice-only by default # Create room parameters room_params = DailyRoomParams(name=room_name, properties=room_properties) diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index edcfbe9c4..28ca81bb9 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -67,16 +67,22 @@ To run locally: import argparse import asyncio +import mimetypes import os import sys +import uuid from contextlib import asynccontextmanager -from typing import Optional +from http import HTTPMethod +from pathlib import Path +from typing import Any, Dict, List, Optional, TypedDict import aiohttp +from fastapi.responses import FileResponse, Response from loguru import logger from pipecat.runner.types import ( DailyRunnerArguments, + RunnerArguments, SmallWebRTCRunnerArguments, WebSocketRunnerArguments, ) @@ -84,7 +90,7 @@ from pipecat.runner.types import ( try: import uvicorn from dotenv import load_dotenv - from fastapi import BackgroundTasks, FastAPI, HTTPException, Request, WebSocket + from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request, WebSocket from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, RedirectResponse except ImportError as e: @@ -98,6 +104,12 @@ except ImportError as e: load_dotenv(override=True) os.environ["ENV"] = "local" +TELEPHONY_TRANSPORTS = ["twilio", "telnyx", "plivo", "exotel"] + +RUNNER_DOWNLOADS_FOLDER: Optional[str] = None +RUNNER_HOST: str = "localhost" +RUNNER_PORT: int = 7860 + def _get_bot_module(): """Get the bot module from the calling script.""" @@ -152,7 +164,13 @@ async def _run_telephony_bot(websocket: WebSocket): def _create_server_app( - transport_type: str, host: str = "localhost", proxy: str = None, esp32_mode: bool = False + *, + transport_type: str, + host: str = "localhost", + proxy: str, + esp32_mode: bool = False, + whatsapp_enabled: bool = False, + folder: Optional[str] = None, ): """Create FastAPI app with transport-specific routes.""" app = FastAPI() @@ -167,25 +185,30 @@ def _create_server_app( # Set up transport-specific routes if transport_type == "webrtc": - _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host) - _setup_whatsapp_routes(app) + _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host, folder=folder) + if whatsapp_enabled: + _setup_whatsapp_routes(app) elif transport_type == "daily": _setup_daily_routes(app) - elif transport_type in ["twilio", "telnyx", "plivo", "exotel"]: - _setup_telephony_routes(app, transport_type, proxy) + elif transport_type in TELEPHONY_TRANSPORTS: + _setup_telephony_routes(app, transport_type=transport_type, proxy=proxy) else: logger.warning(f"Unknown transport type: {transport_type}") return app -def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "localhost"): +def _setup_webrtc_routes( + app: FastAPI, *, esp32_mode: bool = False, host: str = "localhost", folder: Optional[str] = None +): """Set up WebRTC-specific routes.""" try: from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI - from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection + from pipecat.transports.smallwebrtc.connection import IceServer, SmallWebRTCConnection from pipecat.transports.smallwebrtc.request_handler import ( + IceCandidate, + SmallWebRTCPatchRequest, SmallWebRTCRequest, SmallWebRTCRequestHandler, ) @@ -193,6 +216,16 @@ def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "lo logger.error(f"WebRTC transport dependencies not installed: {e}") return + class IceConfig(TypedDict): + iceServers: List[IceServer] + + class StartBotResult(TypedDict, total=False): + sessionId: str + iceConfig: Optional[IceConfig] + + # In-memory store of active sessions: session_id -> session info + active_sessions: Dict[str, Dict[str, Any]] = {} + # Mount the frontend app.mount("/client", SmallWebRTCPrebuiltUI) @@ -201,6 +234,21 @@ def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "lo """Redirect root requests to client interface.""" return RedirectResponse(url="/client/") + @app.get("/files/{filename:path}") + async def download_file(filename: str): + """Handle file downloads.""" + if not folder: + logger.warning(f"Attempting to dowload {filename}, but downloads folder not setup.") + return + + file_path = Path(folder) / filename + if not os.path.exists(file_path): + raise HTTPException(404) + + media_type, _ = mimetypes.guess_type(file_path) + + return FileResponse(path=file_path, media_type=media_type, filename=filename) + # Initialize the SmallWebRTC request handler small_webrtc_handler: SmallWebRTCRequestHandler = SmallWebRTCRequestHandler( esp32_mode=esp32_mode, host=host @@ -223,6 +271,74 @@ def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "lo ) return answer + @app.patch("/api/offer") + async def ice_candidate(request: SmallWebRTCPatchRequest): + """Handle WebRTC new ice candidate requests.""" + logger.debug(f"Received patch request: {request}") + await small_webrtc_handler.handle_patch_request(request) + return {"status": "success"} + + @app.post("/start") + async def rtvi_start(request: Request): + """Mimic Pipecat Cloud's /start endpoint.""" + # Parse the request body + try: + request_data = await request.json() + logger.debug(f"Received request: {request_data}") + except Exception as e: + logger.error(f"Failed to parse request body: {e}") + request_data = {} + + # Store session info immediately in memory, replicate the behavior expected on Pipecat Cloud + session_id = str(uuid.uuid4()) + active_sessions[session_id] = request_data + + result: StartBotResult = {"sessionId": session_id} + if request_data.get("enableDefaultIceServers"): + result["iceConfig"] = IceConfig( + iceServers=[IceServer(urls="stun:stun.l.google.com:19302")] + ) + + return result + + @app.api_route( + "/sessions/{session_id}/{path:path}", + methods=["GET", "POST", "PUT", "PATCH", "DELETE"], + ) + async def proxy_request( + session_id: str, path: str, request: Request, background_tasks: BackgroundTasks + ): + """Mimic Pipecat Cloud's proxy.""" + active_session = active_sessions.get(session_id) + if active_session is None: + return Response(content="Invalid or not-yet-ready session_id", status_code=404) + + if path.endswith("api/offer"): + # Parse the request body and convert to SmallWebRTCRequest + try: + request_data = await request.json() + if request.method == HTTPMethod.POST.value: + webrtc_request = SmallWebRTCRequest( + sdp=request_data["sdp"], + type=request_data["type"], + pc_id=request_data.get("pc_id"), + restart_pc=request_data.get("restart_pc"), + request_data=request_data, + ) + return await offer(webrtc_request, background_tasks) + elif request.method == HTTPMethod.PATCH.value: + patch_request = SmallWebRTCPatchRequest( + pc_id=request_data["pc_id"], + candidates=[IceCandidate(**c) for c in request_data.get("candidates", [])], + ) + return await ice_candidate(patch_request) + except Exception as e: + logger.error(f"Failed to parse WebRTC request: {e}") + return Response(content="Invalid WebRTC request", status_code=400) + + logger.info(f"Received request for path: {path}") + return Response(status_code=200) + @asynccontextmanager async def smallwebrtc_lifespan(app: FastAPI): """Manage FastAPI application lifecycle and cleanup connections.""" @@ -258,6 +374,29 @@ def _add_lifespan_to_app(app: FastAPI, new_lifespan): def _setup_whatsapp_routes(app: FastAPI): """Set up WebRTC-specific routes.""" + WHATSAPP_APP_SECRET = os.getenv("WHATSAPP_APP_SECRET") + WHATSAPP_PHONE_NUMBER_ID = os.getenv("WHATSAPP_PHONE_NUMBER_ID") + WHATSAPP_TOKEN = os.getenv("WHATSAPP_TOKEN") + WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFICATION_TOKEN") + + if not all( + [ + WHATSAPP_APP_SECRET, + WHATSAPP_PHONE_NUMBER_ID, + WHATSAPP_TOKEN, + WHATSAPP_WEBHOOK_VERIFICATION_TOKEN, + ] + ): + logger.error( + """Missing required environment variables for WhatsApp transport: + WHATSAPP_APP_SECRET + WHATSAPP_PHONE_NUMBER_ID + WHATSAPP_TOKEN + WHATSAPP_WEBHOOK_VERIFICATION_TOKEN + """ + ) + return + try: from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI @@ -269,23 +408,7 @@ def _setup_whatsapp_routes(app: FastAPI): from pipecat.transports.whatsapp.api import WhatsAppWebhookRequest from pipecat.transports.whatsapp.client import WhatsAppClient except ImportError as e: - logger.error(f"WebRTC transport dependencies not installed: {e}") - return - - WHATSAPP_TOKEN = os.getenv("WHATSAPP_TOKEN") - WHATSAPP_PHONE_NUMBER_ID = os.getenv("WHATSAPP_PHONE_NUMBER_ID") - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFICATION_TOKEN") - - if not all( - [ - WHATSAPP_TOKEN, - WHATSAPP_PHONE_NUMBER_ID, - WHATSAPP_WEBHOOK_VERIFICATION_TOKEN, - ] - ): - logger.debug( - "Missing required environment variables for WhatsApp transport. Keeping it disabled." - ) + logger.error(f"WhatsApp transport dependencies not installed: {e}") return # Global WhatsApp client instance @@ -325,7 +448,12 @@ def _setup_whatsapp_routes(app: FastAPI): summary="Handle WhatsApp webhook events", description="Processes incoming WhatsApp messages and call events", ) - async def whatsapp_webhook(body: WhatsAppWebhookRequest, background_tasks: BackgroundTasks): + async def whatsapp_webhook( + body: WhatsAppWebhookRequest, + background_tasks: BackgroundTasks, + request: Request, + x_hub_signature_256: str = Header(None), + ): """Handle incoming WhatsApp webhook events. For call events, establishes WebRTC connections and spawns bot instances @@ -357,7 +485,10 @@ def _setup_whatsapp_routes(app: FastAPI): try: # Process the webhook request - result = await whatsapp_client.handle_webhook_request(body, connection_callback) + raw_body = await request.body() + result = await whatsapp_client.handle_webhook_request( + body, connection_callback, sha256_signature=x_hub_signature_256, raw_body=raw_body + ) logger.debug(f"Webhook processed successfully: {result}") return {"status": "success", "message": "Webhook processed successfully"} except ValueError as ve: @@ -376,6 +507,7 @@ def _setup_whatsapp_routes(app: FastAPI): async with aiohttp.ClientSession() as session: whatsapp_client = WhatsAppClient( whatsapp_token=WHATSAPP_TOKEN, + whatsapp_secret=WHATSAPP_APP_SECRET, phone_number_id=WHATSAPP_PHONE_NUMBER_ID, session=session, ) @@ -398,9 +530,9 @@ def _setup_daily_routes(app: FastAPI): """Set up Daily-specific routes.""" @app.get("/") - async def start_agent(): + async def create_room_and_start_agent(): """Launch a Daily bot and redirect to room.""" - print("Starting bot with Daily transport") + print("Starting bot with Daily transport and redirecting to Daily room") import aiohttp @@ -415,11 +547,11 @@ def _setup_daily_routes(app: FastAPI): asyncio.create_task(bot_module.bot(runner_args)) return RedirectResponse(room_url) - async def _handle_rtvi_request(request: Request): - """Common handler for both /start and /connect endpoints. + @app.post("/start") + async def start_agent(request: Request): + """Handler for /start endpoints. Expects POST body like:: - { "createDailyRoom": true, "dailyRoomProperties": { "start_video_off": true }, @@ -436,50 +568,41 @@ def _setup_daily_routes(app: FastAPI): logger.error(f"Failed to parse request body: {e}") request_data = {} - # Extract the body data that should be passed to the bot - # This mimics Pipecat Cloud's behavior - bot_body = request_data.get("body", {}) + create_daily_room = request_data.get("createDailyRoom", False) + body = request_data.get("body", {}) - # Log the extracted body data for debugging - if bot_body: - logger.info(f"Extracted body data for bot: {bot_body}") + bot_module = _get_bot_module() + + existing_room_url = os.getenv("DAILY_SAMPLE_ROOM_URL") + + result = None + + # Configure room if: + # 1. Explicitly requested via createDailyRoom in payload + # 2. Using pre-configured room from DAILY_SAMPLE_ROOM_URL env var + if create_daily_room or existing_room_url: + import aiohttp + + from pipecat.runner.daily import configure + + async with aiohttp.ClientSession() as session: + room_url, token = await configure(session) + runner_args = DailyRunnerArguments(room_url=room_url, token=token, body=body) + result = { + "dailyRoom": room_url, + "dailyToken": token, + "sessionId": str(uuid.uuid4()), + } else: - logger.debug("No body data provided in request") + runner_args = RunnerArguments(body=body) - import aiohttp + # Start the bot in the background + asyncio.create_task(bot_module.bot(runner_args)) - from pipecat.runner.daily import configure - - async with aiohttp.ClientSession() as session: - room_url, token = await configure(session) - - # Start the bot in the background with extracted body data - bot_module = _get_bot_module() - runner_args = DailyRunnerArguments(room_url=room_url, token=token, body=bot_body) - asyncio.create_task(bot_module.bot(runner_args)) - # Match PCC /start endpoint response format: - return {"dailyRoom": room_url, "dailyToken": token} - - @app.post("/start") - async def rtvi_start(request: Request): - """Launch a Daily bot and return connection info for RTVI clients.""" - return await _handle_rtvi_request(request) - - @app.post("/connect") - async def rtvi_connect(request: Request): - """Launch a Daily bot and return connection info for RTVI clients. - - .. deprecated:: 0.0.78 - Use /start instead. This endpoint will be removed in a future version. - """ - logger.warning( - "DEPRECATED: /connect endpoint is deprecated. Please use /start instead. " - "This endpoint will be removed in a future version." - ) - return await _handle_rtvi_request(request) + return result -def _setup_telephony_routes(app: FastAPI, transport_type: str, proxy: str): +def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str): """Set up telephony-specific routes.""" # XML response templates (Exotel doesn't use XML webhooks) XML_TEMPLATES = { @@ -535,8 +658,6 @@ def _setup_telephony_routes(app: FastAPI, transport_type: str, proxy: str): async def _run_daily_direct(): """Run Daily bot with direct connection (no FastAPI server).""" try: - import aiohttp - from pipecat.runner.daily import configure except ImportError as e: logger.error("Daily transport dependencies not installed.") @@ -582,6 +703,21 @@ def _validate_and_clean_proxy(proxy: str) -> str: return proxy +def runner_downloads_folder() -> Optional[str]: + """Returns the folder where files are stored for later download.""" + return RUNNER_DOWNLOADS_FOLDER + + +def runner_host() -> str: + """Returns the host name of this runner.""" + return RUNNER_HOST + + +def runner_port() -> int: + """Returns the port of this runner.""" + return RUNNER_PORT + + def main(): """Start the Pipecat development runner. @@ -602,14 +738,16 @@ def main(): The bot file must contain a `bot(runner_args)` function as the entry point. """ + global RUNNER_DOWNLOADS_FOLDER, RUNNER_HOST, RUNNER_PORT + parser = argparse.ArgumentParser(description="Pipecat Development Runner") - parser.add_argument("--host", type=str, default="localhost", help="Host address") - parser.add_argument("--port", type=int, default=7860, help="Port number") + parser.add_argument("--host", type=str, default=RUNNER_HOST, help="Host address") + parser.add_argument("--port", type=int, default=RUNNER_PORT, help="Port number") parser.add_argument( "-t", "--transport", type=str, - choices=["daily", "webrtc", "twilio", "telnyx", "plivo", "exotel"], + choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS], default="webrtc", help="Transport type", ) @@ -627,9 +765,16 @@ def main(): default=False, help="Connect directly to Daily room (automatically sets transport to daily)", ) + parser.add_argument("-f", "--folder", type=str, help="Path to downloads folder") parser.add_argument( "--verbose", "-v", action="count", default=0, help="Increase logging verbosity" ) + parser.add_argument( + "--whatsapp", + action="store_true", + default=False, + help="Ensure requried WhatsApp environment variables are present", + ) args = parser.parse_args() @@ -668,10 +813,11 @@ def main(): print() if args.esp32: print(f"πŸš€ Bot ready! (ESP32 mode)") - print(f" β†’ Open http://{args.host}:{args.port}/client in your browser") + elif args.whatsapp: + print(f"πŸš€ Bot ready! (WhatsApp)") else: print(f"πŸš€ Bot ready!") - print(f" β†’ Open http://{args.host}:{args.port}/client in your browser") + print(f" β†’ Open http://{args.host}:{args.port}/client in your browser") print() elif args.transport == "daily": print() @@ -679,8 +825,19 @@ def main(): print(f" β†’ Open http://{args.host}:{args.port} in your browser to start a session") print() + RUNNER_DOWNLOADS_FOLDER = args.folder + RUNNER_HOST = args.host + RUNNER_PORT = args.port + # Create the app with transport-specific setup - app = _create_server_app(args.transport, args.host, args.proxy, args.esp32) + app = _create_server_app( + transport_type=args.transport, + host=args.host, + proxy=args.proxy, + esp32_mode=args.esp32, + whatsapp_enabled=args.whatsapp, + folder=args.folder, + ) # Run the server uvicorn.run(app, host=args.host, port=args.port) diff --git a/src/pipecat/runner/types.py b/src/pipecat/runner/types.py index 89aecd84c..fab2404f3 100644 --- a/src/pipecat/runner/types.py +++ b/src/pipecat/runner/types.py @@ -20,9 +20,11 @@ from fastapi import WebSocket class RunnerArguments: """Base class for runner session arguments.""" - handle_sigint: bool = field(init=False) - handle_sigterm: bool = field(init=False) - pipeline_idle_timeout_secs: int = field(init=False) + # Use kw_only so subclasses don't need to worry about ordering. + handle_sigint: bool = field(init=False, kw_only=True) + handle_sigterm: bool = field(init=False, kw_only=True) + pipeline_idle_timeout_secs: int = field(init=False, kw_only=True) + body: Optional[Any] = field(default_factory=dict, kw_only=True) def __post_init__(self): self.handle_sigint = False @@ -42,7 +44,6 @@ class DailyRunnerArguments(RunnerArguments): room_url: str token: Optional[str] = None - body: Optional[Any] = field(default_factory=dict) @dataclass @@ -55,7 +56,6 @@ class WebSocketRunnerArguments(RunnerArguments): """ websocket: WebSocket - body: Optional[Any] = field(default_factory=dict) @dataclass diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py index 3d4188960..f3a34c434 100644 --- a/src/pipecat/serializers/livekit.py +++ b/src/pipecat/serializers/livekit.py @@ -25,11 +25,31 @@ except ModuleNotFoundError as e: class LivekitFrameSerializer(FrameSerializer): """Serializer for converting between Pipecat frames and LiveKit audio frames. + .. deprecated:: 0.0.90 + + This class is deprecated and will be removed in a future version. + Please use LiveKitTransport instead, which handles audio streaming + and frame conversion natively. + This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects to LiveKit AudioFrame objects for transmission, and the reverse conversion for received audio data. """ + def __init__(self): + """Initialize the LiveKit frame serializer.""" + super().__init__() + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "LivekitFrameSerializer is deprecated and will be removed in a future version. " + "Please use LiveKitTransport instead, which handles audio streaming natively.", + DeprecationWarning, + stacklevel=2, + ) + @property def type(self) -> FrameSerializerType: """Get the serializer type. diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index eebdbfb0b..759efcf00 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -97,9 +97,7 @@ class AIService(FrameProcessor): pass async def _update_settings(self, settings: Mapping[str, Any]): - from pipecat.services.openai_realtime_beta.events import ( - SessionProperties, - ) + from pipecat.services.openai.realtime.events import SessionProperties for key, value in settings.items(): logger.debug("Update request for:", key, value) @@ -111,9 +109,7 @@ class AIService(FrameProcessor): logger.debug("Attempting to update", key, value) try: - from pipecat.services.openai_realtime_beta.events import ( - TurnDetection, - ) + from pipecat.services.openai.realtime.events import TurnDetection if isinstance(self._session_properties, SessionProperties): current_properties = self._session_properties diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index b34ec554d..52ea87d87 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -108,6 +108,8 @@ class AssemblyAIConnectionParams(BaseModel): end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection. min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn. max_turn_silence: Maximum silence duration before forcing end-of-turn. + keyterms_prompt: List of key terms to guide transcription. Will be JSON serialized before sending. + speech_model: Select between English and multilingual models. Defaults to "universal-streaming-english". """ sample_rate: int = 16000 @@ -117,3 +119,7 @@ class AssemblyAIConnectionParams(BaseModel): end_of_turn_confidence_threshold: Optional[float] = None min_end_of_turn_silence_when_confident: Optional[int] = None max_turn_silence: Optional[int] = None + keyterms_prompt: Optional[List[str]] = None + speech_model: Literal["universal-streaming-english", "universal-streaming-multilingual"] = ( + "universal-streaming-english" + ) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index aa2fc36bc..b3f20800c 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -174,11 +174,16 @@ class AssemblyAISTTService(STTService): def _build_ws_url(self) -> str: """Build WebSocket URL with query parameters using urllib.parse.urlencode.""" - params = { - k: str(v).lower() if isinstance(v, bool) else v - for k, v in self._connection_params.model_dump().items() - if v is not None - } + params = {} + for k, v in self._connection_params.model_dump().items(): + if v is not None: + if k == "keyterms_prompt": + params[k] = json.dumps(v) + elif isinstance(v, bool): + params[k] = str(v).lower() + else: + params[k] = v + if params: query_string = urlencode(params) return f"{self._api_endpoint_base_url}?{query_string}" @@ -197,6 +202,8 @@ class AssemblyAISTTService(STTService): ) self._connected = True self._receive_task = self.create_task(self._receive_task_handler()) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"Failed to connect to AssemblyAI: {e}") self._connected = False @@ -238,6 +245,7 @@ class AssemblyAISTTService(STTService): self._websocket = None self._connected = False self._receive_task = None + await self._call_event_handler("on_disconnected") async def _receive_task_handler(self): """Handle incoming WebSocket messages.""" diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index b0ddf9275..3e4ff33cc 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -235,6 +235,8 @@ class AsyncAITTSService(InterruptibleTTSService): } await self._get_websocket().send(json.dumps(init_msg)) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -252,6 +254,7 @@ class AsyncAITTSService(InterruptibleTTSService): finally: self._websocket = None self._started = False + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: diff --git a/src/pipecat/services/aws/__init__.py b/src/pipecat/services/aws/__init__.py index b1f157bd3..3cdd4cc5a 100644 --- a/src/pipecat/services/aws/__init__.py +++ b/src/pipecat/services/aws/__init__.py @@ -9,6 +9,7 @@ import sys from pipecat.services import DeprecatedModuleProxy from .llm import * +from .nova_sonic import * from .stt import * from .tts import * diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 377848f76..716aee776 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -720,11 +720,11 @@ class AWSBedrockLLMService(LLMService): additional_model_request_fields: Additional model-specific parameters. """ - max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1) - temperature: Optional[float] = Field(default_factory=lambda: 0.7, ge=0.0, le=1.0) - top_p: Optional[float] = Field(default_factory=lambda: 0.999, ge=0.0, le=1.0) + max_tokens: Optional[int] = Field(default=None, ge=1) + temperature: Optional[float] = Field(default=None, ge=0.0, le=1.0) + top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) stop_sequences: Optional[List[str]] = Field(default_factory=lambda: []) - latency: Optional[str] = Field(default_factory=lambda: "standard") + latency: Optional[str] = Field(default=None) additional_model_request_fields: Optional[Dict[str, Any]] = Field(default_factory=dict) def __init__( @@ -801,6 +801,24 @@ class AWSBedrockLLMService(LLMService): """ return True + def _build_inference_config(self) -> Dict[str, Any]: + """Build inference config with only the parameters that are set. + + This prevents conflicts with models (e.g., Claude Sonnet 4.5) that don't + allow certain parameter combinations like temperature and top_p together. + + Returns: + Dictionary containing only the inference parameters that are not None. + """ + inference_config = {} + if self._settings["max_tokens"] is not None: + inference_config["maxTokens"] = self._settings["max_tokens"] + if self._settings["temperature"] is not None: + inference_config["temperature"] = self._settings["temperature"] + if self._settings["top_p"] is not None: + inference_config["topP"] = self._settings["top_p"] + return inference_config + async def run_inference(self, context: LLMContext | OpenAILLMContext) -> Optional[str]: """Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context. @@ -826,16 +844,16 @@ class AWSBedrockLLMService(LLMService): model_id = self.model_name # Prepare request parameters + inference_config = self._build_inference_config() + request_params = { "modelId": model_id, "messages": messages, - "inferenceConfig": { - "maxTokens": 8192, - "temperature": 0.7, - "topP": 0.9, - }, } + if inference_config: + request_params["inferenceConfig"] = inference_config + if system: request_params["system"] = system @@ -974,21 +992,20 @@ class AWSBedrockLLMService(LLMService): tools = params_from_context["tools"] tool_choice = params_from_context["tool_choice"] - # Set up inference config - inference_config = { - "maxTokens": self._settings["max_tokens"], - "temperature": self._settings["temperature"], - "topP": self._settings["top_p"], - } + # Set up inference config - only include parameters that are set + inference_config = self._build_inference_config() # Prepare request parameters request_params = { "modelId": self.model_name, "messages": messages, - "inferenceConfig": inference_config, "additionalModelRequestFields": self._settings["additional_model_request_fields"], } + # Only add inference config if it has parameters + if inference_config: + request_params["inferenceConfig"] = inference_config + # Add system message if system: request_params["system"] = system diff --git a/src/pipecat/services/aws/nova_sonic/__init__.py b/src/pipecat/services/aws/nova_sonic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/aws/nova_sonic/context.py b/src/pipecat/services/aws/nova_sonic/context.py new file mode 100644 index 000000000..c9aab439f --- /dev/null +++ b/src/pipecat/services/aws/nova_sonic/context.py @@ -0,0 +1,436 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Context management for AWS Nova Sonic LLM service. + +This module provides specialized context aggregators and message handling for AWS Nova Sonic, +including conversation history management and role-specific message processing. + +.. deprecated:: 0.0.91 + AWS Nova Sonic no longer uses types from this module under the hood. + It now uses `LLMContext` and `LLMContextAggregatorPair`. + Using the new patterns should allow you to not need types from this module. + + BEFORE: + ``` + # Setup + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + # Context frame type + frame: OpenAILLMContextFrame + + # Context type + context: AWSNovaSonicLLMContext + # or + context: OpenAILLMContext + ``` + + AFTER: + ``` + # Setup + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + # Context frame type + frame: LLMContextFrame + + # Context type + context: LLMContext + ``` +""" + +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws.nova_sonic.context (or " + "pipecat.services.aws_nova_sonic.context) are deprecated. \n" + "AWS Nova Sonic no longer uses types from this module under the hood. \n" + "It now uses `LLMContext` and `LLMContextAggregatorPair`. \n" + "Using the new patterns should allow you to not need types from this module.\n\n" + "BEFORE:\n" + "```\n" + "# Setup\n" + "context = OpenAILLMContext(messages, tools)\n" + "context_aggregator = llm.create_context_aggregator(context)\n\n" + "# Context frame type\n" + "frame: OpenAILLMContextFrame\n\n" + "# Context type\n" + "context: AWSNovaSonicLLMContext\n" + "# or\n" + "context: OpenAILLMContext\n\n" + "```\n\n" + "AFTER:\n" + "```\n" + "# Setup\n" + "context = LLMContext(messages, tools)\n" + "context_aggregator = LLMContextAggregatorPair(context)\n\n" + "# Context frame type\n" + "frame: LLMContextFrame\n\n" + "# Context type\n" + "context: LLMContext\n\n" + "```", + DeprecationWarning, + stacklevel=2, + ) + +import copy +from dataclasses import dataclass, field +from enum import Enum + +from loguru import logger + +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + DataFrame, + Frame, + FunctionCallResultFrame, + InterruptionFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolChoiceFrame, + LLMSetToolsFrame, + TextFrame, + UserImageRawFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) + + +class Role(Enum): + """Roles supported in AWS Nova Sonic conversations. + + Parameters: + SYSTEM: System-level messages (not used in conversation history). + USER: Messages sent by the user. + ASSISTANT: Messages sent by the assistant. + TOOL: Messages sent by tools (not used in conversation history). + """ + + SYSTEM = "SYSTEM" + USER = "USER" + ASSISTANT = "ASSISTANT" + TOOL = "TOOL" + + +@dataclass +class AWSNovaSonicConversationHistoryMessage: + """A single message in AWS Nova Sonic conversation history. + + Parameters: + role: The role of the message sender (USER or ASSISTANT only). + text: The text content of the message. + """ + + role: Role # only USER and ASSISTANT + text: str + + +@dataclass +class AWSNovaSonicConversationHistory: + """Complete conversation history for AWS Nova Sonic initialization. + + Parameters: + system_instruction: System-level instruction for the conversation. + messages: List of conversation messages between user and assistant. + """ + + system_instruction: str = None + messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) + + +class AWSNovaSonicLLMContext(OpenAILLMContext): + """Specialized LLM context for AWS Nova Sonic service. + + Extends OpenAI context with Nova Sonic-specific message handling, + conversation history management, and text buffering capabilities. + """ + + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize AWS Nova Sonic LLM context. + + Args: + messages: Initial messages for the context. + tools: Available tools for the context. + **kwargs: Additional arguments passed to parent class. + """ + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self, system_instruction: str = ""): + self._assistant_text = "" + self._user_text = "" + self._system_instruction = system_instruction + + @staticmethod + def upgrade_to_nova_sonic( + obj: OpenAILLMContext, system_instruction: str + ) -> "AWSNovaSonicLLMContext": + """Upgrade an OpenAI context to AWS Nova Sonic context. + + Args: + obj: The OpenAI context to upgrade. + system_instruction: System instruction for the context. + + Returns: + The upgraded AWS Nova Sonic context. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): + obj.__class__ = AWSNovaSonicLLMContext + obj.__setup_local(system_instruction) + return obj + + # NOTE: this method has the side-effect of updating _system_instruction from messages + def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: + """Get conversation history for initializing AWS Nova Sonic session. + + Processes stored messages and extracts system instruction and conversation + history in the format expected by AWS Nova Sonic. + + Returns: + Formatted conversation history with system instruction and messages. + """ + history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) + + # Bail if there are no messages + if not self.messages: + return history + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into "instruction" + if messages[0].get("role") == "system": + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + history.system_instruction = content + elif isinstance(content, list): + history.system_instruction = content[0].get("text") + if history.system_instruction: + self._system_instruction = history.system_instruction + + # Process remaining messages to fill out conversation history. + # Nova Sonic supports "user" and "assistant" messages in history. + for message in messages: + history_message = self.from_standard_message(message) + if history_message: + history.messages.append(history_message) + + return history + + def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Returns: + List of messages including system instruction if present. + """ + messages = super().get_messages_for_persistent_storage() + # If we have a system instruction and messages doesn't already contain it, add it + if self._system_instruction and not (messages and messages[0].get("role") == "system"): + messages.insert(0, {"role": "system", "content": self._system_instruction}) + return messages + + def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + """Convert standard message format to Nova Sonic format. + + Args: + message: Standard message dictionary to convert. + + Returns: + Nova Sonic conversation history message, or None if not convertible. + """ + role = message.get("role") + if message.get("role") == "user" or message.get("role") == "assistant": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + # There won't be content if this is an assistant tool call entry. + # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation + # history + if content: + return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova + # Sonic conversation history + + def buffer_user_text(self, text): + """Buffer user text for later flushing to context. + + Args: + text: User text to buffer. + """ + self._user_text += f" {text}" if self._user_text else text + # logger.debug(f"User text buffered: {self._user_text}") + + def flush_aggregated_user_text(self) -> str: + """Flush buffered user text to context as a complete message. + + Returns: + The flushed user text, or empty string if no text was buffered. + """ + if not self._user_text: + return "" + user_text = self._user_text + message = { + "role": "user", + "content": [{"type": "text", "text": user_text}], + } + self._user_text = "" + self.add_message(message) + # logger.debug(f"Context updated (user): {self.get_messages_for_logging()}") + return user_text + + def buffer_assistant_text(self, text): + """Buffer assistant text for later flushing to context. + + Args: + text: Assistant text to buffer. + """ + self._assistant_text += text + # logger.debug(f"Assistant text buffered: {self._assistant_text}") + + def flush_aggregated_assistant_text(self): + """Flush buffered assistant text to context as a complete message.""" + if not self._assistant_text: + return + message = { + "role": "assistant", + "content": [{"type": "text", "text": self._assistant_text}], + } + self._assistant_text = "" + self.add_message(message) + # logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}") + + +@dataclass +class AWSNovaSonicMessagesUpdateFrame(DataFrame): + """Frame containing updated AWS Nova Sonic context. + + Parameters: + context: The updated AWS Nova Sonic LLM context. + """ + + context: AWSNovaSonicLLMContext + + +class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): + """Context aggregator for user messages in AWS Nova Sonic conversations. + + Extends the OpenAI user context aggregator to emit Nova Sonic-specific + context update frames. + """ + + async def process_frame( + self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM + ): + """Process frames and emit Nova Sonic-specific context updates. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + await super().process_frame(frame, direction) + + # Parent does not push LLMMessagesUpdateFrame + if isinstance(frame, LLMMessagesUpdateFrame): + await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) + + +class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Context aggregator for assistant messages in AWS Nova Sonic conversations. + + Provides specialized handling for assistant responses and function calls + in AWS Nova Sonic context, with custom frame processing logic. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Nova Sonic-specific logic. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + # HACK: For now, disable the context aggregator by making it just pass through all frames + # that the parent handles (except the function call stuff, which we still need). + # For an explanation of this hack, see + # AWSNovaSonicLLMService._report_assistant_response_text_added. + if isinstance( + frame, + ( + InterruptionFrame, + LLMFullResponseStartFrame, + LLMFullResponseEndFrame, + TextFrame, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMSetToolChoiceFrame, + UserImageRawFrame, + BotStoppedSpeakingFrame, + ), + ): + await self.push_frame(frame, direction) + else: + await super().process_frame(frame, direction) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call results for AWS Nova Sonic. + + Args: + frame: The function call result frame to handle. + """ + await super().handle_function_call_result(frame) + + # The standard function callback code path pushes the FunctionCallResultFrame from the LLM + # itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side + # context. Let's push a special frame to do that. + await self.push_frame( + AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) + + +@dataclass +class AWSNovaSonicContextAggregatorPair: + """Pair of user and assistant context aggregators for AWS Nova Sonic. + + Parameters: + _user: The user context aggregator. + _assistant: The assistant context aggregator. + """ + + _user: AWSNovaSonicUserContextAggregator + _assistant: AWSNovaSonicAssistantContextAggregator + + def user(self) -> AWSNovaSonicUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ + return self._user + + def assistant(self) -> AWSNovaSonicAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ + return self._assistant diff --git a/src/pipecat/services/aws/nova_sonic/frames.py b/src/pipecat/services/aws/nova_sonic/frames.py new file mode 100644 index 000000000..7d4feb2ae --- /dev/null +++ b/src/pipecat/services/aws/nova_sonic/frames.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Custom frames for AWS Nova Sonic LLM service.""" + +from dataclasses import dataclass + +from pipecat.frames.frames import DataFrame, FunctionCallResultFrame + + +@dataclass +class AWSNovaSonicFunctionCallResultFrame(DataFrame): + """Frame containing function call result for AWS Nova Sonic processing. + + This frame wraps a standard function call result frame to enable + AWS Nova Sonic-specific handling and context updates. + + Parameters: + result_frame: The underlying function call result frame. + """ + + result_frame: FunctionCallResultFrame diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py new file mode 100644 index 000000000..5df3bbd21 --- /dev/null +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -0,0 +1,1265 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AWS Nova Sonic LLM service implementation for Pipecat AI framework. + +This module provides a speech-to-speech LLM service using AWS Nova Sonic, which supports +bidirectional audio streaming, text generation, and function calling capabilities. +""" + +import asyncio +import base64 +import json +import time +import uuid +import wave +from dataclasses import dataclass +from enum import Enum +from importlib.resources import files +from typing import Any, List, Optional + +from loguru import logger +from pydantic import BaseModel, Field + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter, Role +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + Frame, + FunctionCallFromLLM, + InputAudioRawFrame, + InterruptionFrame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + StartFrame, + TranscriptionFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, + TTSTextFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, +) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response import ( + LLMAssistantAggregatorParams, + LLMUserAggregatorParams, +) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.llm_service import LLMService +from pipecat.utils.time import time_now_iso8601 + +try: + from aws_sdk_bedrock_runtime.client import ( + BedrockRuntimeClient, + InvokeModelWithBidirectionalStreamOperationInput, + ) + from aws_sdk_bedrock_runtime.config import Config + from aws_sdk_bedrock_runtime.models import ( + BidirectionalInputPayloadPart, + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamInputChunk, + InvokeModelWithBidirectionalStreamOperationOutput, + InvokeModelWithBidirectionalStreamOutput, + ) + from smithy_aws_core.auth.sigv4 import SigV4AuthScheme + from smithy_aws_core.identity.static import StaticCredentialsResolver + from smithy_core.aio.eventstream import DuplexEventStream +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use AWS services, you need to `pip install pipecat-ai[aws-nova-sonic]`." + ) + raise Exception(f"Missing module: {e}") + + +class AWSNovaSonicUnhandledFunctionException(Exception): + """Exception raised when the LLM attempts to call an unregistered function.""" + + pass + + +class ContentType(Enum): + """Content types supported by AWS Nova Sonic. + + Parameters: + AUDIO: Audio content type. + TEXT: Text content type. + TOOL: Tool content type. + """ + + AUDIO = "AUDIO" + TEXT = "TEXT" + TOOL = "TOOL" + + +class TextStage(Enum): + """Text generation stages in AWS Nova Sonic responses. + + Parameters: + FINAL: Final text that has been fully generated. + SPECULATIVE: Speculative text that is still being generated. + """ + + FINAL = "FINAL" # what has been said + SPECULATIVE = "SPECULATIVE" # what's planned to be said + + +@dataclass +class CurrentContent: + """Represents content currently being received from AWS Nova Sonic. + + Parameters: + type: The type of content (audio, text, or tool). + role: The role generating the content (user, assistant, etc.). + text_stage: The stage of text generation (final or speculative). + text_content: The actual text content if applicable. + """ + + type: ContentType + role: Role + text_stage: TextStage # None if not text + text_content: str # starts as None, then fills in if text + + def __str__(self): + """String representation of the current content.""" + return ( + f"CurrentContent(\n" + f" type={self.type.name},\n" + f" role={self.role.name},\n" + f" text_stage={self.text_stage.name if self.text_stage else 'None'}\n" + f")" + ) + + +class Params(BaseModel): + """Configuration parameters for AWS Nova Sonic. + + Parameters: + input_sample_rate: Audio input sample rate in Hz. + input_sample_size: Audio input sample size in bits. + input_channel_count: Number of input audio channels. + output_sample_rate: Audio output sample rate in Hz. + output_sample_size: Audio output sample size in bits. + output_channel_count: Number of output audio channels. + max_tokens: Maximum number of tokens to generate. + top_p: Nucleus sampling parameter. + temperature: Sampling temperature for text generation. + """ + + # Audio input + input_sample_rate: Optional[int] = Field(default=16000) + input_sample_size: Optional[int] = Field(default=16) + input_channel_count: Optional[int] = Field(default=1) + + # Audio output + output_sample_rate: Optional[int] = Field(default=24000) + output_sample_size: Optional[int] = Field(default=16) + output_channel_count: Optional[int] = Field(default=1) + + # Inference + max_tokens: Optional[int] = Field(default=1024) + top_p: Optional[float] = Field(default=0.9) + temperature: Optional[float] = Field(default=0.7) + + +class AWSNovaSonicLLMService(LLMService): + """AWS Nova Sonic speech-to-speech LLM service. + + Provides bidirectional audio streaming, real-time transcription, text generation, + and function calling capabilities using AWS Nova Sonic model. + """ + + # Override the default adapter to use the AWSNovaSonicLLMAdapter one + adapter_class = AWSNovaSonicLLMAdapter + + def __init__( + self, + *, + secret_access_key: str, + access_key_id: str, + session_token: Optional[str] = None, + region: str, + model: str = "amazon.nova-sonic-v1:0", + voice_id: str = "matthew", # matthew, tiffany, amy + params: Optional[Params] = None, + system_instruction: Optional[str] = None, + tools: Optional[ToolsSchema] = None, + send_transcription_frames: bool = True, + **kwargs, + ): + """Initializes the AWS Nova Sonic LLM service. + + Args: + secret_access_key: AWS secret access key for authentication. + access_key_id: AWS access key ID for authentication. + session_token: AWS session token for authentication. + region: AWS region where the service is hosted. + model: Model identifier. Defaults to "amazon.nova-sonic-v1:0". + voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy. + params: Model parameters for audio configuration and inference. + system_instruction: System-level instruction for the model. + tools: Available tools/functions for the model to use. + send_transcription_frames: Whether to emit transcription frames. + + .. deprecated:: 0.0.91 + This parameter is deprecated and will be removed in a future version. + Transcription frames are always sent. + + **kwargs: Additional arguments passed to the parent LLMService. + """ + super().__init__(**kwargs) + self._secret_access_key = secret_access_key + self._access_key_id = access_key_id + self._session_token = session_token + self._region = region + self._model = model + self._client: Optional[BedrockRuntimeClient] = None + self._voice_id = voice_id + self._params = params or Params() + self._system_instruction = system_instruction + self._tools = tools + + if not send_transcription_frames: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "`send_transcription_frames` is deprecated and will be removed in a future version. " + "Transcription frames are always sent.", + DeprecationWarning, + stacklevel=2, + ) + + self._context: Optional[LLMContext] = None + self._stream: Optional[ + DuplexEventStream[ + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamOutput, + InvokeModelWithBidirectionalStreamOperationOutput, + ] + ] = None + self._receive_task: Optional[asyncio.Task] = None + self._prompt_name: Optional[str] = None + self._input_audio_content_name: Optional[str] = None + self._content_being_received: Optional[CurrentContent] = None + self._assistant_is_responding = False + self._may_need_repush_assistant_text = False + self._ready_to_send_context = False + self._handling_bot_stopped_speaking = False + self._triggering_assistant_response = False + self._waiting_for_trigger_transcription = False + self._disconnecting = False + self._connected_time: Optional[float] = None + self._wants_connection = False + self._user_text_buffer = "" + self._assistant_text_buffer = "" + self._completed_tool_calls = set() + + file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav") + with wave.open(file_path.open("rb"), "rb") as wav_file: + self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) + + # + # standard AIService frame handling + # + + async def start(self, frame: StartFrame): + """Start the service and initiate connection to AWS Nova Sonic. + + Args: + frame: The start frame triggering service initialization. + """ + await super().start(frame) + self._wants_connection = True + await self._start_connecting() + + async def stop(self, frame: EndFrame): + """Stop the service and close connections. + + Args: + frame: The end frame triggering service shutdown. + """ + await super().stop(frame) + self._wants_connection = False + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the service and close connections. + + Args: + frame: The cancel frame triggering service cancellation. + """ + await super().cancel(frame) + self._wants_connection = False + await self._disconnect() + + # + # conversation resetting + # + + async def reset_conversation(self): + """Reset the conversation state while preserving context. + + Handles bot stopped speaking event, disconnects from the service, + and reconnects with the preserved context. + """ + logger.debug("Resetting conversation") + await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False) + + # Grab context to carry through disconnect/reconnect + context = self._context + + await self._disconnect() + await self._start_connecting() + await self._handle_context(context) + + # + # frame processing + # + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle service-specific logic. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)): + context = ( + frame.context + if isinstance(frame, LLMContextFrame) + else LLMContext.from_openai_context(frame.context) + ) + await self._handle_context(context) + elif isinstance(frame, InputAudioRawFrame): + await self._handle_input_audio_frame(frame) + elif isinstance(frame, BotStoppedSpeakingFrame): + await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=True) + elif isinstance(frame, InterruptionFrame): + await self._handle_interruption_frame() + + await self.push_frame(frame, direction) + + async def _handle_context(self, context: LLMContext): + if self._disconnecting: + return + + if not self._context: + # We got our initial context + # Try to finish connecting + self._context = context + await self._finish_connecting_if_context_available() + else: + # We got an updated context + # Send results for any newly-completed function calls + await self._process_completed_function_calls(send_new_results=True) + + async def _handle_input_audio_frame(self, frame: InputAudioRawFrame): + # Wait until we're done sending the assistant response trigger audio before sending audio + # from the user's mic + if self._triggering_assistant_response: + return + + await self._send_user_audio_event(frame.audio) + + async def _handle_bot_stopped_speaking(self, delay_to_catch_trailing_assistant_text: bool): + # Protect against back-to-back BotStoppedSpeaking calls, which I've observed + if self._handling_bot_stopped_speaking: + return + self._handling_bot_stopped_speaking = True + + async def finalize_assistant_response(): + if self._assistant_is_responding: + # Consider the assistant finished with their response (possibly after a short delay, + # to allow for any trailing FINAL assistant text block to come in that need to make + # it into context). + # + # TODO: ideally we could base this solely on the LLM output events, but I couldn't + # figure out a reliable way to determine when we've gotten our last FINAL text block + # after the LLM is done talking. + # + # First I looked at stopReason, but it doesn't seem like the last FINAL text block + # is reliably marked END_TURN (sometimes the *first* one is, but not the last... + # bug?) + # + # Then I considered schemes where we tally or match up SPECULATIVE text blocks with + # FINAL text blocks to know how many or which FINAL blocks to expect, but user + # interruptions throw a wrench in these schemes: depending on the exact timing of + # the interruption, we should or shouldn't expect some FINAL blocks. + if delay_to_catch_trailing_assistant_text: + # This delay length is a balancing act between "catching" trailing assistant + # text that is quite delayed but not waiting so long that user text comes in + # first and results in a bit of context message order scrambling. + await asyncio.sleep(1.25) + self._assistant_is_responding = False + await self._report_assistant_response_ended() + + self._handling_bot_stopped_speaking = False + + # Finalize the assistant response, either now or after a delay + if delay_to_catch_trailing_assistant_text: + self.create_task(finalize_assistant_response()) + else: + await finalize_assistant_response() + + async def _handle_interruption_frame(self): + if self._assistant_is_responding: + self._may_need_repush_assistant_text = True + + # + # LLM communication: lifecycle + # + + async def _start_connecting(self): + try: + logger.info("Connecting...") + + if self._client: + # Here we assume that if we have a client we are connected or connecting + return + + # Set IDs for the connection + self._prompt_name = str(uuid.uuid4()) + self._input_audio_content_name = str(uuid.uuid4()) + + # Create the client + self._client = self._create_client() + + # Start the bidirectional stream + self._stream = await self._client.invoke_model_with_bidirectional_stream( + InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) + ) + + # Send session start event + await self._send_session_start_event() + + # Finish connecting + self._ready_to_send_context = True + await self._finish_connecting_if_context_available() + except Exception as e: + logger.error(f"{self} initialization error: {e}") + await self._disconnect() + + async def _process_completed_function_calls(self, send_new_results: bool): + # Check for set of completed function calls in the context + for message in self._context.get_messages(): + if message.get("role") and message.get("content") != "IN_PROGRESS": + tool_call_id = message.get("tool_call_id") + if tool_call_id and tool_call_id not in self._completed_tool_calls: + # Found a newly-completed function call - send the result to the service + if send_new_results: + await self._send_tool_result(tool_call_id, message.get("content")) + self._completed_tool_calls.add(tool_call_id) + + async def _finish_connecting_if_context_available(self): + # We can only finish connecting once we've gotten our initial context and we're ready to + # send it + if not (self._context and self._ready_to_send_context): + return + + logger.info("Finishing connecting (setting up session)...") + + # Initialize our bookkeeping of already-completed tool calls in the + # context + await self._process_completed_function_calls(send_new_results=False) + + # Read context + adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter() + llm_connection_params = adapter.get_llm_invocation_params(self._context) + + # Send prompt start event, specifying tools. + # Tools from context take priority over self._tools. + tools = ( + llm_connection_params["tools"] + if llm_connection_params["tools"] + else adapter.from_standard_tools(self._tools) + ) + logger.debug(f"Using tools: {tools}") + await self._send_prompt_start_event(tools) + + # Send system instruction. + # Instruction from context takes priority over self._system_instruction. + system_instruction = ( + llm_connection_params["system_instruction"] + if llm_connection_params["system_instruction"] + else self._system_instruction + ) + logger.debug(f"Using system instruction: {system_instruction}") + if system_instruction: + await self._send_text_event(text=system_instruction, role=Role.SYSTEM) + + # Send conversation history + for message in llm_connection_params["messages"]: + # logger.debug(f"Seeding conversation history with message: {message}") + await self._send_text_event(text=message.text, role=message.role) + + # Start audio input + await self._send_audio_input_start_event() + + # Start receiving events + self._receive_task = self.create_task(self._receive_task_handler()) + + # Record finished connecting time (must be done before sending assistant response trigger) + self._connected_time = time.time() + + logger.info("Finished connecting") + + # If we need to, send assistant response trigger (depends on self._connected_time) + if self._triggering_assistant_response: + await self._send_assistant_response_trigger() + + async def _disconnect(self): + try: + logger.info("Disconnecting...") + + # NOTE: see explanation of HACK, below + self._disconnecting = True + + # Clean up client + if self._client: + await self._send_session_end_events() + self._client = None + + # Clean up context + self._context = None + + # Clean up stream + if self._stream: + await self._stream.close() + self._stream = None + + # NOTE: see explanation of HACK, below + await asyncio.sleep(1) + + # Clean up receive task + # HACK: we should ideally be able to cancel the receive task before stopping the input + # stream, above (meaning we wouldn't need self._disconnecting). But for some reason if + # we don't close the input stream and wait a second first, we're getting an error a lot + # like this one: https://github.com/awslabs/amazon-transcribe-streaming-sdk/issues/61. + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=1.0) + self._receive_task = None + + # Reset remaining connection-specific state + # Should be all private state except: + # - _wants_connection + # - _assistant_response_trigger_audio + self._prompt_name = None + self._input_audio_content_name = None + self._content_being_received = None + self._assistant_is_responding = False + self._may_need_repush_assistant_text = False + self._ready_to_send_context = False + self._handling_bot_stopped_speaking = False + self._triggering_assistant_response = False + self._waiting_for_trigger_transcription = False + self._disconnecting = False + self._connected_time = None + self._user_text_buffer = "" + self._assistant_text_buffer = "" + self._completed_tool_calls = set() + + logger.info("Finished disconnecting") + except Exception as e: + logger.error(f"{self} error disconnecting: {e}") + + def _create_client(self) -> BedrockRuntimeClient: + config = Config( + endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", + region=self._region, + aws_access_key_id=self._access_key_id, + aws_secret_access_key=self._secret_access_key, + aws_session_token=self._session_token, + aws_credentials_identity_resolver=StaticCredentialsResolver(), + auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="bedrock")}, + ) + return BedrockRuntimeClient(config=config) + + # + # LLM communication: input events (pipecat -> LLM) + # + + async def _send_session_start_event(self): + session_start = f""" + {{ + "event": {{ + "sessionStart": {{ + "inferenceConfiguration": {{ + "maxTokens": {self._params.max_tokens}, + "topP": {self._params.top_p}, + "temperature": {self._params.temperature} + }} + }} + }} + }} + """ + await self._send_client_event(session_start) + + async def _send_prompt_start_event(self, tools: List[Any]): + if not self._prompt_name: + return + + tools_config = ( + f""", + "toolUseOutputConfiguration": {{ + "mediaType": "application/json" + }}, + "toolConfiguration": {{ + "tools": {json.dumps(tools)} + }} + """ + if tools + else "" + ) + + prompt_start = f''' + {{ + "event": {{ + "promptStart": {{ + "promptName": "{self._prompt_name}", + "textOutputConfiguration": {{ + "mediaType": "text/plain" + }}, + "audioOutputConfiguration": {{ + "mediaType": "audio/lpcm", + "sampleRateHertz": {self._params.output_sample_rate}, + "sampleSizeBits": {self._params.output_sample_size}, + "channelCount": {self._params.output_channel_count}, + "voiceId": "{self._voice_id}", + "encoding": "base64", + "audioType": "SPEECH" + }}{tools_config} + }} + }} + }} + ''' + await self._send_client_event(prompt_start) + + async def _send_audio_input_start_event(self): + if not self._prompt_name: + return + + audio_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{self._input_audio_content_name}", + "type": "AUDIO", + "interactive": true, + "role": "USER", + "audioInputConfiguration": {{ + "mediaType": "audio/lpcm", + "sampleRateHertz": {self._params.input_sample_rate}, + "sampleSizeBits": {self._params.input_sample_size}, + "channelCount": {self._params.input_channel_count}, + "audioType": "SPEECH", + "encoding": "base64" + }} + }} + }} + }} + ''' + await self._send_client_event(audio_content_start) + + async def _send_text_event(self, text: str, role: Role): + if not self._stream or not self._prompt_name or not text: + return + + content_name = str(uuid.uuid4()) + + text_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "type": "TEXT", + "interactive": true, + "role": "{role.value}", + "textInputConfiguration": {{ + "mediaType": "text/plain" + }} + }} + }} + }} + ''' + await self._send_client_event(text_content_start) + + escaped_text = json.dumps(text) # includes quotes + text_input = f''' + {{ + "event": {{ + "textInput": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "content": {escaped_text} + }} + }} + }} + ''' + await self._send_client_event(text_input) + + text_content_end = f''' + {{ + "event": {{ + "contentEnd": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}" + }} + }} + }} + ''' + await self._send_client_event(text_content_end) + + async def _send_user_audio_event(self, audio: bytes): + if not self._stream: + return + + blob = base64.b64encode(audio) + audio_event = f''' + {{ + "event": {{ + "audioInput": {{ + "promptName": "{self._prompt_name}", + "contentName": "{self._input_audio_content_name}", + "content": "{blob.decode("utf-8")}" + }} + }} + }} + ''' + await self._send_client_event(audio_event) + + async def _send_session_end_events(self): + if not self._stream or not self._prompt_name: + return + + prompt_end = f''' + {{ + "event": {{ + "promptEnd": {{ + "promptName": "{self._prompt_name}" + }} + }} + }} + ''' + await self._send_client_event(prompt_end) + + session_end = """ + { + "event": { + "sessionEnd": {} + } + } + """ + await self._send_client_event(session_end) + + async def _send_tool_result(self, tool_call_id, result): + if not self._stream or not self._prompt_name: + return + + content_name = str(uuid.uuid4()) + + result_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "interactive": false, + "type": "TOOL", + "role": "TOOL", + "toolResultInputConfiguration": {{ + "toolUseId": "{tool_call_id}", + "type": "TEXT", + "textInputConfiguration": {{ + "mediaType": "text/plain" + }} + }} + }} + }} + }} + ''' + await self._send_client_event(result_content_start) + + result_content = json.dumps( + { + "event": { + "toolResult": { + "promptName": self._prompt_name, + "contentName": content_name, + "content": json.dumps(result) if isinstance(result, dict) else result, + } + } + } + ) + await self._send_client_event(result_content) + + result_content_end = f""" + {{ + "event": {{ + "contentEnd": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}" + }} + }} + }} + """ + await self._send_client_event(result_content_end) + + async def _send_client_event(self, event_json: str): + if not self._stream: # should never happen + return + + event = InvokeModelWithBidirectionalStreamInputChunk( + value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) + ) + await self._stream.input_stream.send(event) + + # + # LLM communication: output events (LLM -> pipecat) + # + + # Receive events for the session. + # A few different kinds of content can be delivered: + # - Transcription of user audio + # - Tool use + # - Text preview of planned response speech before audio delivered + # - User interruption notification + # - Text of response speech that whose audio was actually delivered + # - Audio of response speech + # Each piece of content is wrapped by "contentStart" and "contentEnd" events. The content is + # delivered sequentially: one piece of content will end before another starts. + # The overall completion is wrapped by "completionStart" and "completionEnd" events. + async def _receive_task_handler(self): + try: + while self._stream and not self._disconnecting: + output = await self._stream.await_output() + result = await output[1].receive() + + if result.value and result.value.bytes_: + response_data = result.value.bytes_.decode("utf-8") + json_data = json.loads(response_data) + + if "event" in json_data: + event_json = json_data["event"] + if "completionStart" in event_json: + # Handle the LLM completion starting + await self._handle_completion_start_event(event_json) + elif "contentStart" in event_json: + # Handle a piece of content starting + await self._handle_content_start_event(event_json) + elif "textOutput" in event_json: + # Handle text output content + await self._handle_text_output_event(event_json) + elif "audioOutput" in event_json: + # Handle audio output content + await self._handle_audio_output_event(event_json) + elif "toolUse" in event_json: + # Handle tool use + await self._handle_tool_use_event(event_json) + elif "contentEnd" in event_json: + # Handle a piece of content ending + await self._handle_content_end_event(event_json) + elif "completionEnd" in event_json: + # Handle the LLM completion ending + await self._handle_completion_end_event(event_json) + except Exception as e: + if self._disconnecting: + # Errors are kind of expected while disconnecting, so just + # ignore them and do nothing + return + logger.error(f"{self} error processing responses: {e}") + if self._wants_connection: + await self.reset_conversation() + + async def _handle_completion_start_event(self, event_json): + pass + + async def _handle_content_start_event(self, event_json): + content_start = event_json["contentStart"] + type = content_start["type"] + role = content_start["role"] + generation_stage = None + if "additionalModelFields" in content_start: + additional_model_fields = json.loads(content_start["additionalModelFields"]) + generation_stage = additional_model_fields.get("generationStage") + + # Bookkeeping: track current content being received + content = CurrentContent( + type=ContentType(type), + role=Role(role), + text_stage=TextStage(generation_stage) if generation_stage else None, + text_content=None, + ) + self._content_being_received = content + + if content.role == Role.ASSISTANT: + if content.type == ContentType.AUDIO: + # Note that an assistant response can comprise of multiple audio blocks + if not self._assistant_is_responding: + # The assistant has started responding. + self._assistant_is_responding = True + await self._report_user_transcription_ended() # Consider user turn over + await self._report_assistant_response_started() + + async def _handle_text_output_event(self, event_json): + if not self._content_being_received: # should never happen + return + content = self._content_being_received + + text_content = event_json["textOutput"]["content"] + + # Bookkeeping: augment the current content being received with text + # Assumption: only one text content per content block + content.text_content = text_content + + async def _handle_audio_output_event(self, event_json): + if not self._content_being_received: # should never happen + return + + # Get audio + audio_content = event_json["audioOutput"]["content"] + + # Push audio frame + audio = base64.b64decode(audio_content) + frame = TTSAudioRawFrame( + audio=audio, + sample_rate=self._params.output_sample_rate, + num_channels=self._params.output_channel_count, + ) + await self.push_frame(frame) + + async def _handle_tool_use_event(self, event_json): + if not self._content_being_received or not self._context: # should never happen + return + + # Consider user turn over + await self._report_user_transcription_ended() + + # Get tool use details + tool_use = event_json["toolUse"] + function_name = tool_use["toolName"] + tool_call_id = tool_use["toolUseId"] + arguments = json.loads(tool_use["content"]) + + # Call tool function + if self.has_function(function_name): + if function_name in self._functions.keys() or None in self._functions.keys(): + function_calls_llm = [ + FunctionCallFromLLM( + context=self._context, + tool_call_id=tool_call_id, + function_name=function_name, + arguments=arguments, + ) + ] + + await self.run_function_calls(function_calls_llm) + else: + raise AWSNovaSonicUnhandledFunctionException( + f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function." + ) + + async def _handle_content_end_event(self, event_json): + if not self._content_being_received: # should never happen + return + content = self._content_being_received + + content_end = event_json["contentEnd"] + stop_reason = content_end["stopReason"] + + # Bookkeeping: clear current content being received + self._content_being_received = None + + if content.role == Role.ASSISTANT: + if content.type == ContentType.TEXT: + # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) + if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": + if self._assistant_is_responding: + # Text added to the ongoing assistant response + await self._report_assistant_response_text_added(content.text_content) + elif content.role == Role.USER: + if content.type == ContentType.TEXT: + if content.text_stage == TextStage.FINAL: + # User transcription text added + await self._report_user_transcription_text_added(content.text_content) + + async def _handle_completion_end_event(self, event_json): + pass + + # + # assistant response reporting + # + # 1. Started + # 2. Text added + # 3. Ended + # + + async def _report_assistant_response_started(self): + logger.debug("Assistant response started") + + # Report the start of the assistant response. + await self.push_frame(LLMFullResponseStartFrame()) + + # Report that equivalent of TTS (this is a speech-to-speech model) started + await self.push_frame(TTSStartedFrame()) + + async def _report_assistant_response_text_added(self, text): + if not self._context: # should never happen + return + + logger.debug(f"Assistant response text added: {text}") + + # Report the text of the assistant response. + await self.push_frame(TTSTextFrame(text)) + + # HACK: here we're also buffering the assistant text ourselves as a + # backup rather than relying solely on the assistant context aggregator + # to do it, because the text arrives from Nova Sonic only after all the + # assistant audio frames have been pushed, meaning that if an + # interruption frame were to arrive we would lose all of it (the text + # frames sitting in the queue would be wiped). + self._assistant_text_buffer += text + + async def _report_assistant_response_ended(self): + if not self._context: # should never happen + return + + logger.debug("Assistant response ended") + + # If an interruption frame arrived while the assistant was responding + # we may have lost all of the assistant text (see HACK, above), so + # re-push it downstream to the aggregator now. + if self._may_need_repush_assistant_text: + # Just in case, check that assistant text hasn't already made it + # into the context (sometimes it does, despite the interruption). + messages = self._context.get_messages() + last_message = messages[-1] if messages else None + if ( + not last_message + or last_message.get("role") != "assistant" + or last_message.get("content") != self._assistant_text_buffer + ): + # We also need to re-push the LLMFullResponseStartFrame since the + # TTSTextFrame would be ignored otherwise (the interruption frame + # would have cleared the assistant aggregator state). + await self.push_frame(LLMFullResponseStartFrame()) + await self.push_frame(TTSTextFrame(self._assistant_text_buffer)) + self._may_need_repush_assistant_text = False + + # Report the end of the assistant response. + await self.push_frame(LLMFullResponseEndFrame()) + + # Report that equivalent of TTS (this is a speech-to-speech model) stopped. + await self.push_frame(TTSStoppedFrame()) + + # Clear out the buffered assistant text + self._assistant_text_buffer = "" + + # + # user transcription reporting + # + # 1. Text added + # 2. Ended + # + # Note: "started" does not need to be reported + # + + async def _report_user_transcription_text_added(self, text): + if not self._context: # should never happen + return + + logger.debug(f"User transcription text added: {text}") + + # HACK: here we're buffering the user text ourselves rather than + # relying on the upstream user context aggregator to do it, because the + # text arrives in fairly large chunks spaced fairly far apart in time. + # That means the user text would be split between different messages in + # context. Even if we sent placeholder InterimTranscriptionFrames in + # between each TranscriptionFrame to tell the aggregator to hold off on + # finalizing the user message, the aggregator would likely get the last + # chunk too late. + self._user_text_buffer += f" {text}" if self._user_text_buffer else text + + async def _report_user_transcription_ended(self): + if not self._context: # should never happen + return + + logger.debug(f"User transcription ended") + + # Report to the upstream user context aggregator that some new user + # transcription text is available. + + # HACK: Check if this transcription was triggered by our own + # assistant response trigger. If so, we need to wrap it with + # UserStarted/StoppedSpeakingFrames; otherwise the user aggregator + # would fire an EmulatedUserStartedSpeakingFrame, which would + # trigger an interruption, which would prevent us from writing the + # assistant response to context. + # + # Sending an EmulateUserStartedSpeakingFrame ourselves doesn't + # work: it just causes the interruption we're trying to avoid. + # + # Setting enable_emulated_vad_interruptions also doesn't work: at + # the time the user aggregator receives the TranscriptionFrame, it + # doesn't yet know the assistant has started responding, so it + # doesn't know that emulating the user starting to speak would + # cause an interruption. + should_wrap_in_user_started_stopped_speaking_frames = ( + self._waiting_for_trigger_transcription + and self._user_text_buffer.strip().lower() == "ready" + ) + + # Start wrapping the upstream transcription in UserStarted/StoppedSpeakingFrames if needed + if should_wrap_in_user_started_stopped_speaking_frames: + logger.debug( + "Wrapping assistant response trigger transcription with upstream UserStarted/StoppedSpeakingFrames" + ) + await self.push_frame(UserStartedSpeakingFrame(), direction=FrameDirection.UPSTREAM) + + # Send the transcription upstream for the user context aggregator + frame = TranscriptionFrame( + text=self._user_text_buffer, user_id="", timestamp=time_now_iso8601() + ) + await self.push_frame(frame, direction=FrameDirection.UPSTREAM) + + # Finish wrapping the upstream transcription in UserStarted/StoppedSpeakingFrames if needed + if should_wrap_in_user_started_stopped_speaking_frames: + await self.push_frame(UserStoppedSpeakingFrame(), direction=FrameDirection.UPSTREAM) + + # Clear out the buffered user text + self._user_text_buffer = "" + + # We're no longer waiting for a trigger transcription + self._waiting_for_trigger_transcription = False + + # + # context + # + + def create_context_aggregator( + self, + context: OpenAILLMContext, + *, + user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), + assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), + ) -> LLMContextAggregatorPair: + """Create context aggregator pair for managing conversation context. + + NOTE: this method exists only for backward compatibility. New code + should instead do: + context = LLMContext(...) + context_aggregator = LLMContextAggregatorPair(context) + + Args: + context: The OpenAI LLM context. + user_params: Parameters for the user context aggregator. + assistant_params: Parameters for the assistant context aggregator. + + Returns: + A pair of user and assistant context aggregators. + """ + context = LLMContext.from_openai_context(context) + return LLMContextAggregatorPair( + context, user_params=user_params, assistant_params=assistant_params + ) + + # + # assistant response trigger (HACK) + # + + # Class variable + AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION = ( + "Start speaking when you hear the user say 'ready', but don't consider that 'ready' to be " + "a meaningful part of the conversation other than as a trigger for you to start speaking." + ) + + async def trigger_assistant_response(self): + """Trigger an assistant response by sending audio cue. + + Sends a pre-recorded "ready" audio trigger to prompt the assistant + to start speaking. This is useful for controlling conversation flow. + + Returns: + False if already triggering a response, True otherwise. + """ + if self._triggering_assistant_response: + return False + + self._triggering_assistant_response = True + + # Send the trigger audio, if we're fully connected and set up + if self._connected_time: + await self._send_assistant_response_trigger() + + async def _send_assistant_response_trigger(self): + if not self._connected_time: + # should never happen + return + + try: + logger.debug("Sending assistant response trigger...") + + self._waiting_for_trigger_transcription = True + + chunk_duration = 0.02 # what we might get from InputAudioRawFrame + chunk_size = int( + chunk_duration + * self._params.input_sample_rate + * self._params.input_channel_count + * (self._params.input_sample_size / 8) + ) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes + + # Lead with a bit of blank audio, if needed. + # It seems like the LLM can't quite "hear" the first little bit of audio sent on a + # connection. + current_time = time.time() + max_blank_audio_duration = 0.5 + blank_audio_duration = ( + max_blank_audio_duration - (current_time - self._connected_time) + if self._connected_time is not None + and (current_time - self._connected_time) < max_blank_audio_duration + else None + ) + if blank_audio_duration: + logger.debug( + f"Leading assistant response trigger with {blank_audio_duration}s of blank audio" + ) + blank_audio_chunk = b"\x00" * chunk_size + num_chunks = int(blank_audio_duration / chunk_duration) + for _ in range(num_chunks): + await self._send_user_audio_event(blank_audio_chunk) + await asyncio.sleep(chunk_duration) + + # Send trigger audio + # NOTE: this audio *will* be transcribed and eventually make it into the context. That's OK: + # if we ever need to seed this service again with context it would make sense to include it + # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the + # context as well. + audio_chunks = [ + self._assistant_response_trigger_audio[i : i + chunk_size] + for i in range(0, len(self._assistant_response_trigger_audio), chunk_size) + ] + for chunk in audio_chunks: + await self._send_user_audio_event(chunk) + await asyncio.sleep(chunk_duration) + finally: + # We need to clean up in case sending the trigger was cancelled, e.g. in the case of a user interruption. + # (An asyncio.CancelledError would be raised in that case.) + self._triggering_assistant_response = False diff --git a/src/pipecat/services/aws_nova_sonic/ready.wav b/src/pipecat/services/aws/nova_sonic/ready.wav similarity index 100% rename from src/pipecat/services/aws_nova_sonic/ready.wav rename to src/pipecat/services/aws/nova_sonic/ready.wav diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index 59cda5865..b019fc058 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -286,6 +286,7 @@ class AWSTranscribeSTTService(STTService): logger.info(f"{self} Successfully connected to AWS Transcribe") + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} Failed to connect to AWS Transcribe: {e}") await self._disconnect() @@ -310,6 +311,7 @@ class AWSTranscribeSTTService(STTService): logger.warning(f"{self} Error closing WebSocket connection: {e}") finally: self._ws_client = None + await self._call_event_handler("on_disconnected") def language_to_service_language(self, language: Language) -> str | None: """Convert internal language enum to AWS Transcribe language code. diff --git a/src/pipecat/services/aws_nova_sonic/__init__.py b/src/pipecat/services/aws_nova_sonic/__init__.py index 4da394cf6..e1cb912b6 100644 --- a/src/pipecat/services/aws_nova_sonic/__init__.py +++ b/src/pipecat/services/aws_nova_sonic/__init__.py @@ -1 +1,19 @@ -from .aws import AWSNovaSonicLLMService, Params +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import warnings + +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService, Params + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic are deprecated. " + "Please use the equivalent types from " + "pipecat.services.aws.nova_sonic.llm instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 432db553f..0524829df 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -10,1150 +10,16 @@ This module provides a speech-to-speech LLM service using AWS Nova Sonic, which bidirectional audio streaming, text generation, and function calling capabilities. """ -import asyncio -import base64 -import json -import time -import uuid -import wave -from dataclasses import dataclass -from enum import Enum -from importlib.resources import files -from typing import Any, List, Optional +import warnings -from loguru import logger -from pydantic import BaseModel, Field +from pipecat.services.aws.nova_sonic.llm import * -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter -from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, - CancelFrame, - EndFrame, - Frame, - FunctionCallFromLLM, - InputAudioRawFrame, - InterimTranscriptionFrame, - LLMContextFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMTextFrame, - StartFrame, - TranscriptionFrame, - TTSAudioRawFrame, - TTSStartedFrame, - TTSStoppedFrame, - TTSTextFrame, -) -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantAggregatorParams, - LLMUserAggregatorParams, -) -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.aws_nova_sonic.context import ( - AWSNovaSonicAssistantContextAggregator, - AWSNovaSonicContextAggregatorPair, - AWSNovaSonicLLMContext, - AWSNovaSonicUserContextAggregator, - Role, -) -from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame -from pipecat.services.llm_service import LLMService -from pipecat.utils.time import time_now_iso8601 - -try: - from aws_sdk_bedrock_runtime.client import ( - BedrockRuntimeClient, - InvokeModelWithBidirectionalStreamOperationInput, +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic.aws are deprecated. " + "Please use the equivalent types from " + "pipecat.services.aws.nova_sonic.llm instead.", + DeprecationWarning, + stacklevel=2, ) - from aws_sdk_bedrock_runtime.config import Config, HTTPAuthSchemeResolver, SigV4AuthScheme - from aws_sdk_bedrock_runtime.models import ( - BidirectionalInputPayloadPart, - InvokeModelWithBidirectionalStreamInput, - InvokeModelWithBidirectionalStreamInputChunk, - InvokeModelWithBidirectionalStreamOperationOutput, - InvokeModelWithBidirectionalStreamOutput, - ) - from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver - from smithy_aws_core.identity import AWSCredentialsIdentity - from smithy_core.aio.eventstream import DuplexEventStream -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use AWS services, you need to `pip install pipecat-ai[aws-nova-sonic]`." - ) - raise Exception(f"Missing module: {e}") - - -class AWSNovaSonicUnhandledFunctionException(Exception): - """Exception raised when the LLM attempts to call an unregistered function.""" - - pass - - -class ContentType(Enum): - """Content types supported by AWS Nova Sonic. - - Parameters: - AUDIO: Audio content type. - TEXT: Text content type. - TOOL: Tool content type. - """ - - AUDIO = "AUDIO" - TEXT = "TEXT" - TOOL = "TOOL" - - -class TextStage(Enum): - """Text generation stages in AWS Nova Sonic responses. - - Parameters: - FINAL: Final text that has been fully generated. - SPECULATIVE: Speculative text that is still being generated. - """ - - FINAL = "FINAL" # what has been said - SPECULATIVE = "SPECULATIVE" # what's planned to be said - - -@dataclass -class CurrentContent: - """Represents content currently being received from AWS Nova Sonic. - - Parameters: - type: The type of content (audio, text, or tool). - role: The role generating the content (user, assistant, etc.). - text_stage: The stage of text generation (final or speculative). - text_content: The actual text content if applicable. - """ - - type: ContentType - role: Role - text_stage: TextStage # None if not text - text_content: str # starts as None, then fills in if text - - def __str__(self): - """String representation of the current content.""" - return ( - f"CurrentContent(\n" - f" type={self.type.name},\n" - f" role={self.role.name},\n" - f" text_stage={self.text_stage.name if self.text_stage else 'None'}\n" - f")" - ) - - -class Params(BaseModel): - """Configuration parameters for AWS Nova Sonic. - - Parameters: - input_sample_rate: Audio input sample rate in Hz. - input_sample_size: Audio input sample size in bits. - input_channel_count: Number of input audio channels. - output_sample_rate: Audio output sample rate in Hz. - output_sample_size: Audio output sample size in bits. - output_channel_count: Number of output audio channels. - max_tokens: Maximum number of tokens to generate. - top_p: Nucleus sampling parameter. - temperature: Sampling temperature for text generation. - """ - - # Audio input - input_sample_rate: Optional[int] = Field(default=16000) - input_sample_size: Optional[int] = Field(default=16) - input_channel_count: Optional[int] = Field(default=1) - - # Audio output - output_sample_rate: Optional[int] = Field(default=24000) - output_sample_size: Optional[int] = Field(default=16) - output_channel_count: Optional[int] = Field(default=1) - - # Inference - max_tokens: Optional[int] = Field(default=1024) - top_p: Optional[float] = Field(default=0.9) - temperature: Optional[float] = Field(default=0.7) - - -class AWSNovaSonicLLMService(LLMService): - """AWS Nova Sonic speech-to-speech LLM service. - - Provides bidirectional audio streaming, real-time transcription, text generation, - and function calling capabilities using AWS Nova Sonic model. - """ - - # Override the default adapter to use the AWSNovaSonicLLMAdapter one - adapter_class = AWSNovaSonicLLMAdapter - - def __init__( - self, - *, - secret_access_key: str, - access_key_id: str, - session_token: Optional[str] = None, - region: str, - model: str = "amazon.nova-sonic-v1:0", - voice_id: str = "matthew", # matthew, tiffany, amy - params: Optional[Params] = None, - system_instruction: Optional[str] = None, - tools: Optional[ToolsSchema] = None, - send_transcription_frames: bool = True, - **kwargs, - ): - """Initializes the AWS Nova Sonic LLM service. - - Args: - secret_access_key: AWS secret access key for authentication. - access_key_id: AWS access key ID for authentication. - session_token: AWS session token for authentication. - region: AWS region where the service is hosted. - model: Model identifier. Defaults to "amazon.nova-sonic-v1:0". - voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy. - params: Model parameters for audio configuration and inference. - system_instruction: System-level instruction for the model. - tools: Available tools/functions for the model to use. - send_transcription_frames: Whether to emit transcription frames. - **kwargs: Additional arguments passed to the parent LLMService. - """ - super().__init__(**kwargs) - self._secret_access_key = secret_access_key - self._access_key_id = access_key_id - self._session_token = session_token - self._region = region - self._model = model - self._client: Optional[BedrockRuntimeClient] = None - self._voice_id = voice_id - self._params = params or Params() - self._system_instruction = system_instruction - self._tools = tools - self._send_transcription_frames = send_transcription_frames - self._context: Optional[AWSNovaSonicLLMContext] = None - self._stream: Optional[ - DuplexEventStream[ - InvokeModelWithBidirectionalStreamInput, - InvokeModelWithBidirectionalStreamOutput, - InvokeModelWithBidirectionalStreamOperationOutput, - ] - ] = None - self._receive_task: Optional[asyncio.Task] = None - self._prompt_name: Optional[str] = None - self._input_audio_content_name: Optional[str] = None - self._content_being_received: Optional[CurrentContent] = None - self._assistant_is_responding = False - self._ready_to_send_context = False - self._handling_bot_stopped_speaking = False - self._triggering_assistant_response = False - self._disconnecting = False - self._connected_time: Optional[float] = None - self._wants_connection = False - - file_path = files("pipecat.services.aws_nova_sonic").joinpath("ready.wav") - with wave.open(file_path.open("rb"), "rb") as wav_file: - self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) - - # - # standard AIService frame handling - # - - async def start(self, frame: StartFrame): - """Start the service and initiate connection to AWS Nova Sonic. - - Args: - frame: The start frame triggering service initialization. - """ - await super().start(frame) - self._wants_connection = True - await self._start_connecting() - - async def stop(self, frame: EndFrame): - """Stop the service and close connections. - - Args: - frame: The end frame triggering service shutdown. - """ - await super().stop(frame) - self._wants_connection = False - await self._disconnect() - - async def cancel(self, frame: CancelFrame): - """Cancel the service and close connections. - - Args: - frame: The cancel frame triggering service cancellation. - """ - await super().cancel(frame) - self._wants_connection = False - await self._disconnect() - - # - # conversation resetting - # - - async def reset_conversation(self): - """Reset the conversation state while preserving context. - - Handles bot stopped speaking event, disconnects from the service, - and reconnects with the preserved context. - """ - logger.debug("Resetting conversation") - await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False) - - # Carry over previous context through disconnect - context = self._context - await self._disconnect() - self._context = context - - await self._start_connecting() - - # - # frame processing - # - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames and handle service-specific logic. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - await super().process_frame(frame, direction) - - if isinstance(frame, OpenAILLMContextFrame): - await self._handle_context(frame.context) - elif isinstance(frame, LLMContextFrame): - raise NotImplementedError( - "Universal LLMContext is not yet supported for AWS Nova Sonic." - ) - elif isinstance(frame, InputAudioRawFrame): - await self._handle_input_audio_frame(frame) - elif isinstance(frame, BotStoppedSpeakingFrame): - await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=True) - elif isinstance(frame, AWSNovaSonicFunctionCallResultFrame): - await self._handle_function_call_result(frame) - - await self.push_frame(frame, direction) - - async def _handle_context(self, context: OpenAILLMContext): - if not self._context: - # We got our initial context - try to finish connecting - self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic( - context, self._system_instruction - ) - await self._finish_connecting_if_context_available() - - async def _handle_input_audio_frame(self, frame: InputAudioRawFrame): - # Wait until we're done sending the assistant response trigger audio before sending audio - # from the user's mic - if self._triggering_assistant_response: - return - - await self._send_user_audio_event(frame.audio) - - async def _handle_bot_stopped_speaking(self, delay_to_catch_trailing_assistant_text: bool): - # Protect against back-to-back BotStoppedSpeaking calls, which I've observed - if self._handling_bot_stopped_speaking: - return - self._handling_bot_stopped_speaking = True - - async def finalize_assistant_response(): - if self._assistant_is_responding: - # Consider the assistant finished with their response (possibly after a short delay, - # to allow for any trailing FINAL assistant text block to come in that need to make - # it into context). - # - # TODO: ideally we could base this solely on the LLM output events, but I couldn't - # figure out a reliable way to determine when we've gotten our last FINAL text block - # after the LLM is done talking. - # - # First I looked at stopReason, but it doesn't seem like the last FINAL text block - # is reliably marked END_TURN (sometimes the *first* one is, but not the last... - # bug?) - # - # Then I considered schemes where we tally or match up SPECULATIVE text blocks with - # FINAL text blocks to know how many or which FINAL blocks to expect, but user - # interruptions throw a wrench in these schemes: depending on the exact timing of - # the interruption, we should or shouldn't expect some FINAL blocks. - if delay_to_catch_trailing_assistant_text: - # This delay length is a balancing act between "catching" trailing assistant - # text that is quite delayed but not waiting so long that user text comes in - # first and results in a bit of context message order scrambling. - await asyncio.sleep(1.25) - self._assistant_is_responding = False - await self._report_assistant_response_ended() - - self._handling_bot_stopped_speaking = False - - # Finalize the assistant response, either now or after a delay - if delay_to_catch_trailing_assistant_text: - self.create_task(finalize_assistant_response()) - else: - await finalize_assistant_response() - - async def _handle_function_call_result(self, frame: AWSNovaSonicFunctionCallResultFrame): - result = frame.result_frame - await self._send_tool_result(tool_call_id=result.tool_call_id, result=result.result) - - # - # LLM communication: lifecycle - # - - async def _start_connecting(self): - try: - logger.info("Connecting...") - - if self._client: - # Here we assume that if we have a client we are connected or connecting - return - - # Set IDs for the connection - self._prompt_name = str(uuid.uuid4()) - self._input_audio_content_name = str(uuid.uuid4()) - - # Create the client - self._client = self._create_client() - - # Start the bidirectional stream - self._stream = await self._client.invoke_model_with_bidirectional_stream( - InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) - ) - - # Send session start event - await self._send_session_start_event() - - # Finish connecting - self._ready_to_send_context = True - await self._finish_connecting_if_context_available() - except Exception as e: - logger.error(f"{self} initialization error: {e}") - await self._disconnect() - - async def _finish_connecting_if_context_available(self): - # We can only finish connecting once we've gotten our initial context and we're ready to - # send it - if not (self._context and self._ready_to_send_context): - return - - logger.info("Finishing connecting (setting up session)...") - - # Read context - history = self._context.get_messages_for_initializing_history() - - # Send prompt start event, specifying tools. - # Tools from context take priority over self._tools. - tools = ( - self._context.tools - if self._context.tools - else self.get_llm_adapter().from_standard_tools(self._tools) - ) - logger.debug(f"Using tools: {tools}") - await self._send_prompt_start_event(tools) - - # Send system instruction. - # Instruction from context takes priority over self._system_instruction. - # (NOTE: this prioritizing occurred automatically behind the scenes: the context was - # initialized with self._system_instruction and then updated itself from its messages when - # get_messages_for_initializing_history() was called). - logger.debug(f"Using system instruction: {history.system_instruction}") - if history.system_instruction: - await self._send_text_event(text=history.system_instruction, role=Role.SYSTEM) - - # Send conversation history - for message in history.messages: - await self._send_text_event(text=message.text, role=message.role) - - # Start audio input - await self._send_audio_input_start_event() - - # Start receiving events - self._receive_task = self.create_task(self._receive_task_handler()) - - # Record finished connecting time (must be done before sending assistant response trigger) - self._connected_time = time.time() - - logger.info("Finished connecting") - - # If we need to, send assistant response trigger (depends on self._connected_time) - if self._triggering_assistant_response: - await self._send_assistant_response_trigger() - - async def _disconnect(self): - try: - logger.info("Disconnecting...") - - # NOTE: see explanation of HACK, below - self._disconnecting = True - - # Clean up client - if self._client: - await self._send_session_end_events() - self._client = None - - # Clean up stream - if self._stream: - await self._stream.input_stream.close() - self._stream = None - - # NOTE: see explanation of HACK, below - await asyncio.sleep(1) - - # Clean up receive task - # HACK: we should ideally be able to cancel the receive task before stopping the input - # stream, above (meaning we wouldn't need self._disconnecting). But for some reason if - # we don't close the input stream and wait a second first, we're getting an error a lot - # like this one: https://github.com/awslabs/amazon-transcribe-streaming-sdk/issues/61. - if self._receive_task: - await self.cancel_task(self._receive_task, timeout=1.0) - self._receive_task = None - - # Reset remaining connection-specific state - self._prompt_name = None - self._input_audio_content_name = None - self._content_being_received = None - self._assistant_is_responding = False - self._ready_to_send_context = False - self._handling_bot_stopped_speaking = False - self._triggering_assistant_response = False - self._disconnecting = False - self._connected_time = None - - logger.info("Finished disconnecting") - except Exception as e: - logger.error(f"{self} error disconnecting: {e}") - - def _create_client(self) -> BedrockRuntimeClient: - config = Config( - endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", - region=self._region, - aws_credentials_identity_resolver=StaticCredentialsResolver( - credentials=AWSCredentialsIdentity( - access_key_id=self._access_key_id, - secret_access_key=self._secret_access_key, - session_token=self._session_token, - ) - ), - http_auth_scheme_resolver=HTTPAuthSchemeResolver(), - http_auth_schemes={"aws.auth#sigv4": SigV4AuthScheme()}, - ) - return BedrockRuntimeClient(config=config) - - # - # LLM communication: input events (pipecat -> LLM) - # - - async def _send_session_start_event(self): - session_start = f""" - {{ - "event": {{ - "sessionStart": {{ - "inferenceConfiguration": {{ - "maxTokens": {self._params.max_tokens}, - "topP": {self._params.top_p}, - "temperature": {self._params.temperature} - }} - }} - }} - }} - """ - await self._send_client_event(session_start) - - async def _send_prompt_start_event(self, tools: List[Any]): - if not self._prompt_name: - return - - tools_config = ( - f""", - "toolUseOutputConfiguration": {{ - "mediaType": "application/json" - }}, - "toolConfiguration": {{ - "tools": {json.dumps(tools)} - }} - """ - if tools - else "" - ) - - prompt_start = f''' - {{ - "event": {{ - "promptStart": {{ - "promptName": "{self._prompt_name}", - "textOutputConfiguration": {{ - "mediaType": "text/plain" - }}, - "audioOutputConfiguration": {{ - "mediaType": "audio/lpcm", - "sampleRateHertz": {self._params.output_sample_rate}, - "sampleSizeBits": {self._params.output_sample_size}, - "channelCount": {self._params.output_channel_count}, - "voiceId": "{self._voice_id}", - "encoding": "base64", - "audioType": "SPEECH" - }}{tools_config} - }} - }} - }} - ''' - await self._send_client_event(prompt_start) - - async def _send_audio_input_start_event(self): - if not self._prompt_name: - return - - audio_content_start = f''' - {{ - "event": {{ - "contentStart": {{ - "promptName": "{self._prompt_name}", - "contentName": "{self._input_audio_content_name}", - "type": "AUDIO", - "interactive": true, - "role": "USER", - "audioInputConfiguration": {{ - "mediaType": "audio/lpcm", - "sampleRateHertz": {self._params.input_sample_rate}, - "sampleSizeBits": {self._params.input_sample_size}, - "channelCount": {self._params.input_channel_count}, - "audioType": "SPEECH", - "encoding": "base64" - }} - }} - }} - }} - ''' - await self._send_client_event(audio_content_start) - - async def _send_text_event(self, text: str, role: Role): - if not self._stream or not self._prompt_name or not text: - return - - content_name = str(uuid.uuid4()) - - text_content_start = f''' - {{ - "event": {{ - "contentStart": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}", - "type": "TEXT", - "interactive": true, - "role": "{role.value}", - "textInputConfiguration": {{ - "mediaType": "text/plain" - }} - }} - }} - }} - ''' - await self._send_client_event(text_content_start) - - escaped_text = json.dumps(text) # includes quotes - text_input = f''' - {{ - "event": {{ - "textInput": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}", - "content": {escaped_text} - }} - }} - }} - ''' - await self._send_client_event(text_input) - - text_content_end = f''' - {{ - "event": {{ - "contentEnd": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}" - }} - }} - }} - ''' - await self._send_client_event(text_content_end) - - async def _send_user_audio_event(self, audio: bytes): - if not self._stream: - return - - blob = base64.b64encode(audio) - audio_event = f''' - {{ - "event": {{ - "audioInput": {{ - "promptName": "{self._prompt_name}", - "contentName": "{self._input_audio_content_name}", - "content": "{blob.decode("utf-8")}" - }} - }} - }} - ''' - await self._send_client_event(audio_event) - - async def _send_session_end_events(self): - if not self._stream or not self._prompt_name: - return - - prompt_end = f''' - {{ - "event": {{ - "promptEnd": {{ - "promptName": "{self._prompt_name}" - }} - }} - }} - ''' - await self._send_client_event(prompt_end) - - session_end = """ - { - "event": { - "sessionEnd": {} - } - } - """ - await self._send_client_event(session_end) - - async def _send_tool_result(self, tool_call_id, result): - if not self._stream or not self._prompt_name: - return - - content_name = str(uuid.uuid4()) - - result_content_start = f''' - {{ - "event": {{ - "contentStart": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}", - "interactive": false, - "type": "TOOL", - "role": "TOOL", - "toolResultInputConfiguration": {{ - "toolUseId": "{tool_call_id}", - "type": "TEXT", - "textInputConfiguration": {{ - "mediaType": "text/plain" - }} - }} - }} - }} - }} - ''' - await self._send_client_event(result_content_start) - - result_content = json.dumps( - { - "event": { - "toolResult": { - "promptName": self._prompt_name, - "contentName": content_name, - "content": json.dumps(result) if isinstance(result, dict) else result, - } - } - } - ) - await self._send_client_event(result_content) - - result_content_end = f""" - {{ - "event": {{ - "contentEnd": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}" - }} - }} - }} - """ - await self._send_client_event(result_content_end) - - async def _send_client_event(self, event_json: str): - if not self._stream: # should never happen - return - - event = InvokeModelWithBidirectionalStreamInputChunk( - value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) - ) - await self._stream.input_stream.send(event) - - # - # LLM communication: output events (LLM -> pipecat) - # - - # Receive events for the session. - # A few different kinds of content can be delivered: - # - Transcription of user audio - # - Tool use - # - Text preview of planned response speech before audio delivered - # - User interruption notification - # - Text of response speech that whose audio was actually delivered - # - Audio of response speech - # Each piece of content is wrapped by "contentStart" and "contentEnd" events. The content is - # delivered sequentially: one piece of content will end before another starts. - # The overall completion is wrapped by "completionStart" and "completionEnd" events. - async def _receive_task_handler(self): - try: - while self._stream and not self._disconnecting: - output = await self._stream.await_output() - result = await output[1].receive() - - if result.value and result.value.bytes_: - response_data = result.value.bytes_.decode("utf-8") - json_data = json.loads(response_data) - - if "event" in json_data: - event_json = json_data["event"] - if "completionStart" in event_json: - # Handle the LLM completion starting - await self._handle_completion_start_event(event_json) - elif "contentStart" in event_json: - # Handle a piece of content starting - await self._handle_content_start_event(event_json) - elif "textOutput" in event_json: - # Handle text output content - await self._handle_text_output_event(event_json) - elif "audioOutput" in event_json: - # Handle audio output content - await self._handle_audio_output_event(event_json) - elif "toolUse" in event_json: - # Handle tool use - await self._handle_tool_use_event(event_json) - elif "contentEnd" in event_json: - # Handle a piece of content ending - await self._handle_content_end_event(event_json) - elif "completionEnd" in event_json: - # Handle the LLM completion ending - await self._handle_completion_end_event(event_json) - except Exception as e: - logger.error(f"{self} error processing responses: {e}") - if self._wants_connection: - await self.reset_conversation() - - async def _handle_completion_start_event(self, event_json): - pass - - async def _handle_content_start_event(self, event_json): - content_start = event_json["contentStart"] - type = content_start["type"] - role = content_start["role"] - generation_stage = None - if "additionalModelFields" in content_start: - additional_model_fields = json.loads(content_start["additionalModelFields"]) - generation_stage = additional_model_fields.get("generationStage") - - # Bookkeeping: track current content being received - content = CurrentContent( - type=ContentType(type), - role=Role(role), - text_stage=TextStage(generation_stage) if generation_stage else None, - text_content=None, - ) - self._content_being_received = content - - if content.role == Role.ASSISTANT: - if content.type == ContentType.AUDIO: - # Note that an assistant response can comprise of multiple audio blocks - if not self._assistant_is_responding: - # The assistant has started responding. - self._assistant_is_responding = True - await self._report_user_transcription_ended() # Consider user turn over - await self._report_assistant_response_started() - - async def _handle_text_output_event(self, event_json): - if not self._content_being_received: # should never happen - return - content = self._content_being_received - - text_content = event_json["textOutput"]["content"] - - # Bookkeeping: augment the current content being received with text - # Assumption: only one text content per content block - content.text_content = text_content - - async def _handle_audio_output_event(self, event_json): - if not self._content_being_received: # should never happen - return - - # Get audio - audio_content = event_json["audioOutput"]["content"] - - # Push audio frame - audio = base64.b64decode(audio_content) - frame = TTSAudioRawFrame( - audio=audio, - sample_rate=self._params.output_sample_rate, - num_channels=self._params.output_channel_count, - ) - await self.push_frame(frame) - - async def _handle_tool_use_event(self, event_json): - if not self._content_being_received or not self._context: # should never happen - return - - # Consider user turn over - await self._report_user_transcription_ended() - - # Get tool use details - tool_use = event_json["toolUse"] - function_name = tool_use["toolName"] - tool_call_id = tool_use["toolUseId"] - arguments = json.loads(tool_use["content"]) - - # Call tool function - if self.has_function(function_name): - if function_name in self._functions.keys() or None in self._functions.keys(): - function_calls_llm = [ - FunctionCallFromLLM( - context=self._context, - tool_call_id=tool_call_id, - function_name=function_name, - arguments=arguments, - ) - ] - - await self.run_function_calls(function_calls_llm) - else: - raise AWSNovaSonicUnhandledFunctionException( - f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function." - ) - - async def _handle_content_end_event(self, event_json): - if not self._content_being_received: # should never happen - return - content = self._content_being_received - - content_end = event_json["contentEnd"] - stop_reason = content_end["stopReason"] - - # Bookkeeping: clear current content being received - self._content_being_received = None - - if content.role == Role.ASSISTANT: - if content.type == ContentType.TEXT: - # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) - if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": - if self._assistant_is_responding: - # Text added to the ongoing assistant response - await self._report_assistant_response_text_added(content.text_content) - elif content.role == Role.USER: - if content.type == ContentType.TEXT: - if content.text_stage == TextStage.FINAL: - # User transcription text added - await self._report_user_transcription_text_added(content.text_content) - - async def _handle_completion_end_event(self, event_json): - pass - - # - # assistant response reporting - # - # 1. Started - # 2. Text added - # 3. Ended - # - - async def _report_assistant_response_started(self): - logger.debug("Assistant response started") - - # Report that the assistant has started their response. - await self.push_frame(LLMFullResponseStartFrame()) - - # Report that equivalent of TTS (this is a speech-to-speech model) started - await self.push_frame(TTSStartedFrame()) - - async def _report_assistant_response_text_added(self, text): - if not self._context: # should never happen - return - - logger.debug(f"Assistant response text added: {text}") - - # Report some text added to the ongoing assistant response - await self.push_frame(LLMTextFrame(text)) - - # Report some text added to the *equivalent* of TTS (this is a speech-to-speech model) - await self.push_frame(TTSTextFrame(text)) - - # TODO: this is a (hopefully temporary) HACK. Here we directly manipulate the context rather - # than relying on the frames pushed to the assistant context aggregator. The pattern of - # receiving full-sentence text after the assistant has spoken does not easily fit with the - # Pipecat expectation of chunks of text streaming in while the assistant is speaking. - # Interruption handling was especially challenging. Rather than spend days trying to fit a - # square peg in a round hole, I decided on this hack for the time being. We can most cleanly - # abandon this hack if/when AWS Nova Sonic implements streaming smaller text chunks - # interspersed with audio. Note that when we move away from this hack, we need to make sure - # that on an interruption we avoid sending LLMFullResponseEndFrame, which gets the - # LLMAssistantContextAggregator into a bad state. - self._context.buffer_assistant_text(text) - - async def _report_assistant_response_ended(self): - if not self._context: # should never happen - return - - logger.debug("Assistant response ended") - - # Report that the assistant has finished their response. - await self.push_frame(LLMFullResponseEndFrame()) - - # Report that equivalent of TTS (this is a speech-to-speech model) stopped. - await self.push_frame(TTSStoppedFrame()) - - # For an explanation of this hack, see _report_assistant_response_text_added. - self._context.flush_aggregated_assistant_text() - - # - # user transcription reporting - # - # 1. Text added - # 2. Ended - # - # Note: "started" does not need to be reported - # - - async def _report_user_transcription_text_added(self, text): - if not self._context: # should never happen - return - - logger.debug(f"User transcription text added: {text}") - - # Manually add new user transcription text to context. - # We can't rely on the user context aggregator to do this since it's upstream from the LLM. - self._context.buffer_user_text(text) - - # Report that some new user transcription text is available. - if self._send_transcription_frames: - await self.push_frame( - InterimTranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) - ) - - async def _report_user_transcription_ended(self): - if not self._context: # should never happen - return - - # Manually add user transcription to context (if any has been buffered). - # We can't rely on the user context aggregator to do this since it's upstream from the LLM. - transcription = self._context.flush_aggregated_user_text() - - if not transcription: - return - - logger.debug(f"User transcription ended") - - if self._send_transcription_frames: - await self.push_frame( - TranscriptionFrame(text=transcription, user_id="", timestamp=time_now_iso8601()) - ) - - # - # context - # - - def create_context_aggregator( - self, - context: OpenAILLMContext, - *, - user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), - assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), - ) -> AWSNovaSonicContextAggregatorPair: - """Create context aggregator pair for managing conversation context. - - Args: - context: The OpenAI LLM context to upgrade. - user_params: Parameters for the user context aggregator. - assistant_params: Parameters for the assistant context aggregator. - - Returns: - A pair of user and assistant context aggregators. - """ - context.set_llm_adapter(self.get_llm_adapter()) - - user = AWSNovaSonicUserContextAggregator(context=context, params=user_params) - assistant = AWSNovaSonicAssistantContextAggregator(context=context, params=assistant_params) - - return AWSNovaSonicContextAggregatorPair(user, assistant) - - # - # assistant response trigger (HACK) - # - - # Class variable - AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION = ( - "Start speaking when you hear the user say 'ready', but don't consider that 'ready' to be " - "a meaningful part of the conversation other than as a trigger for you to start speaking." - ) - - async def trigger_assistant_response(self): - """Trigger an assistant response by sending audio cue. - - Sends a pre-recorded "ready" audio trigger to prompt the assistant - to start speaking. This is useful for controlling conversation flow. - - Returns: - False if already triggering a response, True otherwise. - """ - if self._triggering_assistant_response: - return False - - self._triggering_assistant_response = True - - # Send the trigger audio, if we're fully connected and set up - if self._connected_time: - await self._send_assistant_response_trigger() - - async def _send_assistant_response_trigger(self): - if not self._connected_time: - # should never happen - return - - try: - logger.debug("Sending assistant response trigger...") - - chunk_duration = 0.02 # what we might get from InputAudioRawFrame - chunk_size = int( - chunk_duration - * self._params.input_sample_rate - * self._params.input_channel_count - * (self._params.input_sample_size / 8) - ) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes - - # Lead with a bit of blank audio, if needed. - # It seems like the LLM can't quite "hear" the first little bit of audio sent on a - # connection. - current_time = time.time() - max_blank_audio_duration = 0.5 - blank_audio_duration = ( - max_blank_audio_duration - (current_time - self._connected_time) - if self._connected_time is not None - and (current_time - self._connected_time) < max_blank_audio_duration - else None - ) - if blank_audio_duration: - logger.debug( - f"Leading assistant response trigger with {blank_audio_duration}s of blank audio" - ) - blank_audio_chunk = b"\x00" * chunk_size - num_chunks = int(blank_audio_duration / chunk_duration) - for _ in range(num_chunks): - await self._send_user_audio_event(blank_audio_chunk) - await asyncio.sleep(chunk_duration) - - # Send trigger audio - # NOTE: this audio *will* be transcribed and eventually make it into the context. That's OK: - # if we ever need to seed this service again with context it would make sense to include it - # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the - # context as well. - audio_chunks = [ - self._assistant_response_trigger_audio[i : i + chunk_size] - for i in range(0, len(self._assistant_response_trigger_audio), chunk_size) - ] - for chunk in audio_chunks: - await self._send_user_audio_event(chunk) - await asyncio.sleep(chunk_duration) - finally: - # We need to clean up in case sending the trigger was cancelled, e.g. in the case of a user interruption. - # (An asyncio.CancelledError would be raised in that case.) - self._triggering_assistant_response = False diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 0ce5ce033..5728e4ff0 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -8,360 +8,14 @@ This module provides specialized context aggregators and message handling for AWS Nova Sonic, including conversation history management and role-specific message processing. + +.. deprecated:: 0.0.91 + AWS Nova Sonic no longer uses types from this module under the hood. + It now uses `LLMContext` and `LLMContextAggregatorPair`. + Using the new patterns should allow you to not need types from this module. + + See deprecation warning in pipecat.services.aws.nova_sonic.context for more + details. """ -import copy -from dataclasses import dataclass, field -from enum import Enum - -from loguru import logger - -from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, - DataFrame, - Frame, - FunctionCallResultFrame, - InterruptionFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMMessagesAppendFrame, - LLMMessagesUpdateFrame, - LLMSetToolChoiceFrame, - LLMSetToolsFrame, - TextFrame, - UserImageRawFrame, -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) - - -class Role(Enum): - """Roles supported in AWS Nova Sonic conversations. - - Parameters: - SYSTEM: System-level messages (not used in conversation history). - USER: Messages sent by the user. - ASSISTANT: Messages sent by the assistant. - TOOL: Messages sent by tools (not used in conversation history). - """ - - SYSTEM = "SYSTEM" - USER = "USER" - ASSISTANT = "ASSISTANT" - TOOL = "TOOL" - - -@dataclass -class AWSNovaSonicConversationHistoryMessage: - """A single message in AWS Nova Sonic conversation history. - - Parameters: - role: The role of the message sender (USER or ASSISTANT only). - text: The text content of the message. - """ - - role: Role # only USER and ASSISTANT - text: str - - -@dataclass -class AWSNovaSonicConversationHistory: - """Complete conversation history for AWS Nova Sonic initialization. - - Parameters: - system_instruction: System-level instruction for the conversation. - messages: List of conversation messages between user and assistant. - """ - - system_instruction: str = None - messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) - - -class AWSNovaSonicLLMContext(OpenAILLMContext): - """Specialized LLM context for AWS Nova Sonic service. - - Extends OpenAI context with Nova Sonic-specific message handling, - conversation history management, and text buffering capabilities. - """ - - def __init__(self, messages=None, tools=None, **kwargs): - """Initialize AWS Nova Sonic LLM context. - - Args: - messages: Initial messages for the context. - tools: Available tools for the context. - **kwargs: Additional arguments passed to parent class. - """ - super().__init__(messages=messages, tools=tools, **kwargs) - self.__setup_local() - - def __setup_local(self, system_instruction: str = ""): - self._assistant_text = "" - self._user_text = "" - self._system_instruction = system_instruction - - @staticmethod - def upgrade_to_nova_sonic( - obj: OpenAILLMContext, system_instruction: str - ) -> "AWSNovaSonicLLMContext": - """Upgrade an OpenAI context to AWS Nova Sonic context. - - Args: - obj: The OpenAI context to upgrade. - system_instruction: System instruction for the context. - - Returns: - The upgraded AWS Nova Sonic context. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): - obj.__class__ = AWSNovaSonicLLMContext - obj.__setup_local(system_instruction) - return obj - - # NOTE: this method has the side-effect of updating _system_instruction from messages - def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: - """Get conversation history for initializing AWS Nova Sonic session. - - Processes stored messages and extracts system instruction and conversation - history in the format expected by AWS Nova Sonic. - - Returns: - Formatted conversation history with system instruction and messages. - """ - history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) - - # Bail if there are no messages - if not self.messages: - return history - - messages = copy.deepcopy(self.messages) - - # If we have a "system" message as our first message, let's pull that out into "instruction" - if messages[0].get("role") == "system": - system = messages.pop(0) - content = system.get("content") - if isinstance(content, str): - history.system_instruction = content - elif isinstance(content, list): - history.system_instruction = content[0].get("text") - if history.system_instruction: - self._system_instruction = history.system_instruction - - # Process remaining messages to fill out conversation history. - # Nova Sonic supports "user" and "assistant" messages in history. - for message in messages: - history_message = self.from_standard_message(message) - if history_message: - history.messages.append(history_message) - - return history - - def get_messages_for_persistent_storage(self): - """Get messages formatted for persistent storage. - - Returns: - List of messages including system instruction if present. - """ - messages = super().get_messages_for_persistent_storage() - # If we have a system instruction and messages doesn't already contain it, add it - if self._system_instruction and not (messages and messages[0].get("role") == "system"): - messages.insert(0, {"role": "system", "content": self._system_instruction}) - return messages - - def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: - """Convert standard message format to Nova Sonic format. - - Args: - message: Standard message dictionary to convert. - - Returns: - Nova Sonic conversation history message, or None if not convertible. - """ - role = message.get("role") - if message.get("role") == "user" or message.get("role") == "assistant": - content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): - if c.get("type") == "text": - content += " " + c.get("text") - else: - logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" - ) - # There won't be content if this is an assistant tool call entry. - # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation - # history - if content: - return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) - # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova - # Sonic conversation history - - def buffer_user_text(self, text): - """Buffer user text for later flushing to context. - - Args: - text: User text to buffer. - """ - self._user_text += f" {text}" if self._user_text else text - # logger.debug(f"User text buffered: {self._user_text}") - - def flush_aggregated_user_text(self) -> str: - """Flush buffered user text to context as a complete message. - - Returns: - The flushed user text, or empty string if no text was buffered. - """ - if not self._user_text: - return "" - user_text = self._user_text - message = { - "role": "user", - "content": [{"type": "text", "text": user_text}], - } - self._user_text = "" - self.add_message(message) - # logger.debug(f"Context updated (user): {self.get_messages_for_logging()}") - return user_text - - def buffer_assistant_text(self, text): - """Buffer assistant text for later flushing to context. - - Args: - text: Assistant text to buffer. - """ - self._assistant_text += text - # logger.debug(f"Assistant text buffered: {self._assistant_text}") - - def flush_aggregated_assistant_text(self): - """Flush buffered assistant text to context as a complete message.""" - if not self._assistant_text: - return - message = { - "role": "assistant", - "content": [{"type": "text", "text": self._assistant_text}], - } - self._assistant_text = "" - self.add_message(message) - # logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}") - - -@dataclass -class AWSNovaSonicMessagesUpdateFrame(DataFrame): - """Frame containing updated AWS Nova Sonic context. - - Parameters: - context: The updated AWS Nova Sonic LLM context. - """ - - context: AWSNovaSonicLLMContext - - -class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): - """Context aggregator for user messages in AWS Nova Sonic conversations. - - Extends the OpenAI user context aggregator to emit Nova Sonic-specific - context update frames. - """ - - async def process_frame( - self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM - ): - """Process frames and emit Nova Sonic-specific context updates. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - await super().process_frame(frame, direction) - - # Parent does not push LLMMessagesUpdateFrame - if isinstance(frame, LLMMessagesUpdateFrame): - await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) - - -class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Context aggregator for assistant messages in AWS Nova Sonic conversations. - - Provides specialized handling for assistant responses and function calls - in AWS Nova Sonic context, with custom frame processing logic. - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames with Nova Sonic-specific logic. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - # HACK: For now, disable the context aggregator by making it just pass through all frames - # that the parent handles (except the function call stuff, which we still need). - # For an explanation of this hack, see - # AWSNovaSonicLLMService._report_assistant_response_text_added. - if isinstance( - frame, - ( - InterruptionFrame, - LLMFullResponseStartFrame, - LLMFullResponseEndFrame, - TextFrame, - LLMMessagesAppendFrame, - LLMMessagesUpdateFrame, - LLMSetToolsFrame, - LLMSetToolChoiceFrame, - UserImageRawFrame, - BotStoppedSpeakingFrame, - ), - ): - await self.push_frame(frame, direction) - else: - await super().process_frame(frame, direction) - - async def handle_function_call_result(self, frame: FunctionCallResultFrame): - """Handle function call results for AWS Nova Sonic. - - Args: - frame: The function call result frame to handle. - """ - await super().handle_function_call_result(frame) - - # The standard function callback code path pushes the FunctionCallResultFrame from the LLM - # itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side - # context. Let's push a special frame to do that. - await self.push_frame( - AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) - - -@dataclass -class AWSNovaSonicContextAggregatorPair: - """Pair of user and assistant context aggregators for AWS Nova Sonic. - - Parameters: - _user: The user context aggregator. - _assistant: The assistant context aggregator. - """ - - _user: AWSNovaSonicUserContextAggregator - _assistant: AWSNovaSonicAssistantContextAggregator - - def user(self) -> AWSNovaSonicUserContextAggregator: - """Get the user context aggregator. - - Returns: - The user context aggregator instance. - """ - return self._user - - def assistant(self) -> AWSNovaSonicAssistantContextAggregator: - """Get the assistant context aggregator. - - Returns: - The assistant context aggregator instance. - """ - return self._assistant +from pipecat.services.aws.nova_sonic.context import * diff --git a/src/pipecat/services/aws_nova_sonic/frames.py b/src/pipecat/services/aws_nova_sonic/frames.py index 7d4feb2ae..def5f26c4 100644 --- a/src/pipecat/services/aws_nova_sonic/frames.py +++ b/src/pipecat/services/aws_nova_sonic/frames.py @@ -6,20 +6,16 @@ """Custom frames for AWS Nova Sonic LLM service.""" -from dataclasses import dataclass +import warnings -from pipecat.frames.frames import DataFrame, FunctionCallResultFrame +from pipecat.services.aws.nova_sonic.frames import * - -@dataclass -class AWSNovaSonicFunctionCallResultFrame(DataFrame): - """Frame containing function call result for AWS Nova Sonic processing. - - This frame wraps a standard function call result frame to enable - AWS Nova Sonic-specific handling and context updates. - - Parameters: - result_frame: The underlying function call result frame. - """ - - result_frame: FunctionCallResultFrame +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic.frames are deprecated. " + "Please use the equivalent types from " + "pipecat.services.aws.nova_sonic.frames instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/azure/realtime/__init__.py b/src/pipecat/services/azure/realtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/azure/realtime/llm.py b/src/pipecat/services/azure/realtime/llm.py new file mode 100644 index 000000000..1193b82d4 --- /dev/null +++ b/src/pipecat/services/azure/realtime/llm.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Azure OpenAI Realtime LLM service implementation.""" + +from loguru import logger + +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService + +try: + from websockets.asyncio.client import connect as websocket_connect +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Azure Realtime, you need to `pip install pipecat-ai[openai]`.") + raise Exception(f"Missing module: {e}") + + +class AzureRealtimeLLMService(OpenAIRealtimeLLMService): + """Azure OpenAI Realtime LLM service with Azure-specific authentication. + + Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, + using Azure's authentication headers and endpoint format. Provides the same + real-time audio and text communication capabilities as the base OpenAI service. + """ + + def __init__( + self, + *, + api_key: str, + base_url: str, + **kwargs, + ): + """Initialize Azure Realtime LLM service. + + Args: + api_key: The API key for the Azure OpenAI service. + base_url: The full Azure WebSocket endpoint URL including api-version and deployment. + Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + **kwargs: Additional arguments passed to parent OpenAIRealtimeLLMService. + """ + super().__init__(base_url=base_url, api_key=api_key, **kwargs) + self.api_key = api_key + self.base_url = base_url + + async def _connect(self): + try: + if self._websocket: + # Here we assume that if we have a websocket, we are connected. We + # handle disconnections in the send/recv code paths. + return + + logger.info(f"Connecting to {self.base_url}") + self._websocket = await websocket_connect( + uri=self.base_url, + additional_headers={ + "api-key": self.api_key, + }, + ) + self._receive_task = self.create_task(self._receive_task_handler()) + except Exception as e: + logger.error(f"{self} initialization error: {e}") + self._websocket = None diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index 5412c422c..b4e232c4a 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -28,13 +28,12 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.stt_service import STTService +from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt try: - import websockets from websockets.asyncio.client import connect as websocket_connect from websockets.protocol import State except ModuleNotFoundError as e: @@ -124,7 +123,7 @@ class CartesiaLiveOptions: return cls(**json.loads(json_str)) -class CartesiaSTTService(STTService): +class CartesiaSTTService(WebsocketSTTService): """Speech-to-text service using Cartesia Live API. Provides real-time speech transcription through WebSocket connection @@ -176,8 +175,7 @@ class CartesiaSTTService(STTService): self.set_model_name(merged_options.model) self._api_key = api_key self._base_url = base_url or "api.cartesia.ai" - self._connection = None - self._receiver_task = None + self._receive_task = None def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -214,6 +212,27 @@ class CartesiaSTTService(STTService): await super().cancel(frame) await self._disconnect() + async def start_metrics(self): + """Start performance metrics collection for transcription processing.""" + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle speech events. + + Args: + frame: The frame to process. + direction: Direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, UserStartedSpeakingFrame): + await self.start_metrics() + elif isinstance(frame, UserStoppedSpeakingFrame): + # Send finalize command to flush the transcription session + if self._websocket and self._websocket.state is State.OPEN: + await self._websocket.send("finalize") + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process audio data for speech-to-text transcription. @@ -224,45 +243,71 @@ class CartesiaSTTService(STTService): None - transcription results are handled via WebSocket responses. """ # If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again - if not self._connection or self._connection.state is State.CLOSED: + if not self._websocket or self._websocket.state is State.CLOSED: await self._connect() - await self._connection.send(audio) + await self._websocket.send(audio) yield None async def _connect(self): - params = self._settings.to_dict() - ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" - logger.debug(f"Connecting to Cartesia: {ws_url}") - headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} + await self._connect_websocket() + if self._websocket and not self._receive_task: + self._receive_task = asyncio.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): try: - self._connection = await websocket_connect(ws_url, additional_headers=headers) - # Setup the receiver task to handle the incoming messages from the Cartesia server - if self._receiver_task is None or self._receiver_task.done(): - self._receiver_task = asyncio.create_task(self._receive_messages()) - logger.debug(f"Connected to Cartesia") + if self._websocket and self._websocket.state is State.OPEN: + return + logger.debug("Connecting to Cartesia STT") + + params = self._settings.to_dict() + ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" + headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} + + self._websocket = await websocket_connect(ws_url, additional_headers=headers) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self}: unable to connect to Cartesia: {e}") - async def _receive_messages(self): + async def _disconnect_websocket(self): try: - while True: - if not self._connection or self._connection.state is State.CLOSED: - break - - message = await self._connection.recv() - try: - data = json.loads(message) - await self._process_response(data) - except json.JSONDecodeError: - logger.warning(f"Received non-JSON message: {message}") - except asyncio.CancelledError: - pass - except websockets.exceptions.ConnectionClosed as e: - logger.debug(f"WebSocket connection closed: {e}") + if self._websocket and self._websocket.state is State.OPEN: + logger.debug("Disconnecting from Cartesia STT") + await self._websocket.close() except Exception as e: - logger.error(f"Error in message receiver: {e}") + logger.error(f"{self} error closing websocket: {e}") + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _process_messages(self): + async for message in self._get_websocket(): + try: + data = json.loads(message) + await self._process_response(data) + except json.JSONDecodeError: + logger.warning(f"Received non-JSON message: {message}") + + async def _receive_messages(self): + while True: + await self._process_messages() + # Cartesia times out after 5 minutes of innactivity (no keepalive + # mechanism is available). So, we try to reconnect. + logger.debug(f"{self} Cartesia connection was disconnected (timeout?), reconnecting") + await self._connect_websocket() async def _process_response(self, data): if "type" in data: @@ -316,41 +361,3 @@ class CartesiaSTTService(STTService): language, ) ) - - async def _disconnect(self): - if self._receiver_task: - self._receiver_task.cancel() - try: - await self._receiver_task - except asyncio.CancelledError: - pass - except Exception as e: - logger.exception(f"Unexpected exception while cancelling task: {e}") - self._receiver_task = None - - if self._connection and self._connection.state is State.OPEN: - logger.debug("Disconnecting from Cartesia") - - await self._connection.close() - self._connection = None - - async def start_metrics(self): - """Start performance metrics collection for transcription processing.""" - await self.start_ttfb_metrics() - await self.start_processing_metrics() - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames and handle speech events. - - Args: - frame: The frame to process. - direction: Direction of frame flow in the pipeline. - """ - await super().process_frame(frame, direction) - - if isinstance(frame, UserStartedSpeakingFrame): - await self.start_metrics() - elif isinstance(frame, UserStoppedSpeakingFrame): - # Send finalize command to flush the transcription session - if self._connection and self._connection.state is State.OPEN: - await self._connection.send("finalize") diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 3b81da5d4..3c0fe279c 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -119,7 +119,7 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_id: str, cartesia_version: str = "2025-04-16", url: str = "wss://api.cartesia.ai/tts/websocket", - model: str = "sonic-2", + model: str = "sonic-3", sample_rate: Optional[int] = None, encoding: str = "pcm_s16le", container: str = "raw", @@ -135,7 +135,7 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_id: ID of the voice to use for synthesis. cartesia_version: API version string for Cartesia service. url: WebSocket URL for Cartesia TTS API. - model: TTS model to use (e.g., "sonic-2"). + model: TTS model to use (e.g., "sonic-3"). sample_rate: Audio sample rate. If None, uses default. encoding: Audio encoding format. container: Audio container format. @@ -344,10 +344,11 @@ class CartesiaTTSService(AudioContextWordTTSService): try: if self._websocket and self._websocket.state is State.OPEN: return - logger.debug("Connecting to Cartesia") + logger.debug("Connecting to Cartesia TTS") self._websocket = await websocket_connect( f"{self._url}?api_key={self._api_key}&cartesia_version={self._cartesia_version}" ) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -365,6 +366,7 @@ class CartesiaTTSService(AudioContextWordTTSService): finally: self._context_id = None self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: @@ -496,7 +498,7 @@ class CartesiaHttpTTSService(TTSService): *, api_key: str, voice_id: str, - model: str = "sonic-2", + model: str = "sonic-3", base_url: str = "https://api.cartesia.ai", cartesia_version: str = "2024-11-13", sample_rate: Optional[int] = None, @@ -510,7 +512,7 @@ class CartesiaHttpTTSService(TTSService): Args: api_key: Cartesia API key for authentication. voice_id: ID of the voice to use for synthesis. - model: TTS model to use (e.g., "sonic-2"). + model: TTS model to use (e.g., "sonic-3"). base_url: Base URL for Cartesia HTTP API. cartesia_version: API version string for Cartesia service. sample_rate: Audio sample rate. If None, uses default. diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 493bece80..f0b1a5baa 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -205,6 +205,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): additional_headers={"Authorization": f"Token {self._api_key}"}, ) logger.debug("Connected to Deepgram Flux Websocket") + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -225,6 +226,9 @@ class DeepgramFluxSTTService(WebsocketSTTService): await self._websocket.close() except Exception as e: logger.error(f"{self} error closing websocket: {e}") + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") async def _send_close_stream(self) -> None: """Sends a CloseStream control message to the Deepgram Flux WebSocket API. diff --git a/src/pipecat/services/elevenlabs/__init__.py b/src/pipecat/services/elevenlabs/__init__.py index e5a76e71a..fca70eaec 100644 --- a/src/pipecat/services/elevenlabs/__init__.py +++ b/src/pipecat/services/elevenlabs/__init__.py @@ -8,6 +8,7 @@ import sys from pipecat.services import DeprecatedModuleProxy +from .stt import * from .tts import * -sys.modules[__name__] = DeprecatedModuleProxy(globals(), "elevenlabs", "elevenlabs.tts") +sys.modules[__name__] = DeprecatedModuleProxy(globals(), "elevenlabs", "elevenlabs.[stt,tts]") diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 3d6d5bd4c..460b23d18 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -168,16 +168,24 @@ def build_elevenlabs_voice_settings( def calculate_word_times( - alignment_info: Mapping[str, Any], cumulative_time: float -) -> List[Tuple[str, float]]: + alignment_info: Mapping[str, Any], + cumulative_time: float, + partial_word: str = "", + partial_word_start_time: float = 0.0, +) -> tuple[List[Tuple[str, float]], str, float]: """Calculate word timestamps from character alignment information. Args: alignment_info: Character alignment data from ElevenLabs API. cumulative_time: Base time offset for this chunk. + partial_word: Partial word carried over from previous chunk. + partial_word_start_time: Start time of the partial word. Returns: - List of (word, timestamp) tuples. + Tuple of (word_times, new_partial_word, new_partial_word_start_time): + - word_times: List of (word, timestamp) tuples for complete words + - new_partial_word: Incomplete word at end of chunk (empty if chunk ends with space) + - new_partial_word_start_time: Start time of the incomplete word """ chars = alignment_info["chars"] char_start_times_ms = alignment_info["charStartTimesMs"] @@ -186,41 +194,37 @@ def calculate_word_times( logger.error( f"calculate_word_times: length mismatch - chars={len(chars)}, times={len(char_start_times_ms)}" ) - return [] + return ([], partial_word, partial_word_start_time) # Build words and track their start positions words = [] - word_start_indices = [] - current_word = "" - word_start_index = None + word_start_times = [] + current_word = partial_word # Start with any partial word from previous chunk + word_start_time = partial_word_start_time if partial_word else None for i, char in enumerate(chars): if char == " ": # End of current word if current_word: # Only add non-empty words words.append(current_word) - word_start_indices.append(word_start_index) + word_start_times.append(word_start_time) current_word = "" - word_start_index = None + word_start_time = None else: # Building a word - if word_start_index is None: # First character of new word - word_start_index = i + if word_start_time is None: # First character of new word + # Convert from milliseconds to seconds and add cumulative offset + word_start_time = cumulative_time + (char_start_times_ms[i] / 1000.0) current_word += char - # Handle the last word if there's no trailing space - if current_word and word_start_index is not None: - words.append(current_word) - word_start_indices.append(word_start_index) + # Build result for complete words + word_times = list(zip(words, word_start_times)) - # Calculate timestamps for each word - word_times = [] - for word, start_idx in zip(words, word_start_indices): - # Convert from milliseconds to seconds and add cumulative offset - start_time_seconds = cumulative_time + (char_start_times_ms[start_idx] / 1000.0) - word_times.append((word, start_time_seconds)) + # Return any incomplete word at the end of this chunk + new_partial_word = current_word if current_word else "" + new_partial_word_start_time = word_start_time if word_start_time is not None else 0.0 - return word_times + return (word_times, new_partial_word, new_partial_word_start_time) class ElevenLabsTTSService(AudioContextWordTTSService): @@ -332,6 +336,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService): # there's an interruption or TTSStoppedFrame. self._started = False self._cumulative_time = 0 + # Track partial words that span across alignment chunks + self._partial_word = "" + self._partial_word_start_time = 0.0 # Context management for v1 multi API self._context_id = None @@ -521,6 +528,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): url, max_size=16 * 1024 * 1024, additional_headers={"xi-api-key": self._api_key} ) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -543,6 +551,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._started = False self._context_id = None self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: @@ -570,6 +579,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): logger.error(f"Error closing context on interruption: {e}") self._context_id = None self._started = False + self._partial_word = "" + self._partial_word_start_time = 0.0 async def _receive_messages(self): """Handle incoming WebSocket messages from ElevenLabs.""" @@ -609,7 +620,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService): if msg.get("alignment"): alignment = msg["alignment"] - word_times = calculate_word_times(alignment, self._cumulative_time) + word_times, self._partial_word, self._partial_word_start_time = ( + calculate_word_times( + alignment, + self._cumulative_time, + self._partial_word, + self._partial_word_start_time, + ) + ) if word_times: await self.add_word_timestamps(word_times) @@ -683,6 +701,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): yield TTSStartedFrame() self._started = True self._cumulative_time = 0 + self._partial_word = "" + self._partial_word_start_time = 0.0 # If a context ID does not exist, create a new one and # register it. If an ID exists, that means the Pipeline is # configured for allow_interruptions=False, so continue @@ -756,6 +776,7 @@ class ElevenLabsHttpTTSService(WordTTSService): base_url: str = "https://api.elevenlabs.io", sample_rate: Optional[int] = None, params: Optional[InputParams] = None, + aggregate_sentences: Optional[bool] = True, **kwargs, ): """Initialize the ElevenLabs HTTP TTS service. @@ -768,10 +789,11 @@ class ElevenLabsHttpTTSService(WordTTSService): base_url: Base URL for ElevenLabs HTTP API. sample_rate: Audio sample rate. If None, uses default. params: Additional input parameters for voice customization. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to the parent service. """ super().__init__( - aggregate_sentences=True, + aggregate_sentences=aggregate_sentences, push_text_frames=False, push_stop_frames=True, sample_rate=sample_rate, @@ -809,6 +831,10 @@ class ElevenLabsHttpTTSService(WordTTSService): # Store previous text for context within a turn self._previous_text = "" + # Track partial words that span across alignment chunks + self._partial_word = "" + self._partial_word_start_time = 0.0 + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert pipecat Language to ElevenLabs language code. @@ -836,6 +862,8 @@ class ElevenLabsHttpTTSService(WordTTSService): self._cumulative_time = 0 self._started = False self._previous_text = "" + self._partial_word = "" + self._partial_word_start_time = 0.0 logger.debug(f"{self}: Reset internal state") async def start(self, frame: StartFrame): @@ -870,11 +898,13 @@ class ElevenLabsHttpTTSService(WordTTSService): def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]: """Calculate word timing from character alignment data. + This method handles partial words that may span across multiple alignment chunks. + Args: alignment_info: Character timing data from ElevenLabs. Returns: - List of (word, timestamp) pairs. + List of (word, timestamp) pairs for complete words in this chunk. Example input data:: @@ -900,30 +930,28 @@ class ElevenLabsHttpTTSService(WordTTSService): # Build the words and find their start times words = [] word_start_times = [] - current_word = "" - first_char_idx = -1 + # Start with any partial word from previous chunk + current_word = self._partial_word + word_start_time = self._partial_word_start_time if self._partial_word else None for i, char in enumerate(chars): if char == " ": if current_word: # Only add non-empty words words.append(current_word) - # Use time of the first character of the word, offset by cumulative time - word_start_times.append( - self._cumulative_time + char_start_times[first_char_idx] - ) + word_start_times.append(word_start_time) current_word = "" - first_char_idx = -1 + word_start_time = None else: - if not current_word: # This is the first character of a new word - first_char_idx = i + if word_start_time is None: # First character of a new word + # Use time of the first character of the word, offset by cumulative time + word_start_time = self._cumulative_time + char_start_times[i] current_word += char - # Don't forget the last word if there's no trailing space - if current_word and first_char_idx >= 0: - words.append(current_word) - word_start_times.append(self._cumulative_time + char_start_times[first_char_idx]) + # Store any incomplete word at the end of this chunk + self._partial_word = current_word if current_word else "" + self._partial_word_start_time = word_start_time if word_start_time is not None else 0.0 - # Create word-time pairs + # Create word-time pairs for complete words only word_times = list(zip(words, word_start_times)) return word_times @@ -959,6 +987,9 @@ class ElevenLabsHttpTTSService(WordTTSService): if self._voice_settings: payload["voice_settings"] = self._voice_settings + if self._settings["apply_text_normalization"] is not None: + payload["apply_text_normalization"] = self._settings["apply_text_normalization"] + language = self._settings["language"] if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language: payload["language_code"] = language @@ -979,8 +1010,6 @@ class ElevenLabsHttpTTSService(WordTTSService): } if self._settings["optimize_streaming_latency"] is not None: params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"] - if self._settings["apply_text_normalization"] is not None: - params["apply_text_normalization"] = self._settings["apply_text_normalization"] try: await self.start_ttfb_metrics() @@ -1041,6 +1070,14 @@ class ElevenLabsHttpTTSService(WordTTSService): logger.error(f"Error processing response: {e}", exc_info=True) continue + # After processing all chunks, emit any remaining partial word + # since this is the end of the utterance + if self._partial_word: + final_word_time = [(self._partial_word, self._partial_word_start_time)] + await self.add_word_timestamps(final_word_time) + self._partial_word = "" + self._partial_word_start_time = 0.0 + # After processing all chunks, add the total utterance duration # to the cumulative time to ensure next utterance starts after this one if utterance_duration > 0: diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index b39b775e5..669d2ce97 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -225,6 +225,8 @@ class FishAudioTTSService(InterruptibleTTSService): start_message = {"event": "start", "request": {"text": "", **self._settings}} await self._websocket.send(ormsgpack.packb(start_message)) logger.debug("Sent start message to Fish Audio") + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"Fish Audio initialization error: {e}") self._websocket = None @@ -245,6 +247,7 @@ class FishAudioTTSService(InterruptibleTTSService): self._request_id = None self._started = False self._websocket = None + await self._call_event_handler("on_disconnected") async def flush_audio(self): """Flush any buffered audio by sending a flush event to Fish Audio.""" diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index 1766cf806..be69033aa 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -4,527 +4,41 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Event models and utilities for Google Gemini Multimodal Live API.""" - -import base64 -import io -import json -from enum import Enum -from typing import List, Literal, Optional - -from PIL import Image -from pydantic import BaseModel, Field - -from pipecat.frames.frames import ImageRawFrame - -# -# Client events -# - - -class MediaChunk(BaseModel): - """Represents a chunk of media data for transmission. - - Parameters: - mimeType: MIME type of the media content. - data: Base64-encoded media data. - """ - - mimeType: str - data: str - - -class ContentPart(BaseModel): - """Represents a part of content that can contain text or media. - - Parameters: - text: Text content. Defaults to None. - inlineData: Inline media data. Defaults to None. - """ - - text: Optional[str] = Field(default=None, validate_default=False) - inlineData: Optional[MediaChunk] = Field(default=None, validate_default=False) - fileData: Optional["FileData"] = Field(default=None, validate_default=False) - - -class FileData(BaseModel): - """Represents a file reference in the Gemini File API.""" - - mimeType: str - fileUri: str - - -ContentPart.model_rebuild() # Rebuild model to resolve forward reference - - -class Turn(BaseModel): - """Represents a conversational turn in the dialogue. - - Parameters: - role: The role of the speaker, either "user" or "model". Defaults to "user". - parts: List of content parts that make up the turn. - """ - - role: Literal["user", "model"] = "user" - parts: List[ContentPart] - - -class StartSensitivity(str, Enum): - """Determines how start of speech is detected.""" - - UNSPECIFIED = "START_SENSITIVITY_UNSPECIFIED" # Default is HIGH - HIGH = "START_SENSITIVITY_HIGH" # Detect start of speech more often - LOW = "START_SENSITIVITY_LOW" # Detect start of speech less often - - -class EndSensitivity(str, Enum): - """Determines how end of speech is detected.""" - - UNSPECIFIED = "END_SENSITIVITY_UNSPECIFIED" # Default is HIGH - HIGH = "END_SENSITIVITY_HIGH" # End speech more often - LOW = "END_SENSITIVITY_LOW" # End speech less often - - -class AutomaticActivityDetection(BaseModel): - """Configures automatic detection of voice activity. - - Parameters: - disabled: Whether automatic activity detection is disabled. Defaults to None. - start_of_speech_sensitivity: Sensitivity for detecting speech start. Defaults to None. - prefix_padding_ms: Padding before speech start in milliseconds. Defaults to None. - end_of_speech_sensitivity: Sensitivity for detecting speech end. Defaults to None. - silence_duration_ms: Duration of silence to detect speech end. Defaults to None. - """ - - disabled: Optional[bool] = None - start_of_speech_sensitivity: Optional[StartSensitivity] = None - prefix_padding_ms: Optional[int] = None - end_of_speech_sensitivity: Optional[EndSensitivity] = None - silence_duration_ms: Optional[int] = None - - -class RealtimeInputConfig(BaseModel): - """Configures the realtime input behavior. - - Parameters: - automatic_activity_detection: Voice activity detection configuration. Defaults to None. - """ - - automatic_activity_detection: Optional[AutomaticActivityDetection] = None - - -class RealtimeInput(BaseModel): - """Contains realtime input media chunks and text. - - Parameters: - mediaChunks: List of media chunks for realtime processing. - text: Text for realtime processing. - """ - - mediaChunks: Optional[List[MediaChunk]] = None - text: Optional[str] = None - - -class ClientContent(BaseModel): - """Content sent from client to the Gemini Live API. - - Parameters: - turns: List of conversation turns. Defaults to None. - turnComplete: Whether the client's turn is complete. Defaults to False. - """ - - turns: Optional[List[Turn]] = None - turnComplete: bool = False - - -class AudioInputMessage(BaseModel): - """Message containing audio input data. - - Parameters: - realtimeInput: Realtime input containing audio chunks. - """ - - realtimeInput: RealtimeInput - - @classmethod - def from_raw_audio(cls, raw_audio: bytes, sample_rate: int) -> "AudioInputMessage": - """Create an audio input message from raw audio data. - - Args: - raw_audio: Raw audio bytes. - sample_rate: Audio sample rate in Hz. - - Returns: - AudioInputMessage instance with encoded audio data. - """ - data = base64.b64encode(raw_audio).decode("utf-8") - return cls( - realtimeInput=RealtimeInput( - mediaChunks=[MediaChunk(mimeType=f"audio/pcm;rate={sample_rate}", data=data)] - ) - ) - - -class VideoInputMessage(BaseModel): - """Message containing video/image input data. - - Parameters: - realtimeInput: Realtime input containing video/image chunks. - """ - - realtimeInput: RealtimeInput - - @classmethod - def from_image_frame(cls, frame: ImageRawFrame) -> "VideoInputMessage": - """Create a video input message from an image frame. - - Args: - frame: Image frame to encode. - - Returns: - VideoInputMessage instance with encoded image data. - """ - buffer = io.BytesIO() - Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") - data = base64.b64encode(buffer.getvalue()).decode("utf-8") - return cls( - realtimeInput=RealtimeInput(mediaChunks=[MediaChunk(mimeType=f"image/jpeg", data=data)]) - ) - - -class TextInputMessage(BaseModel): - """Message containing text input data.""" - - realtimeInput: RealtimeInput - - @classmethod - def from_text(cls, text: str) -> "TextInputMessage": - """Create a text input message from a string. - - Args: - text: The text to send. - - Returns: - A TextInputMessage instance. - """ - return cls(realtimeInput=RealtimeInput(text=text)) - - -class ClientContentMessage(BaseModel): - """Message containing client content for the API. - - Parameters: - clientContent: The client content to send. - """ - - clientContent: ClientContent - - -class SystemInstruction(BaseModel): - """System instruction for the model. - - Parameters: - parts: List of content parts that make up the system instruction. - """ - - parts: List[ContentPart] - - -class AudioTranscriptionConfig(BaseModel): - """Configuration for audio transcription.""" - - pass - - -class Setup(BaseModel): - """Setup configuration for the Gemini Live session. - - Parameters: - model: Model identifier to use. - system_instruction: System instruction for the model. Defaults to None. - tools: List of available tools/functions. Defaults to None. - generation_config: Generation configuration parameters. Defaults to None. - input_audio_transcription: Input audio transcription config. Defaults to None. - output_audio_transcription: Output audio transcription config. Defaults to None. - realtime_input_config: Realtime input configuration. Defaults to None. - """ - - model: str - system_instruction: Optional[SystemInstruction] = None - tools: Optional[List[dict]] = None - generation_config: Optional[dict] = None - input_audio_transcription: Optional[AudioTranscriptionConfig] = None - output_audio_transcription: Optional[AudioTranscriptionConfig] = None - realtime_input_config: Optional[RealtimeInputConfig] = None - - -class Config(BaseModel): - """Configuration message for session setup. - - Parameters: - setup: Setup configuration for the session. - """ - - setup: Setup - - -# -# Grounding metadata models -# - - -class SearchEntryPoint(BaseModel): - """Represents the search entry point with rendered content for search suggestions.""" - - renderedContent: Optional[str] = None - - -class WebSource(BaseModel): - """Represents a web source from grounding chunks.""" - - uri: Optional[str] = None - title: Optional[str] = None - - -class GroundingChunk(BaseModel): - """Represents a grounding chunk containing web source information.""" - - web: Optional[WebSource] = None - - -class GroundingSegment(BaseModel): - """Represents a segment of text that is grounded.""" - - startIndex: Optional[int] = None - endIndex: Optional[int] = None - text: Optional[str] = None - - -class GroundingSupport(BaseModel): - """Represents support information for grounded text segments.""" - - segment: Optional[GroundingSegment] = None - groundingChunkIndices: Optional[List[int]] = None - confidenceScores: Optional[List[float]] = None - - -class GroundingMetadata(BaseModel): - """Represents grounding metadata from Google Search.""" - - searchEntryPoint: Optional[SearchEntryPoint] = None - groundingChunks: Optional[List[GroundingChunk]] = None - groundingSupports: Optional[List[GroundingSupport]] = None - webSearchQueries: Optional[List[str]] = None - - -# -# Server events -# - - -class SetupComplete(BaseModel): - """Indicates that session setup is complete.""" - - pass - - -class InlineData(BaseModel): - """Inline data embedded in server responses. - - Parameters: - mimeType: MIME type of the data. - data: Base64-encoded data content. - """ - - mimeType: str - data: str - - -class Part(BaseModel): - """Part of a server response containing data or text. - - Parameters: - inlineData: Inline binary data. Defaults to None. - text: Text content. Defaults to None. - """ - - inlineData: Optional[InlineData] = None - text: Optional[str] = None - - -class ModelTurn(BaseModel): - """Represents a turn from the model in the conversation. - - Parameters: - parts: List of content parts in the model's response. - """ - - parts: List[Part] - - -class ServerContentInterrupted(BaseModel): - """Indicates server content was interrupted. - - Parameters: - interrupted: Whether the content was interrupted. - """ - - interrupted: bool - - -class ServerContentTurnComplete(BaseModel): - """Indicates the server's turn is complete. - - Parameters: - turnComplete: Whether the turn is complete. - """ - - turnComplete: bool - - -class BidiGenerateContentTranscription(BaseModel): - """Transcription data from bidirectional content generation. - - Parameters: - text: The transcribed text content. - """ - - text: str - - -class ServerContent(BaseModel): - """Content sent from server to client. - - Parameters: - modelTurn: Model's conversational turn. Defaults to None. - interrupted: Whether content was interrupted. Defaults to None. - turnComplete: Whether the turn is complete. Defaults to None. - inputTranscription: Transcription of input audio. Defaults to None. - outputTranscription: Transcription of output audio. Defaults to None. - """ - - modelTurn: Optional[ModelTurn] = None - interrupted: Optional[bool] = None - turnComplete: Optional[bool] = None - inputTranscription: Optional[BidiGenerateContentTranscription] = None - outputTranscription: Optional[BidiGenerateContentTranscription] = None - groundingMetadata: Optional[GroundingMetadata] = None - - -class FunctionCall(BaseModel): - """Represents a function call from the model. - - Parameters: - id: Unique identifier for the function call. - name: Name of the function to call. - args: Arguments to pass to the function. - """ - - id: str - name: str - args: dict - - -class ToolCall(BaseModel): - """Contains one or more function calls. - - Parameters: - functionCalls: List of function calls to execute. - """ - - functionCalls: List[FunctionCall] - - -class Modality(str, Enum): - """Modality types in token counts.""" - - UNSPECIFIED = "MODALITY_UNSPECIFIED" - TEXT = "TEXT" - IMAGE = "IMAGE" - AUDIO = "AUDIO" - VIDEO = "VIDEO" - - -class ModalityTokenCount(BaseModel): - """Token count for a specific modality. - - Parameters: - modality: The modality type. - tokenCount: Number of tokens for this modality. - """ - - modality: Modality - tokenCount: int - - -class UsageMetadata(BaseModel): - """Usage metadata about the API response. - - Parameters: - promptTokenCount: Number of tokens in the prompt. Defaults to None. - cachedContentTokenCount: Number of cached content tokens. Defaults to None. - responseTokenCount: Number of tokens in the response. Defaults to None. - toolUsePromptTokenCount: Number of tokens for tool use prompts. Defaults to None. - thoughtsTokenCount: Number of tokens for model thoughts. Defaults to None. - totalTokenCount: Total number of tokens used. Defaults to None. - promptTokensDetails: Detailed breakdown of prompt tokens by modality. Defaults to None. - cacheTokensDetails: Detailed breakdown of cache tokens by modality. Defaults to None. - responseTokensDetails: Detailed breakdown of response tokens by modality. Defaults to None. - toolUsePromptTokensDetails: Detailed breakdown of tool use tokens by modality. Defaults to None. - """ - - promptTokenCount: Optional[int] = None - cachedContentTokenCount: Optional[int] = None - responseTokenCount: Optional[int] = None - toolUsePromptTokenCount: Optional[int] = None - thoughtsTokenCount: Optional[int] = None - totalTokenCount: Optional[int] = None - promptTokensDetails: Optional[List[ModalityTokenCount]] = None - cacheTokensDetails: Optional[List[ModalityTokenCount]] = None - responseTokensDetails: Optional[List[ModalityTokenCount]] = None - toolUsePromptTokensDetails: Optional[List[ModalityTokenCount]] = None - - -class ServerEvent(BaseModel): - """Server event received from the Gemini Live API. - - Parameters: - setupComplete: Setup completion notification. Defaults to None. - serverContent: Content from the server. Defaults to None. - toolCall: Tool/function call request. Defaults to None. - usageMetadata: Token usage metadata. Defaults to None. - """ - - setupComplete: Optional[SetupComplete] = None - serverContent: Optional[ServerContent] = None - toolCall: Optional[ToolCall] = None - usageMetadata: Optional[UsageMetadata] = None - - -def parse_server_event(str): - """Parse a server event from JSON string. - - Args: - str: JSON string containing the server event. - - Returns: - ServerEvent instance if parsing succeeds, None otherwise. - """ - try: - evt = json.loads(str) - return ServerEvent.model_validate(evt) - except Exception as e: - print(f"Error parsing server event: {e}") - return None - - -class ContextWindowCompressionConfig(BaseModel): - """Configuration for context window compression. - - Parameters: - sliding_window: Whether to use sliding window compression. Defaults to True. - trigger_tokens: Token count threshold to trigger compression. Defaults to None. - """ - - sliding_window: Optional[bool] = Field(default=True) - trigger_tokens: Optional[int] = Field(default=None) +"""Event models and utilities for Google Gemini Multimodal Live API. + +.. deprecated:: 0.0.90 + Importing StartSensitivity and EndSensitivity from this module is deprecated. + Import them directly from google.genai.types instead. +""" + +import warnings + +from loguru import logger + +try: + from google.genai.types import ( + EndSensitivity as _EndSensitivity, + ) + from google.genai.types import ( + StartSensitivity as _StartSensitivity, + ) +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") + raise Exception(f"Missing module: {e}") + +# These aliases are just here for backward compatibility, since we used to +# define public-facing StartSensitivity and EndSensitivity enums in this +# module. +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Importing StartSensitivity and EndSensitivity from " + "pipecat.services.gemini_multimodal_live.events is deprecated. " + "Please import them directly from google.genai.types instead.", + DeprecationWarning, + stacklevel=2, + ) + +StartSensitivity = _StartSensitivity +EndSensitivity = _EndSensitivity diff --git a/src/pipecat/services/gemini_multimodal_live/file_api.py b/src/pipecat/services/gemini_multimodal_live/file_api.py index 5ae7fdbb7..367d6797a 100644 --- a/src/pipecat/services/gemini_multimodal_live/file_api.py +++ b/src/pipecat/services/gemini_multimodal_live/file_api.py @@ -9,181 +9,31 @@ This module provides a client for Google's Gemini File API, enabling file uploads, metadata retrieval, listing, and deletion. Files uploaded through this API can be referenced in Gemini generative model calls. + +.. deprecated:: 0.0.90 + Importing GeminiFileAPI from this module is deprecated. + Import it from pipecat.services.google.gemini_live.file_api instead. """ -import mimetypes -from typing import Any, Dict, Optional +import warnings -import aiohttp from loguru import logger +try: + from pipecat.services.google.gemini_live.file_api import GeminiFileAPI as _GeminiFileAPI +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") + raise Exception(f"Missing module: {e}") -class GeminiFileAPI: - """Client for the Gemini File API. - - This class provides methods for uploading, fetching, listing, and deleting files - through Google's Gemini File API. - - Files uploaded through this API remain available for 48 hours and can be referenced - in calls to the Gemini generative models. Maximum file size is 2GB, with total - project storage limited to 20GB. - """ - - def __init__( - self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files" - ): - """Initialize the Gemini File API client. - - Args: - api_key: Google AI API key - base_url: Base URL for the Gemini File API (default is the v1beta endpoint) - """ - self._api_key = api_key - self._base_url = base_url - # Upload URL uses the /upload/ path - self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files" - - async def upload_file( - self, file_path: str, display_name: Optional[str] = None - ) -> Dict[str, Any]: - """Upload a file to the Gemini File API using the correct resumable upload protocol. - - Args: - file_path: Path to the file to upload - display_name: Optional display name for the file - - Returns: - File metadata including uri, name, and display_name - """ - logger.info(f"Uploading file: {file_path}") - - async with aiohttp.ClientSession() as session: - # Determine the file's MIME type - mime_type, _ = mimetypes.guess_type(file_path) - if not mime_type: - mime_type = "application/octet-stream" - - # Read the file - with open(file_path, "rb") as f: - file_data = f.read() - - # Create the metadata payload - metadata = {} - if display_name: - metadata = {"file": {"display_name": display_name}} - - # Step 1: Initial resumable request to get upload URL - headers = { - "X-Goog-Upload-Protocol": "resumable", - "X-Goog-Upload-Command": "start", - "X-Goog-Upload-Header-Content-Length": str(len(file_data)), - "X-Goog-Upload-Header-Content-Type": mime_type, - "Content-Type": "application/json", - } - - logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}") - async with session.post( - f"{self.upload_base_url}?key={self._api_key}", headers=headers, json=metadata - ) as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error initiating file upload: {error_text}") - raise Exception(f"Failed to initiate upload: {response.status} - {error_text}") - - # Get the upload URL from the response header - upload_url = response.headers.get("X-Goog-Upload-URL") - if not upload_url: - logger.error(f"Response headers: {dict(response.headers)}") - raise Exception("No upload URL in response headers") - - logger.debug(f"Got upload URL: {upload_url}") - - # Step 2: Upload the actual file data - upload_headers = { - "Content-Length": str(len(file_data)), - "X-Goog-Upload-Offset": "0", - "X-Goog-Upload-Command": "upload, finalize", - } - - logger.debug(f"Step 2: Uploading file data to {upload_url}") - async with session.post(upload_url, headers=upload_headers, data=file_data) as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error uploading file data: {error_text}") - raise Exception(f"Failed to upload file: {response.status} - {error_text}") - - file_info = await response.json() - logger.info(f"File uploaded successfully: {file_info.get('file', {}).get('name')}") - return file_info - - async def get_file(self, name: str) -> Dict[str, Any]: - """Get metadata for a file. - - Args: - name: File name (or full path) - - Returns: - File metadata - """ - # Extract just the name part if a full path is provided - if "/" in name: - name = name.split("/")[-1] - - async with aiohttp.ClientSession() as session: - async with session.get(f"{self._base_url}/{name}?key={self._api_key}") as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error getting file metadata: {error_text}") - raise Exception(f"Failed to get file metadata: {response.status}") - - file_info = await response.json() - return file_info - - async def list_files( - self, page_size: int = 10, page_token: Optional[str] = None - ) -> Dict[str, Any]: - """List uploaded files. - - Args: - page_size: Number of files to return per page - page_token: Token for pagination - - Returns: - List of files and next page token if available - """ - params = {"key": self._api_key, "pageSize": page_size} - - if page_token: - params["pageToken"] = page_token - - async with aiohttp.ClientSession() as session: - async with session.get(self._base_url, params=params) as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error listing files: {error_text}") - raise Exception(f"Failed to list files: {response.status}") - - result = await response.json() - return result - - async def delete_file(self, name: str) -> bool: - """Delete a file. - - Args: - name: File name (or full path) - - Returns: - True if deleted successfully - """ - # Extract just the name part if a full path is provided - if "/" in name: - name = name.split("/")[-1] - - async with aiohttp.ClientSession() as session: - async with session.delete(f"{self._base_url}/{name}?key={self._api_key}") as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error deleting file: {error_text}") - raise Exception(f"Failed to delete file: {response.status}") - - return True +# These aliases are just here for backward compatibility, since we used to +# define public-facing StartSensitivity and EndSensitivity enums in this +# module. +warnings.warn( + "Importing GeminiFileAPI from " + "pipecat.services.gemini_multimodal_live.file_api is deprecated. " + "Please import it from pipecat.services.google.gemini_live.file_api instead.", + DeprecationWarning, + stacklevel=2, +) +GeminiFileAPI = _GeminiFileAPI diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index df560358f..e56e34e9d 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -4,1416 +4,54 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Google Gemini Multimodal Live API service implementation. +"""Google Gemini Live API service implementation. This module provides real-time conversational AI capabilities using Google's -Gemini Multimodal Live API, supporting both text and audio modalities with +Gemini Live API, supporting both text and audio modalities with voice transcription, streaming responses, and tool usage. + +.. deprecated:: 0.0.90 + This module is deprecated. Please use the equivalent types from + pipecat.services.google.gemini_live.llm instead. Note that the new type names + do not include 'Multimodal'. """ -import base64 -import json -import time -from dataclasses import dataclass -from enum import Enum -from typing import Any, Dict, List, Optional, Union +import warnings -from loguru import logger -from pydantic import BaseModel, Field - -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter -from pipecat.frames.frames import ( - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - CancelFrame, - EndFrame, - ErrorFrame, - Frame, - InputAudioRawFrame, - InputImageRawFrame, - InputTextRawFrame, - InterruptionFrame, - LLMContextFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMMessagesAppendFrame, - LLMSetToolsFrame, - LLMTextFrame, - LLMUpdateSettingsFrame, - StartFrame, - TranscriptionFrame, - TTSAudioRawFrame, - TTSStartedFrame, - TTSStoppedFrame, - TTSTextFrame, - UserImageRawFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, +from pipecat.services.google.gemini_live.llm import ( + ContextWindowCompressionParams as _ContextWindowCompressionParams, ) -from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantAggregatorParams, - LLMUserAggregatorParams, +from pipecat.services.google.gemini_live.llm import ( + GeminiLiveAssistantContextAggregator, + GeminiLiveContext, + GeminiLiveContextAggregatorPair, + GeminiLiveLLMService, + GeminiLiveUserContextAggregator, + GeminiModalities, ) -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult -from pipecat.services.llm_service import FunctionCallFromLLM, LLMService -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) -from pipecat.transcriptions.language import Language -from pipecat.utils.string import match_endofsentence -from pipecat.utils.time import time_now_iso8601 -from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt - -from . import events -from .file_api import GeminiFileAPI - -try: - from websockets.asyncio.client import connect as websocket_connect -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") - raise Exception(f"Missing module: {e}") - - -def language_to_gemini_language(language: Language) -> Optional[str]: - """Maps a Language enum value to a Gemini Live supported language code. - - Source: - https://ai.google.dev/api/generate-content#MediaResolution - - Args: - language: The language enum value to convert. - - Returns: - The Gemini language code string, or None if the language is not supported. - """ - language_map = { - # Arabic - Language.AR: "ar-XA", - # Bengali - Language.BN_IN: "bn-IN", - # Chinese (Mandarin) - Language.CMN: "cmn-CN", - Language.CMN_CN: "cmn-CN", - Language.ZH: "cmn-CN", # Map general Chinese to Mandarin for Gemini - Language.ZH_CN: "cmn-CN", # Map Simplified Chinese to Mandarin for Gemini - # German - Language.DE: "de-DE", - Language.DE_DE: "de-DE", - # English - Language.EN: "en-US", # Default to US English (though not explicitly listed in supported codes) - Language.EN_US: "en-US", - Language.EN_AU: "en-AU", - Language.EN_GB: "en-GB", - Language.EN_IN: "en-IN", - # Spanish - Language.ES: "es-ES", # Default to Spain Spanish - Language.ES_ES: "es-ES", - Language.ES_US: "es-US", - # French - Language.FR: "fr-FR", # Default to France French - Language.FR_FR: "fr-FR", - Language.FR_CA: "fr-CA", - # Gujarati - Language.GU: "gu-IN", - Language.GU_IN: "gu-IN", - # Hindi - Language.HI: "hi-IN", - Language.HI_IN: "hi-IN", - # Indonesian - Language.ID: "id-ID", - Language.ID_ID: "id-ID", - # Italian - Language.IT: "it-IT", - Language.IT_IT: "it-IT", - # Japanese - Language.JA: "ja-JP", - Language.JA_JP: "ja-JP", - # Kannada - Language.KN: "kn-IN", - Language.KN_IN: "kn-IN", - # Korean - Language.KO: "ko-KR", - Language.KO_KR: "ko-KR", - # Malayalam - Language.ML: "ml-IN", - Language.ML_IN: "ml-IN", - # Marathi - Language.MR: "mr-IN", - Language.MR_IN: "mr-IN", - # Dutch - Language.NL: "nl-NL", - Language.NL_NL: "nl-NL", - # Polish - Language.PL: "pl-PL", - Language.PL_PL: "pl-PL", - # Portuguese (Brazil) - Language.PT_BR: "pt-BR", - # Russian - Language.RU: "ru-RU", - Language.RU_RU: "ru-RU", - # Tamil - Language.TA: "ta-IN", - Language.TA_IN: "ta-IN", - # Telugu - Language.TE: "te-IN", - Language.TE_IN: "te-IN", - # Thai - Language.TH: "th-TH", - Language.TH_TH: "th-TH", - # Turkish - Language.TR: "tr-TR", - Language.TR_TR: "tr-TR", - # Vietnamese - Language.VI: "vi-VN", - Language.VI_VN: "vi-VN", - } - return language_map.get(language) - - -class GeminiMultimodalLiveContext(OpenAILLMContext): - """Extended OpenAI context for Gemini Multimodal Live API. - - Provides Gemini-specific context management including system instruction - extraction and message format conversion for the Live API. - """ - - @staticmethod - def upgrade(obj: OpenAILLMContext) -> "GeminiMultimodalLiveContext": - """Upgrade an OpenAI context to Gemini context. - - Args: - obj: The OpenAI context to upgrade. - - Returns: - The upgraded Gemini context instance. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiMultimodalLiveContext): - logger.debug(f"Upgrading to Gemini Multimodal Live Context: {obj}") - obj.__class__ = GeminiMultimodalLiveContext - obj._restructure_from_openai_messages() - return obj - - def _restructure_from_openai_messages(self): - pass - - def extract_system_instructions(self): - """Extract system instructions from context messages. - - Returns: - Combined system instruction text from all system messages. - """ - system_instruction = "" - for item in self.messages: - if item.get("role") == "system": - content = item.get("content", "") - if content: - if system_instruction and not system_instruction.endswith("\n"): - system_instruction += "\n" - system_instruction += str(content) - return system_instruction - - def add_file_reference(self, file_uri: str, mime_type: str, text: Optional[str] = None): - """Add a file reference to the context. - - This adds a user message with a file reference that will be sent during context initialization. - - Args: - file_uri: URI of the uploaded file - mime_type: MIME type of the file - text: Optional text prompt to accompany the file - """ - # Create parts list with file reference - parts = [] - if text: - parts.append({"type": "text", "text": text}) - - # Add file reference part - parts.append( - {"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}} - ) - - # Add to messages - message = {"role": "user", "content": parts} - self.messages.append(message) - logger.info(f"Added file reference to context: {file_uri}") - - def get_messages_for_initializing_history(self): - """Get messages formatted for Gemini history initialization. - - Returns: - List of messages in Gemini format for conversation history. - """ - messages = [] - for item in self.messages: - role = item.get("role") - - if role == "system": - continue - - elif role == "assistant": - role = "model" - - content = item.get("content") - parts = [] - if isinstance(content, str): - parts = [{"text": content}] - elif isinstance(content, list): - for part in content: - if part.get("type") == "text": - parts.append({"text": part.get("text")}) - elif part.get("type") == "file_data": - file_data = part.get("file_data", {}) - - parts.append( - { - "fileData": { - "mimeType": file_data.get("mime_type"), - "fileUri": file_data.get("file_uri"), - } - } - ) - else: - logger.warning(f"Unsupported content type: {str(part)[:80]}") - else: - logger.warning(f"Unsupported content type: {str(content)[:80]}") - messages.append({"role": role, "parts": parts}) - return messages - - -class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): - """User context aggregator for Gemini Multimodal Live. - - Extends OpenAI user aggregator to handle Gemini-specific message passing - while maintaining compatibility with the standard aggregation pipeline. - """ - - async def process_frame(self, frame, direction): - """Process incoming frames for user context aggregation. - - Args: - frame: The frame to process. - direction: The frame processing direction. - """ - await super().process_frame(frame, direction) - # kind of a hack just to pass the LLMMessagesAppendFrame through, but it's fine for now - if isinstance(frame, LLMMessagesAppendFrame): - await self.push_frame(frame, direction) - - -class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Assistant context aggregator for Gemini Multimodal Live. - - Handles assistant response aggregation while filtering out LLMTextFrames - to prevent duplicate context entries, as Gemini Live pushes both - LLMTextFrames and TTSTextFrames. - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames for assistant context aggregation. - - Args: - frame: The frame to process. - direction: The frame processing direction. - """ - # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, - # but the GeminiMultimodalLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We - # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames - # are process. This ensures that the context gets only one set of messages. - if not isinstance(frame, LLMTextFrame): - await super().process_frame(frame, direction) - - async def handle_user_image_frame(self, frame: UserImageRawFrame): - """Handle user image frames. - - Args: - frame: The user image frame to handle. - """ - # We don't want to store any images in the context. Revisit this later - # when the API evolves. - pass - - -@dataclass -class GeminiMultimodalLiveContextAggregatorPair: - """Pair of user and assistant context aggregators for Gemini Multimodal Live. - - Parameters: - _user: The user context aggregator instance. - _assistant: The assistant context aggregator instance. - """ - - _user: GeminiMultimodalLiveUserContextAggregator - _assistant: GeminiMultimodalLiveAssistantContextAggregator - - def user(self) -> GeminiMultimodalLiveUserContextAggregator: - """Get the user context aggregator. - - Returns: - The user context aggregator instance. - """ - return self._user - - def assistant(self) -> GeminiMultimodalLiveAssistantContextAggregator: - """Get the assistant context aggregator. - - Returns: - The assistant context aggregator instance. - """ - return self._assistant - - -class GeminiMultimodalModalities(Enum): - """Supported modalities for Gemini Multimodal Live. - - Parameters: - TEXT: Text responses. - AUDIO: Audio responses. - """ - - TEXT = "TEXT" - AUDIO = "AUDIO" - - -class GeminiMediaResolution(str, Enum): - """Media resolution options for Gemini Multimodal Live. - - Parameters: - UNSPECIFIED: Use default resolution setting. - LOW: Low resolution with 64 tokens. - MEDIUM: Medium resolution with 256 tokens. - HIGH: High resolution with zoomed reframing and 256 tokens. - """ - - UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default - LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens - MEDIUM = "MEDIA_RESOLUTION_MEDIUM" # 256 tokens - HIGH = "MEDIA_RESOLUTION_HIGH" # Zoomed reframing with 256 tokens - - -class GeminiVADParams(BaseModel): - """Voice Activity Detection parameters for Gemini Live. - - Parameters: - disabled: Whether to disable VAD. Defaults to None. - start_sensitivity: Sensitivity for speech start detection. Defaults to None. - end_sensitivity: Sensitivity for speech end detection. Defaults to None. - prefix_padding_ms: Prefix padding in milliseconds. Defaults to None. - silence_duration_ms: Silence duration threshold in milliseconds. Defaults to None. - """ - - disabled: Optional[bool] = Field(default=None) - start_sensitivity: Optional[events.StartSensitivity] = Field(default=None) - end_sensitivity: Optional[events.EndSensitivity] = Field(default=None) - prefix_padding_ms: Optional[int] = Field(default=None) - silence_duration_ms: Optional[int] = Field(default=None) - - -class ContextWindowCompressionParams(BaseModel): - """Parameters for context window compression in Gemini Live. - - Parameters: - enabled: Whether compression is enabled. Defaults to False. - trigger_tokens: Token count to trigger compression. None uses 80% of context window. - """ - - enabled: bool = Field(default=False) - trigger_tokens: Optional[int] = Field( - default=None - ) # None = use default (80% of context window) - - -class InputParams(BaseModel): - """Input parameters for Gemini Multimodal Live generation. - - Parameters: - frequency_penalty: Frequency penalty for generation (0.0-2.0). Defaults to None. - max_tokens: Maximum tokens to generate. Must be >= 1. Defaults to 4096. - presence_penalty: Presence penalty for generation (0.0-2.0). Defaults to None. - temperature: Sampling temperature (0.0-2.0). Defaults to None. - top_k: Top-k sampling parameter. Must be >= 0. Defaults to None. - top_p: Top-p sampling parameter (0.0-1.0). Defaults to None. - modalities: Response modalities. Defaults to AUDIO. - language: Language for generation. Defaults to EN_US. - media_resolution: Media resolution setting. Defaults to UNSPECIFIED. - vad: Voice activity detection parameters. Defaults to None. - context_window_compression: Context compression settings. Defaults to None. - extra: Additional parameters. Defaults to empty dict. - """ - - frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) - max_tokens: Optional[int] = Field(default=4096, ge=1) - presence_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) - temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) - top_k: Optional[int] = Field(default=None, ge=0) - top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) - modalities: Optional[GeminiMultimodalModalities] = Field( - default=GeminiMultimodalModalities.AUDIO +from pipecat.services.google.gemini_live.llm import GeminiMediaResolution as _GeminiMediaResolution +from pipecat.services.google.gemini_live.llm import GeminiVADParams as _GeminiVADParams +from pipecat.services.google.gemini_live.llm import InputParams as _InputParams + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.gemini_multimodal_live.gemini are deprecated. " + "Please use the equivalent types from " + "pipecat.services.google.gemini_live.llm instead. Note that the new type " + "names do not include 'Multimodal' " + "(e.g. `GeminiMultimodalLiveLLMService` is now `GeminiLiveLLMService`).", + DeprecationWarning, + stacklevel=2, ) - language: Optional[Language] = Field(default=Language.EN_US) - media_resolution: Optional[GeminiMediaResolution] = Field( - default=GeminiMediaResolution.UNSPECIFIED - ) - vad: Optional[GeminiVADParams] = Field(default=None) - context_window_compression: Optional[ContextWindowCompressionParams] = Field(default=None) - extra: Optional[Dict[str, Any]] = Field(default_factory=dict) - -class GeminiMultimodalLiveLLMService(LLMService): - """Provides access to Google's Gemini Multimodal Live API. - - This service enables real-time conversations with Gemini, supporting both - text and audio modalities. It handles voice transcription, streaming audio - responses, and tool usage. - """ - - # Overriding the default adapter to use the Gemini one. - adapter_class = GeminiLLMAdapter - - def __init__( - self, - *, - api_key: str, - base_url: str = "generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", - model="models/gemini-2.0-flash-live-001", - voice_id: str = "Charon", - start_audio_paused: bool = False, - start_video_paused: bool = False, - system_instruction: Optional[str] = None, - tools: Optional[Union[List[dict], ToolsSchema]] = None, - params: Optional[InputParams] = None, - inference_on_context_initialization: bool = True, - file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files", - **kwargs, - ): - """Initialize the Gemini Multimodal Live LLM service. - - Args: - api_key: Google AI API key for authentication. - base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint. - model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001". - voice_id: TTS voice identifier. Defaults to "Charon". - start_audio_paused: Whether to start with audio input paused. Defaults to False. - start_video_paused: Whether to start with video input paused. Defaults to False. - system_instruction: System prompt for the model. Defaults to None. - tools: Tools/functions available to the model. Defaults to None. - params: Configuration parameters for the model. Defaults to InputParams(). - inference_on_context_initialization: Whether to generate a response when context - is first set. Defaults to True. - file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint. - **kwargs: Additional arguments passed to parent LLMService. - """ - super().__init__(base_url=base_url, **kwargs) - - params = params or InputParams() - - self._last_sent_time = 0 - self._api_key = api_key - self._base_url = base_url - self.set_model_name(model) - self._voice_id = voice_id - self._language_code = params.language - - self._system_instruction = system_instruction - self._tools = tools - self._inference_on_context_initialization = inference_on_context_initialization - self._needs_turn_complete_message = False - - self._audio_input_paused = start_audio_paused - self._video_input_paused = start_video_paused - self._context = None - self._websocket = None - self._receive_task = None - - self._disconnecting = False - self._api_session_ready = False - self._run_llm_when_api_session_ready = False - - self._user_is_speaking = False - self._bot_is_speaking = False - self._user_audio_buffer = bytearray() - self._user_transcription_buffer = "" - self._last_transcription_sent = "" - self._bot_audio_buffer = bytearray() - self._bot_text_buffer = "" - self._llm_output_buffer = "" - - self._sample_rate = 24000 - - self._language = params.language - self._language_code = ( - language_to_gemini_language(params.language) if params.language else "en-US" - ) - self._vad_params = params.vad - - self._settings = { - "frequency_penalty": params.frequency_penalty, - "max_tokens": params.max_tokens, - "presence_penalty": params.presence_penalty, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "modalities": params.modalities, - "language": self._language_code, - "media_resolution": params.media_resolution, - "vad": params.vad, - "context_window_compression": params.context_window_compression.model_dump() - if params.context_window_compression - else {}, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } - - # Initialize the File API client - self.file_api = GeminiFileAPI(api_key=api_key, base_url=file_api_base_url) - - # Grounding metadata tracking - self._search_result_buffer = "" - self._accumulated_grounding_metadata = None - - def can_generate_metrics(self) -> bool: - """Check if the service can generate usage metrics. - - Returns: - True as Gemini Live supports token usage metrics. - """ - return True - - def needs_mcp_alternate_schema(self) -> bool: - """Check if this LLM service requires alternate MCP schema. - - Google/Gemini has stricter JSON schema validation and requires - certain properties to be removed or modified for compatibility. - - Returns: - True for Google/Gemini services. - """ - return True - - def set_audio_input_paused(self, paused: bool): - """Set the audio input pause state. - - Args: - paused: Whether to pause audio input. - """ - self._audio_input_paused = paused - - def set_video_input_paused(self, paused: bool): - """Set the video input pause state. - - Args: - paused: Whether to pause video input. - """ - self._video_input_paused = paused - - def set_model_modalities(self, modalities: GeminiMultimodalModalities): - """Set the model response modalities. - - Args: - modalities: The modalities to use for responses. - """ - self._settings["modalities"] = modalities - - def set_language(self, language: Language): - """Set the language for generation. - - Args: - language: The language to use for generation. - """ - self._language = language - self._language_code = language_to_gemini_language(language) or "en-US" - self._settings["language"] = self._language_code - logger.info(f"Set Gemini language to: {self._language_code}") - - async def set_context(self, context: OpenAILLMContext): - """Set the context explicitly from outside the pipeline. - - This is useful when initializing a conversation because in server-side VAD mode we might not have a - way to trigger the pipeline. This sends the history to the server. The `inference_on_context_initialization` - flag controls whether to set the turnComplete flag when we do this. Without that flag, the model will - not respond. This is often what we want when setting the context at the beginning of a conversation. - - Args: - context: The OpenAI LLM context to set. - """ - if self._context: - logger.error( - "Context already set. Can only set up Gemini Multimodal Live context once." - ) - return - self._context = GeminiMultimodalLiveContext.upgrade(context) - await self._create_initial_response() - - # - # standard AIService frame handling - # - - async def start(self, frame: StartFrame): - """Start the service and establish websocket connection. - - Args: - frame: The start frame. - """ - await super().start(frame) - await self._connect() - - async def stop(self, frame: EndFrame): - """Stop the service and close connections. - - Args: - frame: The end frame. - """ - await super().stop(frame) - await self._disconnect() - - async def cancel(self, frame: CancelFrame): - """Cancel the service and close connections. - - Args: - frame: The cancel frame. - """ - await super().cancel(frame) - await self._disconnect() - - # - # speech and interruption handling - # - - async def _handle_interruption(self): - self._bot_is_speaking = False - await self.push_frame(TTSStoppedFrame()) - await self.push_frame(LLMFullResponseEndFrame()) - - async def _handle_user_started_speaking(self, frame): - self._user_is_speaking = True - pass - - async def _handle_user_stopped_speaking(self, frame): - self._user_is_speaking = False - self._user_audio_buffer = bytearray() - await self.start_ttfb_metrics() - if self._needs_turn_complete_message: - self._needs_turn_complete_message = False - evt = events.ClientContentMessage.model_validate( - {"clientContent": {"turnComplete": True}} - ) - await self.send_client_event(evt) - - # - # frame processing - # - # StartFrame, StopFrame, CancelFrame implemented in base class - # - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames for the Gemini Live service. - - Args: - frame: The frame to process. - direction: The frame processing direction. - """ - await super().process_frame(frame, direction) - - if isinstance(frame, TranscriptionFrame): - await self.push_frame(frame, direction) - elif isinstance(frame, OpenAILLMContextFrame): - context: GeminiMultimodalLiveContext = GeminiMultimodalLiveContext.upgrade( - frame.context - ) - # For now, we'll only trigger inference here when either: - # 1. We have not seen a context frame before - # 2. The last message is a tool call result - if not self._context: - self._context = context - if frame.context.tools: - self._tools = frame.context.tools - await self._create_initial_response() - elif context.messages and context.messages[-1].get("role") == "tool": - # Support just one tool call per context frame for now - tool_result_message = context.messages[-1] - await self._tool_result(tool_result_message) - elif isinstance(frame, LLMContextFrame): - raise NotImplementedError( - "Universal LLMContext is not yet supported for Gemini Multimodal Live." - ) - elif isinstance(frame, InputTextRawFrame): - await self._send_user_text(frame.text) - await self.push_frame(frame, direction) - elif isinstance(frame, InputAudioRawFrame): - await self._send_user_audio(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, InputImageRawFrame): - await self._send_user_video(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, InterruptionFrame): - await self._handle_interruption() - await self.push_frame(frame, direction) - elif isinstance(frame, UserStartedSpeakingFrame): - await self._handle_user_started_speaking(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, UserStoppedSpeakingFrame): - await self._handle_user_stopped_speaking(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, BotStartedSpeakingFrame): - # Ignore this frame. Use the serverContent API message instead - await self.push_frame(frame, direction) - elif isinstance(frame, BotStoppedSpeakingFrame): - # ignore this frame. Use the serverContent.turnComplete API message - await self.push_frame(frame, direction) - elif isinstance(frame, LLMMessagesAppendFrame): - await self._create_single_response(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) - elif isinstance(frame, LLMSetToolsFrame): - await self._update_settings() - else: - await self.push_frame(frame, direction) - - # - # websocket communication - # - - async def send_client_event(self, event): - """Send a client event to the Gemini Live API. - - Args: - event: The event to send. - """ - await self._ws_send(event.model_dump(exclude_none=True)) - - async def _connect(self): - """Establish WebSocket connection to Gemini Live API.""" - if self._websocket: - # Here we assume that if we have a websocket, we are connected. We - # handle disconnections in the send/recv code paths. - return - - logger.info("Connecting to Gemini service") - try: - logger.info(f"Connecting to wss://{self._base_url}") - uri = f"wss://{self._base_url}?key={self._api_key}" - self._websocket = await websocket_connect(uri=uri) - self._receive_task = self.create_task(self._receive_task_handler()) - - # Create the basic configuration - config_data = { - "setup": { - "model": self._model_name, - "generation_config": { - "frequency_penalty": self._settings["frequency_penalty"], - "max_output_tokens": self._settings["max_tokens"], - "presence_penalty": self._settings["presence_penalty"], - "temperature": self._settings["temperature"], - "top_k": self._settings["top_k"], - "top_p": self._settings["top_p"], - "response_modalities": self._settings["modalities"].value, - "speech_config": { - "voice_config": { - "prebuilt_voice_config": {"voice_name": self._voice_id} - }, - "language_code": self._settings["language"], - }, - "media_resolution": self._settings["media_resolution"].value, - }, - "input_audio_transcription": {}, - "output_audio_transcription": {}, - } - } - - # Add context window compression if enabled - if self._settings.get("context_window_compression", {}).get("enabled", False): - compression_config = {} - # Add sliding window (always true if compression is enabled) - compression_config["sliding_window"] = {} - - # Add trigger_tokens if specified - trigger_tokens = self._settings.get("context_window_compression", {}).get( - "trigger_tokens" - ) - if trigger_tokens is not None: - compression_config["trigger_tokens"] = trigger_tokens - - config_data["setup"]["context_window_compression"] = compression_config - - # Add VAD configuration if provided - if self._settings.get("vad"): - vad_config = {} - vad_params = self._settings["vad"] - - # Only add parameters that are explicitly set - if vad_params.disabled is not None: - vad_config["disabled"] = vad_params.disabled - - if vad_params.start_sensitivity: - vad_config["start_of_speech_sensitivity"] = vad_params.start_sensitivity.value - - if vad_params.end_sensitivity: - vad_config["end_of_speech_sensitivity"] = vad_params.end_sensitivity.value - - if vad_params.prefix_padding_ms is not None: - vad_config["prefix_padding_ms"] = vad_params.prefix_padding_ms - - if vad_params.silence_duration_ms is not None: - vad_config["silence_duration_ms"] = vad_params.silence_duration_ms - - # Only add automatic_activity_detection if we have VAD settings - if vad_config: - realtime_config = {"automatic_activity_detection": vad_config} - - config_data["setup"]["realtime_input_config"] = realtime_config - - config = events.Config.model_validate(config_data) - - # Add system instruction if available - system_instruction = self._system_instruction or "" - if self._context and hasattr(self._context, "extract_system_instructions"): - system_instruction += "\n" + self._context.extract_system_instructions() - if system_instruction: - logger.debug(f"Setting system instruction: {system_instruction}") - config.setup.system_instruction = events.SystemInstruction( - parts=[events.ContentPart(text=system_instruction)] - ) - - # Add tools if available - if self._tools: - logger.debug(f"Gemini is configuring to use tools{self._tools}") - config.setup.tools = self.get_llm_adapter().from_standard_tools(self._tools) - - # Send the configuration - await self.send_client_event(config) - - except Exception as e: - logger.error(f"{self} initialization error: {e}") - self._websocket = None - - async def _disconnect(self): - """Disconnect from Gemini Live API and clean up resources.""" - logger.info("Disconnecting from Gemini service") - try: - self._disconnecting = True - self._api_session_ready = False - await self.stop_all_metrics() - if self._websocket: - await self._websocket.close() - self._websocket = None - if self._receive_task: - await self.cancel_task(self._receive_task, timeout=1.0) - self._receive_task = None - self._disconnecting = False - except Exception as e: - logger.error(f"{self} error disconnecting: {e}") - - async def _ws_send(self, message): - """Send a message to the WebSocket connection.""" - # logger.debug(f"Sending message to websocket: {message}") - try: - if self._websocket: - await self._websocket.send(json.dumps(message)) - except Exception as e: - if self._disconnecting: - return - logger.error(f"Error sending message to websocket: {e}") - # In server-to-server contexts, a WebSocket error should be quite rare. Given how hard - # it is to recover from a send-side error with proper state management, and that exponential - # backoff for retries can have cost/stability implications for a service cluster, let's just - # treat a send-side error as fatal. - await self.push_error(ErrorFrame(error=f"Error sending client event: {e}", fatal=True)) - - # - # inbound server event handling - # todo: docs link here - # - - async def _receive_task_handler(self): - """Handle incoming messages from the WebSocket connection.""" - async for message in self._websocket: - evt = events.parse_server_event(message) - # logger.debug(f"Received event: {message[:500]}") - # logger.debug(f"Received event: {evt}") - - if evt.setupComplete: - await self._handle_evt_setup_complete(evt) - elif evt.serverContent and evt.serverContent.modelTurn: - await self._handle_evt_model_turn(evt) - elif evt.serverContent and evt.serverContent.turnComplete and evt.usageMetadata: - await self._handle_evt_turn_complete(evt) - await self._handle_evt_usage_metadata(evt) - elif evt.serverContent and evt.serverContent.inputTranscription: - await self._handle_evt_input_transcription(evt) - elif evt.serverContent and evt.serverContent.outputTranscription: - await self._handle_evt_output_transcription(evt) - elif evt.serverContent and evt.serverContent.groundingMetadata: - await self._handle_evt_grounding_metadata(evt) - elif evt.toolCall: - await self._handle_evt_tool_call(evt) - elif False: # !!! todo: error events? - await self._handle_evt_error(evt) - # errors are fatal, so exit the receive loop - return - - # - # - # - - async def _send_user_audio(self, frame): - """Send user audio frame to Gemini Live API.""" - if self._audio_input_paused: - return - # Send all audio to Gemini - evt = events.AudioInputMessage.from_raw_audio(frame.audio, frame.sample_rate) - await self.send_client_event(evt) - # Manage a buffer of audio to use for transcription - audio = frame.audio - if self._user_is_speaking: - self._user_audio_buffer.extend(audio) - else: - # Keep 1/2 second of audio in the buffer even when not speaking. - self._user_audio_buffer.extend(audio) - length = int((frame.sample_rate * frame.num_channels * 2) * 0.5) - self._user_audio_buffer = self._user_audio_buffer[-length:] - - async def _send_user_text(self, text: str): - """Send user text via Gemini Live API's realtime input stream. - - This method sends text through the realtimeInput stream (via TextInputMessage) - rather than the clientContent stream. This ensures text input is synchronized - with audio and video inputs, preventing temporal misalignment that can occur - when different modalities are processed through separate API pathways. - - For realtimeInput, turn completion is automatically inferred by the API based - on user activity, so no explicit turnComplete signal is needed. - - Args: - text: The text to send as user input. - """ - evt = events.TextInputMessage.from_text(text) - await self.send_client_event(evt) - - async def _send_user_video(self, frame): - """Send user video frame to Gemini Live API.""" - if self._video_input_paused: - return - - now = time.time() - if now - self._last_sent_time < 1: - return # Ignore if less than 1 second has passed - - self._last_sent_time = now # Update last sent time - logger.debug(f"Sending video frame to Gemini: {frame}") - evt = events.VideoInputMessage.from_image_frame(frame) - await self.send_client_event(evt) - - async def _create_initial_response(self): - """Create initial response based on context history.""" - if not self._api_session_ready: - self._run_llm_when_api_session_ready = True - return - - messages = self._context.get_messages_for_initializing_history() - if not messages: - return - - logger.debug(f"Creating initial response: {messages}") - - await self.start_ttfb_metrics() - - evt = events.ClientContentMessage.model_validate( - { - "clientContent": { - "turns": messages, - "turnComplete": self._inference_on_context_initialization, - } - } - ) - await self.send_client_event(evt) - if not self._inference_on_context_initialization: - self._needs_turn_complete_message = True - - async def _create_single_response(self, messages_list): - """Create a single response from a list of messages.""" - # Refactor to combine this logic with same logic in GeminiMultimodalLiveContext - messages = [] - for item in messages_list: - role = item.get("role") - - if role == "system": - continue - - elif role == "assistant": - role = "model" - - content = item.get("content") - parts = [] - if isinstance(content, str): - parts = [{"text": content}] - elif isinstance(content, list): - for part in content: - if part.get("type") == "text": - parts.append({"text": part.get("text")}) - elif part.get("type") == "file_data": - file_data = part.get("file_data", {}) - - parts.append( - { - "fileData": { - "mimeType": file_data.get("mime_type"), - "fileUri": file_data.get("file_uri"), - } - } - ) - else: - logger.warning(f"Unsupported content type: {str(part)[:80]}") - else: - logger.warning(f"Unsupported content type: {str(content)[:80]}") - messages.append({"role": role, "parts": parts}) - if not messages: - return - logger.debug(f"Creating response: {messages}") - - await self.start_ttfb_metrics() - - evt = events.ClientContentMessage.model_validate( - { - "clientContent": { - "turns": messages, - "turnComplete": True, - } - } - ) - await self.send_client_event(evt) - - @traced_gemini_live(operation="llm_tool_result") - async def _tool_result(self, tool_result_message): - """Send tool result back to the API.""" - # For now we're shoving the name into the tool_call_id field, so this - # will work until we revisit that. - id = tool_result_message.get("tool_call_id") - name = tool_result_message.get("tool_call_name") - result = json.loads(tool_result_message.get("content") or "") - response_message = json.dumps( - { - "toolResponse": { - "functionResponses": [ - { - "id": id, - "name": name, - "response": { - "result": result, - }, - } - ], - } - } - ) - await self._websocket.send(response_message) - # await self._websocket.send(json.dumps({"clientContent": {"turnComplete": True}})) - - @traced_gemini_live(operation="llm_setup") - async def _handle_evt_setup_complete(self, evt): - """Handle the setup complete event.""" - # If this is our first context frame, run the LLM - self._api_session_ready = True - # Now that we've configured the session, we can run the LLM if we need to. - if self._run_llm_when_api_session_ready: - self._run_llm_when_api_session_ready = False - await self._create_initial_response() - - async def _handle_evt_model_turn(self, evt): - """Handle the model turn event.""" - part = evt.serverContent.modelTurn.parts[0] - if not part: - return - - await self.stop_ttfb_metrics() - - # part.text is added when `modalities` is set to TEXT; otherwise, it's None - text = part.text - if text: - if not self._bot_text_buffer: - await self.push_frame(LLMFullResponseStartFrame()) - - self._bot_text_buffer += text - self._search_result_buffer += text # Also accumulate for grounding - await self.push_frame(LLMTextFrame(text=text)) - - # Check for grounding metadata in server content - if evt.serverContent and evt.serverContent.groundingMetadata: - self._accumulated_grounding_metadata = evt.serverContent.groundingMetadata - - inline_data = part.inlineData - if not inline_data: - return - if inline_data.mimeType != f"audio/pcm;rate={self._sample_rate}": - logger.warning(f"Unrecognized server_content format {inline_data.mimeType}") - return - - audio = base64.b64decode(inline_data.data) - if not audio: - return - - if not self._bot_is_speaking: - self._bot_is_speaking = True - await self.push_frame(TTSStartedFrame()) - await self.push_frame(LLMFullResponseStartFrame()) - - self._bot_audio_buffer.extend(audio) - frame = TTSAudioRawFrame( - audio=audio, - sample_rate=self._sample_rate, - num_channels=1, - ) - await self.push_frame(frame) - - @traced_gemini_live(operation="llm_tool_call") - async def _handle_evt_tool_call(self, evt): - """Handle tool call events.""" - function_calls = evt.toolCall.functionCalls - if not function_calls: - return - if not self._context: - logger.error("Function calls are not supported without a context object.") - - function_calls_llm = [ - FunctionCallFromLLM( - context=self._context, - tool_call_id=f.id, - function_name=f.name, - arguments=f.args, - ) - for f in function_calls - ] - - await self.run_function_calls(function_calls_llm) - - @traced_gemini_live(operation="llm_response") - async def _handle_evt_turn_complete(self, evt): - """Handle the turn complete event.""" - self._bot_is_speaking = False - text = self._bot_text_buffer - - # Determine output and modality for tracing - if text: - # TEXT modality - output_text = text - output_modality = "TEXT" - else: - # AUDIO modality - output_text = self._llm_output_buffer - output_modality = "AUDIO" - - # Trace the complete LLM response (this will be handled by the decorator) - # The decorator will extract the output text and usage metadata from the event - - self._bot_text_buffer = "" - self._llm_output_buffer = "" - - # Process grounding metadata if we have accumulated any - if self._accumulated_grounding_metadata: - await self._process_grounding_metadata( - self._accumulated_grounding_metadata, self._search_result_buffer - ) - - # Reset grounding tracking for next response - self._search_result_buffer = "" - self._accumulated_grounding_metadata = None - - # Only push the TTSStoppedFrame if the bot is outputting audio - # when text is found, modalities is set to TEXT and no audio - # is produced. - if not text: - await self.push_frame(TTSStoppedFrame()) - - await self.push_frame(LLMFullResponseEndFrame()) - - @traced_stt - async def _handle_user_transcription( - self, transcript: str, is_final: bool, language: Optional[Language] = None - ): - """Handle a transcription result with tracing.""" - pass - - async def _handle_evt_input_transcription(self, evt): - """Handle the input transcription event. - - Gemini Live sends user transcriptions in either single words or multi-word - phrases. As a result, we have to aggregate the input transcription. This handler - aggregates into sentences, splitting on the end of sentence markers. - """ - if not evt.serverContent.inputTranscription: - return - - text = evt.serverContent.inputTranscription.text - - if not text: - return - - # Strip leading space from sentence starts if buffer is empty - if text.startswith(" ") and not self._user_transcription_buffer: - text = text.lstrip() - - # Accumulate text in the buffer - self._user_transcription_buffer += text - - # Check for complete sentences - while True: - eos_end_marker = match_endofsentence(self._user_transcription_buffer) - if not eos_end_marker: - break - - # Extract the complete sentence - complete_sentence = self._user_transcription_buffer[:eos_end_marker] - # Keep the remainder for the next chunk - self._user_transcription_buffer = self._user_transcription_buffer[eos_end_marker:] - - # Send a TranscriptionFrame with the complete sentence - logger.debug(f"[Transcription:user] [{complete_sentence}]") - await self._handle_user_transcription( - complete_sentence, True, self._settings["language"] - ) - await self.push_frame( - TranscriptionFrame( - text=complete_sentence, - user_id="", - timestamp=time_now_iso8601(), - result=evt, - ), - FrameDirection.UPSTREAM, - ) - - async def _handle_evt_output_transcription(self, evt): - """Handle the output transcription event.""" - if not evt.serverContent.outputTranscription: - return - - # This is the output transcription text when modalities is set to AUDIO. - # In this case, we push LLMTextFrame and TTSTextFrame to be handled by the - # downstream assistant context aggregator. - text = evt.serverContent.outputTranscription.text - - if not text: - return - - # Accumulate text for grounding as well - self._search_result_buffer += text - - # Check for grounding metadata in server content - if evt.serverContent and evt.serverContent.groundingMetadata: - self._accumulated_grounding_metadata = evt.serverContent.groundingMetadata - # Collect text for tracing - self._llm_output_buffer += text - - await self.push_frame(LLMTextFrame(text=text)) - await self.push_frame(TTSTextFrame(text=text)) - - async def _handle_evt_grounding_metadata(self, evt): - """Handle dedicated grounding metadata events.""" - if evt.serverContent and evt.serverContent.groundingMetadata: - grounding_metadata = evt.serverContent.groundingMetadata - # Process the grounding metadata immediately - await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer) - - async def _process_grounding_metadata( - self, grounding_metadata: events.GroundingMetadata, search_result: str = "" - ): - """Process grounding metadata and emit LLMSearchResponseFrame.""" - if not grounding_metadata: - return - - # Extract rendered content for search suggestions - rendered_content = None - if ( - grounding_metadata.searchEntryPoint - and grounding_metadata.searchEntryPoint.renderedContent - ): - rendered_content = grounding_metadata.searchEntryPoint.renderedContent - - # Convert grounding chunks and supports to LLMSearchOrigin format - origins = [] - - if grounding_metadata.groundingChunks and grounding_metadata.groundingSupports: - # Create a mapping of chunk indices to origins - chunk_to_origin = {} - - for index, chunk in enumerate(grounding_metadata.groundingChunks): - if chunk.web: - origin = LLMSearchOrigin( - site_uri=chunk.web.uri, site_title=chunk.web.title, results=[] - ) - chunk_to_origin[index] = origin - origins.append(origin) - - # Add grounding support results to the appropriate origins - for support in grounding_metadata.groundingSupports: - if support.segment and support.groundingChunkIndices: - text = support.segment.text or "" - confidence_scores = support.confidenceScores or [] - - # Add this result to all origins referenced by this support - for chunk_index in support.groundingChunkIndices: - if chunk_index in chunk_to_origin: - result = LLMSearchResult(text=text, confidence=confidence_scores) - chunk_to_origin[chunk_index].results.append(result) - - # Create and push the search response frame - search_frame = LLMSearchResponseFrame( - search_result=search_result, origins=origins, rendered_content=rendered_content - ) - - await self.push_frame(search_frame) - - async def _handle_evt_usage_metadata(self, evt): - """Handle the usage metadata event.""" - if not evt.usageMetadata: - return - - usage = evt.usageMetadata - - # Ensure we have valid integers for all token counts - prompt_tokens = usage.promptTokenCount or 0 - completion_tokens = usage.responseTokenCount or 0 - total_tokens = usage.totalTokenCount or (prompt_tokens + completion_tokens) - - tokens = LLMTokenUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) - - await self.start_llm_usage_metrics(tokens) - - def create_context_aggregator( - self, - context: OpenAILLMContext, - *, - user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), - assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), - ) -> GeminiMultimodalLiveContextAggregatorPair: - """Create an instance of GeminiMultimodalLiveContextAggregatorPair from an OpenAILLMContext. - - Constructor keyword arguments for both the user and assistant aggregators can be provided. - - Args: - context: The LLM context to use. - user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams(). - assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams(). - - Returns: - GeminiMultimodalLiveContextAggregatorPair: A pair of context - aggregators, one for the user and one for the assistant, - encapsulated in an GeminiMultimodalLiveContextAggregatorPair. - """ - context.set_llm_adapter(self.get_llm_adapter()) - - GeminiMultimodalLiveContext.upgrade(context) - user = GeminiMultimodalLiveUserContextAggregator(context, params=user_params) - - assistant_params.expect_stripped_words = False - assistant = GeminiMultimodalLiveAssistantContextAggregator(context, params=assistant_params) - return GeminiMultimodalLiveContextAggregatorPair(_user=user, _assistant=assistant) +GeminiMultimodalLiveContext = GeminiLiveContext +GeminiMultimodalLiveUserContextAggregator = GeminiLiveUserContextAggregator +GeminiMultimodalLiveAssistantContextAggregator = GeminiLiveAssistantContextAggregator +GeminiMultimodalLiveContextAggregatorPair = GeminiLiveContextAggregatorPair +GeminiMultimodalModalities = GeminiModalities +GeminiMediaResolution = _GeminiMediaResolution +GeminiVADParams = _GeminiVADParams +ContextWindowCompressionParams = _ContextWindowCompressionParams +InputParams = _InputParams +GeminiMultimodalLiveLLMService = GeminiLiveLLMService diff --git a/src/pipecat/services/google/__init__.py b/src/pipecat/services/google/__init__.py index ec187000f..08e295b9e 100644 --- a/src/pipecat/services/google/__init__.py +++ b/src/pipecat/services/google/__init__.py @@ -9,6 +9,7 @@ import sys from pipecat.services import DeprecatedModuleProxy from .frames import * +from .gemini_live import * from .image import * from .llm import * from .llm_openai import * diff --git a/src/pipecat/services/google/gemini_live/__init__.py b/src/pipecat/services/google/gemini_live/__init__.py new file mode 100644 index 000000000..142ca2a83 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/__init__.py @@ -0,0 +1,3 @@ +from .file_api import GeminiFileAPI +from .llm import GeminiLiveLLMService +from .llm_vertex import GeminiLiveVertexLLMService diff --git a/src/pipecat/services/google/gemini_live/file_api.py b/src/pipecat/services/google/gemini_live/file_api.py new file mode 100644 index 000000000..5ae7fdbb7 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/file_api.py @@ -0,0 +1,189 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Gemini File API client for uploading and managing files. + +This module provides a client for Google's Gemini File API, enabling file +uploads, metadata retrieval, listing, and deletion. Files uploaded through +this API can be referenced in Gemini generative model calls. +""" + +import mimetypes +from typing import Any, Dict, Optional + +import aiohttp +from loguru import logger + + +class GeminiFileAPI: + """Client for the Gemini File API. + + This class provides methods for uploading, fetching, listing, and deleting files + through Google's Gemini File API. + + Files uploaded through this API remain available for 48 hours and can be referenced + in calls to the Gemini generative models. Maximum file size is 2GB, with total + project storage limited to 20GB. + """ + + def __init__( + self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files" + ): + """Initialize the Gemini File API client. + + Args: + api_key: Google AI API key + base_url: Base URL for the Gemini File API (default is the v1beta endpoint) + """ + self._api_key = api_key + self._base_url = base_url + # Upload URL uses the /upload/ path + self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files" + + async def upload_file( + self, file_path: str, display_name: Optional[str] = None + ) -> Dict[str, Any]: + """Upload a file to the Gemini File API using the correct resumable upload protocol. + + Args: + file_path: Path to the file to upload + display_name: Optional display name for the file + + Returns: + File metadata including uri, name, and display_name + """ + logger.info(f"Uploading file: {file_path}") + + async with aiohttp.ClientSession() as session: + # Determine the file's MIME type + mime_type, _ = mimetypes.guess_type(file_path) + if not mime_type: + mime_type = "application/octet-stream" + + # Read the file + with open(file_path, "rb") as f: + file_data = f.read() + + # Create the metadata payload + metadata = {} + if display_name: + metadata = {"file": {"display_name": display_name}} + + # Step 1: Initial resumable request to get upload URL + headers = { + "X-Goog-Upload-Protocol": "resumable", + "X-Goog-Upload-Command": "start", + "X-Goog-Upload-Header-Content-Length": str(len(file_data)), + "X-Goog-Upload-Header-Content-Type": mime_type, + "Content-Type": "application/json", + } + + logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}") + async with session.post( + f"{self.upload_base_url}?key={self._api_key}", headers=headers, json=metadata + ) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error initiating file upload: {error_text}") + raise Exception(f"Failed to initiate upload: {response.status} - {error_text}") + + # Get the upload URL from the response header + upload_url = response.headers.get("X-Goog-Upload-URL") + if not upload_url: + logger.error(f"Response headers: {dict(response.headers)}") + raise Exception("No upload URL in response headers") + + logger.debug(f"Got upload URL: {upload_url}") + + # Step 2: Upload the actual file data + upload_headers = { + "Content-Length": str(len(file_data)), + "X-Goog-Upload-Offset": "0", + "X-Goog-Upload-Command": "upload, finalize", + } + + logger.debug(f"Step 2: Uploading file data to {upload_url}") + async with session.post(upload_url, headers=upload_headers, data=file_data) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error uploading file data: {error_text}") + raise Exception(f"Failed to upload file: {response.status} - {error_text}") + + file_info = await response.json() + logger.info(f"File uploaded successfully: {file_info.get('file', {}).get('name')}") + return file_info + + async def get_file(self, name: str) -> Dict[str, Any]: + """Get metadata for a file. + + Args: + name: File name (or full path) + + Returns: + File metadata + """ + # Extract just the name part if a full path is provided + if "/" in name: + name = name.split("/")[-1] + + async with aiohttp.ClientSession() as session: + async with session.get(f"{self._base_url}/{name}?key={self._api_key}") as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error getting file metadata: {error_text}") + raise Exception(f"Failed to get file metadata: {response.status}") + + file_info = await response.json() + return file_info + + async def list_files( + self, page_size: int = 10, page_token: Optional[str] = None + ) -> Dict[str, Any]: + """List uploaded files. + + Args: + page_size: Number of files to return per page + page_token: Token for pagination + + Returns: + List of files and next page token if available + """ + params = {"key": self._api_key, "pageSize": page_size} + + if page_token: + params["pageToken"] = page_token + + async with aiohttp.ClientSession() as session: + async with session.get(self._base_url, params=params) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error listing files: {error_text}") + raise Exception(f"Failed to list files: {response.status}") + + result = await response.json() + return result + + async def delete_file(self, name: str) -> bool: + """Delete a file. + + Args: + name: File name (or full path) + + Returns: + True if deleted successfully + """ + # Extract just the name part if a full path is provided + if "/" in name: + name = name.split("/")[-1] + + async with aiohttp.ClientSession() as session: + async with session.delete(f"{self._base_url}/{name}?key={self._api_key}") as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error deleting file: {error_text}") + raise Exception(f"Failed to delete file: {response.status}") + + return True diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py new file mode 100644 index 000000000..4b5f05209 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -0,0 +1,1582 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Google Gemini Live API service implementation. + +This module provides real-time conversational AI capabilities using Google's +Gemini Live API, supporting both text and audio modalities with +voice transcription, streaming responses, and tool usage. +""" + +import base64 +import io +import json +import random +import time +import uuid +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from loguru import logger +from PIL import Image +from pydantic import BaseModel, Field + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + InputAudioRawFrame, + InputImageRawFrame, + InputTextRawFrame, + InterruptionFrame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesAppendFrame, + LLMSetToolsFrame, + LLMTextFrame, + LLMUpdateSettingsFrame, + StartFrame, + TranscriptionFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, + TTSTextFrame, + UserImageRawFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, +) +from pipecat.metrics.metrics import LLMTokenUsage +from pipecat.processors.aggregators.llm_response import ( + LLMAssistantAggregatorParams, + LLMUserAggregatorParams, +) +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult +from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) +from pipecat.transcriptions.language import Language +from pipecat.utils.string import match_endofsentence +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt + +from .file_api import GeminiFileAPI + +try: + from google.genai import Client + from google.genai.live import AsyncSession + from google.genai.types import ( + AudioTranscriptionConfig, + AutomaticActivityDetection, + Blob, + Content, + ContextWindowCompressionConfig, + EndSensitivity, + FileData, + FunctionResponse, + GenerationConfig, + GroundingMetadata, + HttpOptions, + LiveConnectConfig, + LiveServerMessage, + MediaResolution, + Modality, + Part, + ProactivityConfig, + RealtimeInputConfig, + SessionResumptionConfig, + SlidingWindow, + SpeechConfig, + StartSensitivity, + ThinkingConfig, + VoiceConfig, + ) +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") + raise Exception(f"Missing module: {e}") + + +# Connection management constants +MAX_CONSECUTIVE_FAILURES = 3 +CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds + + +def language_to_gemini_language(language: Language) -> Optional[str]: + """Maps a Language enum value to a Gemini Live supported language code. + + Source: + https://ai.google.dev/api/generate-content#MediaResolution + + Args: + language: The language enum value to convert. + + Returns: + The Gemini language code string, or None if the language is not supported. + """ + language_map = { + # Arabic + Language.AR: "ar-XA", + # Bengali + Language.BN_IN: "bn-IN", + # Chinese (Mandarin) + Language.CMN: "cmn-CN", + Language.CMN_CN: "cmn-CN", + Language.ZH: "cmn-CN", # Map general Chinese to Mandarin for Gemini + Language.ZH_CN: "cmn-CN", # Map Simplified Chinese to Mandarin for Gemini + # German + Language.DE: "de-DE", + Language.DE_DE: "de-DE", + # English + Language.EN: "en-US", # Default to US English (though not explicitly listed in supported codes) + Language.EN_US: "en-US", + Language.EN_AU: "en-AU", + Language.EN_GB: "en-GB", + Language.EN_IN: "en-IN", + # Spanish + Language.ES: "es-ES", # Default to Spain Spanish + Language.ES_ES: "es-ES", + Language.ES_US: "es-US", + # French + Language.FR: "fr-FR", # Default to France French + Language.FR_FR: "fr-FR", + Language.FR_CA: "fr-CA", + # Gujarati + Language.GU: "gu-IN", + Language.GU_IN: "gu-IN", + # Hindi + Language.HI: "hi-IN", + Language.HI_IN: "hi-IN", + # Indonesian + Language.ID: "id-ID", + Language.ID_ID: "id-ID", + # Italian + Language.IT: "it-IT", + Language.IT_IT: "it-IT", + # Japanese + Language.JA: "ja-JP", + Language.JA_JP: "ja-JP", + # Kannada + Language.KN: "kn-IN", + Language.KN_IN: "kn-IN", + # Korean + Language.KO: "ko-KR", + Language.KO_KR: "ko-KR", + # Malayalam + Language.ML: "ml-IN", + Language.ML_IN: "ml-IN", + # Marathi + Language.MR: "mr-IN", + Language.MR_IN: "mr-IN", + # Dutch + Language.NL: "nl-NL", + Language.NL_NL: "nl-NL", + # Polish + Language.PL: "pl-PL", + Language.PL_PL: "pl-PL", + # Portuguese (Brazil) + Language.PT_BR: "pt-BR", + # Russian + Language.RU: "ru-RU", + Language.RU_RU: "ru-RU", + # Tamil + Language.TA: "ta-IN", + Language.TA_IN: "ta-IN", + # Telugu + Language.TE: "te-IN", + Language.TE_IN: "te-IN", + # Thai + Language.TH: "th-TH", + Language.TH_TH: "th-TH", + # Turkish + Language.TR: "tr-TR", + Language.TR_TR: "tr-TR", + # Vietnamese + Language.VI: "vi-VN", + Language.VI_VN: "vi-VN", + } + return language_map.get(language) + + +class GeminiLiveContext(OpenAILLMContext): + """Extended OpenAI context for Gemini Live API. + + Provides Gemini-specific context management including system instruction + extraction and message format conversion for the Live API. + """ + + @staticmethod + def upgrade(obj: OpenAILLMContext) -> "GeminiLiveContext": + """Upgrade an OpenAI context to Gemini context. + + Args: + obj: The OpenAI context to upgrade. + + Returns: + The upgraded Gemini context instance. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiLiveContext): + logger.debug(f"Upgrading to Gemini Live Context: {obj}") + obj.__class__ = GeminiLiveContext + obj._restructure_from_openai_messages() + return obj + + def _restructure_from_openai_messages(self): + pass + + def extract_system_instructions(self): + """Extract system instructions from context messages. + + Returns: + Combined system instruction text from all system messages. + """ + system_instruction = "" + for item in self.messages: + if item.get("role") == "system": + content = item.get("content", "") + if content: + if system_instruction and not system_instruction.endswith("\n"): + system_instruction += "\n" + system_instruction += str(content) + return system_instruction + + def add_file_reference(self, file_uri: str, mime_type: str, text: Optional[str] = None): + """Add a file reference to the context. + + This adds a user message with a file reference that will be sent during context initialization. + + Args: + file_uri: URI of the uploaded file + mime_type: MIME type of the file + text: Optional text prompt to accompany the file + """ + # Create parts list with file reference + parts = [] + if text: + parts.append({"type": "text", "text": text}) + + # Add file reference part + parts.append( + {"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}} + ) + + # Add to messages + message = {"role": "user", "content": parts} + self.messages.append(message) + logger.info(f"Added file reference to context: {file_uri}") + + def get_messages_for_initializing_history(self) -> List[Content]: + """Get messages formatted for Gemini history initialization. + + Returns: + List of messages in Gemini format for conversation history. + """ + messages: List[Content] = [] + for item in self.messages: + role = item.get("role") + + if role == "system": + continue + + elif role == "assistant": + role = "model" + + content = item.get("content") + parts: List[Part] = [] + if isinstance(content, str): + parts = [Part(text=content)] + elif isinstance(content, list): + for part in content: + if part.get("type") == "text": + parts.append(Part(text=part.get("text"))) + elif part.get("type") == "file_data": + file_data = part.get("file_data", {}) + parts.append( + Part( + file_data=FileData( + mime_type=file_data.get("mime_type"), + file_uri=file_data.get("file_uri"), + ) + ) + ) + else: + logger.warning(f"Unsupported content type: {str(part)[:80]}") + else: + logger.warning(f"Unsupported content type: {str(content)[:80]}") + messages.append(Content(role=role, parts=parts)) + return messages + + +class GeminiLiveUserContextAggregator(OpenAIUserContextAggregator): + """User context aggregator for Gemini Live. + + Extends OpenAI user aggregator to handle Gemini-specific message passing + while maintaining compatibility with the standard aggregation pipeline. + """ + + async def process_frame(self, frame, direction): + """Process incoming frames for user context aggregation. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ + await super().process_frame(frame, direction) + # kind of a hack just to pass the LLMMessagesAppendFrame through, but it's fine for now + if isinstance(frame, LLMMessagesAppendFrame): + await self.push_frame(frame, direction) + + +class GeminiLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Assistant context aggregator for Gemini Live. + + Handles assistant response aggregation while filtering out LLMTextFrames + to prevent duplicate context entries, as Gemini Live pushes both + LLMTextFrames and TTSTextFrames. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames for assistant context aggregation. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the GeminiLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. + if not isinstance(frame, LLMTextFrame): + await super().process_frame(frame, direction) + + async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle user image frames. + + Args: + frame: The user image frame to handle. + """ + # We don't want to store any images in the context. Revisit this later + # when the API evolves. + pass + + +@dataclass +class GeminiLiveContextAggregatorPair: + """Pair of user and assistant context aggregators for Gemini Live. + + Parameters: + _user: The user context aggregator instance. + _assistant: The assistant context aggregator instance. + """ + + _user: GeminiLiveUserContextAggregator + _assistant: GeminiLiveAssistantContextAggregator + + def user(self) -> GeminiLiveUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ + return self._user + + def assistant(self) -> GeminiLiveAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ + return self._assistant + + +class GeminiModalities(Enum): + """Supported modalities for Gemini Live. + + Parameters: + TEXT: Text responses. + AUDIO: Audio responses. + """ + + TEXT = "TEXT" + AUDIO = "AUDIO" + + +class GeminiMediaResolution(str, Enum): + """Media resolution options for Gemini Live. + + Parameters: + UNSPECIFIED: Use default resolution setting. + LOW: Low resolution with 64 tokens. + MEDIUM: Medium resolution with 256 tokens. + HIGH: High resolution with zoomed reframing and 256 tokens. + """ + + UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default + LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens + MEDIUM = "MEDIA_RESOLUTION_MEDIUM" # 256 tokens + HIGH = "MEDIA_RESOLUTION_HIGH" # Zoomed reframing with 256 tokens + + +class GeminiVADParams(BaseModel): + """Voice Activity Detection parameters for Gemini Live. + + Parameters: + disabled: Whether to disable VAD. Defaults to None. + start_sensitivity: Sensitivity for speech start detection. Defaults to None. + end_sensitivity: Sensitivity for speech end detection. Defaults to None. + prefix_padding_ms: Prefix padding in milliseconds. Defaults to None. + silence_duration_ms: Silence duration threshold in milliseconds. Defaults to None. + """ + + disabled: Optional[bool] = Field(default=None) + start_sensitivity: Optional[StartSensitivity] = Field(default=None) + end_sensitivity: Optional[EndSensitivity] = Field(default=None) + prefix_padding_ms: Optional[int] = Field(default=None) + silence_duration_ms: Optional[int] = Field(default=None) + + +class ContextWindowCompressionParams(BaseModel): + """Parameters for context window compression in Gemini Live. + + Parameters: + enabled: Whether compression is enabled. Defaults to False. + trigger_tokens: Token count to trigger compression. None uses 80% of context window. + """ + + enabled: bool = Field(default=False) + trigger_tokens: Optional[int] = Field( + default=None + ) # None = use default (80% of context window) + + +class InputParams(BaseModel): + """Input parameters for Gemini Live generation. + + Parameters: + frequency_penalty: Frequency penalty for generation (0.0-2.0). Defaults to None. + max_tokens: Maximum tokens to generate. Must be >= 1. Defaults to 4096. + presence_penalty: Presence penalty for generation (0.0-2.0). Defaults to None. + temperature: Sampling temperature (0.0-2.0). Defaults to None. + top_k: Top-k sampling parameter. Must be >= 0. Defaults to None. + top_p: Top-p sampling parameter (0.0-1.0). Defaults to None. + modalities: Response modalities. Defaults to AUDIO. + language: Language for generation. Defaults to EN_US. + media_resolution: Media resolution setting. Defaults to UNSPECIFIED. + vad: Voice activity detection parameters. Defaults to None. + context_window_compression: Context compression settings. Defaults to None. + thinking: Thinking settings. Defaults to None. + Note that these settings may require specifying a model that + supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". + enable_affective_dialog: Enable affective dialog, which allows Gemini + to adapt to expression and tone. Defaults to None. + Note that these settings may require specifying a model that + supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". + Also note that this setting may require specifying an API version that + supports it, e.g. HttpOptions(api_version="v1alpha"). + proactivity: Proactivity settings, which allows Gemini to proactively + decide how to behave, such as whether to avoid responding to + content that is not relevant. Defaults to None. + Note that these settings may require specifying a model that + supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". + Also note that this setting may require specifying an API version that + supports it, e.g. HttpOptions(api_version="v1alpha"). + extra: Additional parameters. Defaults to empty dict. + """ + + frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) + max_tokens: Optional[int] = Field(default=4096, ge=1) + presence_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) + temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) + top_k: Optional[int] = Field(default=None, ge=0) + top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) + modalities: Optional[GeminiModalities] = Field(default=GeminiModalities.AUDIO) + language: Optional[Language] = Field(default=Language.EN_US) + media_resolution: Optional[GeminiMediaResolution] = Field( + default=GeminiMediaResolution.UNSPECIFIED + ) + vad: Optional[GeminiVADParams] = Field(default=None) + context_window_compression: Optional[ContextWindowCompressionParams] = Field(default=None) + thinking: Optional[ThinkingConfig] = Field(default=None) + enable_affective_dialog: Optional[bool] = Field(default=None) + proactivity: Optional[ProactivityConfig] = Field(default=None) + extra: Optional[Dict[str, Any]] = Field(default_factory=dict) + + +class GeminiLiveLLMService(LLMService): + """Provides access to Google's Gemini Live API. + + This service enables real-time conversations with Gemini, supporting both + text and audio modalities. It handles voice transcription, streaming audio + responses, and tool usage. + """ + + # Overriding the default adapter to use the Gemini one. + adapter_class = GeminiLLMAdapter + + def __init__( + self, + *, + api_key: str, + base_url: Optional[str] = None, + model="models/gemini-2.0-flash-live-001", + voice_id: str = "Charon", + start_audio_paused: bool = False, + start_video_paused: bool = False, + system_instruction: Optional[str] = None, + tools: Optional[Union[List[dict], ToolsSchema]] = None, + params: Optional[InputParams] = None, + inference_on_context_initialization: bool = True, + file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files", + http_options: Optional[HttpOptions] = None, + **kwargs, + ): + """Initialize the Gemini Live LLM service. + + Args: + api_key: Google AI API key for authentication. + base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint. + + .. deprecated:: 0.0.90 + This parameter is deprecated and no longer has any effect. + Please use `http_options` to customize requests made by the + API client. + + model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001". + voice_id: TTS voice identifier. Defaults to "Charon". + start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_video_paused: Whether to start with video input paused. Defaults to False. + system_instruction: System prompt for the model. Defaults to None. + tools: Tools/functions available to the model. Defaults to None. + params: Configuration parameters for the model. Defaults to InputParams(). + inference_on_context_initialization: Whether to generate a response when context + is first set. Defaults to True. + file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint. + http_options: HTTP options for the client. + **kwargs: Additional arguments passed to parent LLMService. + """ + # Check for deprecated parameter usage + if base_url is not None: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'base_url' is deprecated and no longer has any effect. Please use 'http_options' to customize requests made by the API client.", + DeprecationWarning, + stacklevel=2, + ) + + super().__init__(base_url=base_url, **kwargs) + + params = params or InputParams() + + self._last_sent_time = 0 + self._base_url = base_url + self.set_model_name(model) + self._voice_id = voice_id + self._language_code = params.language + + self._system_instruction = system_instruction + self._tools = tools + self._inference_on_context_initialization = inference_on_context_initialization + self._needs_turn_complete_message = False + + self._audio_input_paused = start_audio_paused + self._video_input_paused = start_video_paused + self._context = None + self._api_key = api_key + self._http_options = http_options + self._session: AsyncSession = None + self._connection_task = None + + self._disconnecting = False + self._run_llm_when_session_ready = False + + self._user_is_speaking = False + self._bot_is_speaking = False + self._user_audio_buffer = bytearray() + self._user_transcription_buffer = "" + self._last_transcription_sent = "" + self._bot_audio_buffer = bytearray() + self._bot_text_buffer = "" + self._llm_output_buffer = "" + + self._sample_rate = 24000 + + self._language = params.language + self._language_code = ( + language_to_gemini_language(params.language) if params.language else "en-US" + ) + self._vad_params = params.vad + + # Reconnection tracking + self._consecutive_failures = 0 + self._connection_start_time = None + + self._settings = { + "frequency_penalty": params.frequency_penalty, + "max_tokens": params.max_tokens, + "presence_penalty": params.presence_penalty, + "temperature": params.temperature, + "top_k": params.top_k, + "top_p": params.top_p, + "modalities": params.modalities, + "language": self._language_code, + "media_resolution": params.media_resolution, + "vad": params.vad, + "context_window_compression": params.context_window_compression.model_dump() + if params.context_window_compression + else {}, + "thinking": params.thinking or {}, + "enable_affective_dialog": params.enable_affective_dialog or False, + "proactivity": params.proactivity or {}, + "extra": params.extra if isinstance(params.extra, dict) else {}, + } + + self._file_api_base_url = file_api_base_url + self._file_api: Optional[GeminiFileAPI] = None + + # Grounding metadata tracking + self._search_result_buffer = "" + self._accumulated_grounding_metadata = None + + # Session resumption + self._session_resumption_handle: Optional[str] = None + + # Bookkeeping for ending gracefully (i.e. after the bot is finished) + self._end_frame_pending_bot_turn_finished: Optional[EndFrame] = None + + # Initialize the API client. Subclasses can override this if needed. + self.create_client() + + def create_client(self): + """Create the Gemini API client instance. Subclasses can override this.""" + self._client = Client(api_key=self._api_key, http_options=self._http_options) + + @property + def file_api(self) -> GeminiFileAPI: + """Get the Gemini File API client instance. Subclasses can override this. + + Returns: + The Gemini File API client. + """ + if not self._file_api: + self._file_api = GeminiFileAPI(api_key=self._api_key, base_url=self._file_api_base_url) + return self._file_api + + def can_generate_metrics(self) -> bool: + """Check if the service can generate usage metrics. + + Returns: + True as Gemini Live supports token usage metrics. + """ + return True + + def needs_mcp_alternate_schema(self) -> bool: + """Check if this LLM service requires alternate MCP schema. + + Google/Gemini has stricter JSON schema validation and requires + certain properties to be removed or modified for compatibility. + + Returns: + True for Google/Gemini services. + """ + return True + + def set_audio_input_paused(self, paused: bool): + """Set the audio input pause state. + + Args: + paused: Whether to pause audio input. + """ + self._audio_input_paused = paused + + def set_video_input_paused(self, paused: bool): + """Set the video input pause state. + + Args: + paused: Whether to pause video input. + """ + self._video_input_paused = paused + + def set_model_modalities(self, modalities: GeminiModalities): + """Set the model response modalities. + + Args: + modalities: The modalities to use for responses. + """ + self._settings["modalities"] = modalities + + def set_language(self, language: Language): + """Set the language for generation. + + Args: + language: The language to use for generation. + """ + self._language = language + self._language_code = language_to_gemini_language(language) or "en-US" + self._settings["language"] = self._language_code + logger.info(f"Set Gemini language to: {self._language_code}") + + async def set_context(self, context: OpenAILLMContext): + """Set the context explicitly from outside the pipeline. + + This is useful when initializing a conversation because in server-side VAD mode we might not have a + way to trigger the pipeline. This sends the history to the server. The `inference_on_context_initialization` + flag controls whether to set the turnComplete flag when we do this. Without that flag, the model will + not respond. This is often what we want when setting the context at the beginning of a conversation. + + Args: + context: The OpenAI LLM context to set. + """ + if self._context: + logger.error("Context already set. Can only set up Gemini Live context once.") + return + self._context = GeminiLiveContext.upgrade(context) + await self._create_initial_response() + + # + # standard AIService frame handling + # + + async def start(self, frame: StartFrame): + """Start the service and establish connection. + + Args: + frame: The start frame. + """ + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the service and close connections. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the service and close connections. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + # + # speech and interruption handling + # + + async def _handle_interruption(self): + await self._set_bot_is_speaking(False) + await self.push_frame(TTSStoppedFrame()) + await self.push_frame(LLMFullResponseEndFrame()) + + async def _handle_user_started_speaking(self, frame): + self._user_is_speaking = True + pass + + async def _handle_user_stopped_speaking(self, frame): + self._user_is_speaking = False + self._user_audio_buffer = bytearray() + await self.start_ttfb_metrics() + if self._needs_turn_complete_message: + self._needs_turn_complete_message = False + # NOTE: without this, the model ignores the context it's been + # seeded with before the user started speaking + await self._session.send_client_content(turn_complete=True) + + # + # frame processing + # + # StartFrame, StopFrame, CancelFrame implemented in base class + # + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames for the Gemini Live service. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ + # Defer EndFrame handling until after the bot turn is finished + if isinstance(frame, EndFrame): + if self._bot_is_speaking: + logger.debug("Deferring handling EndFrame until bot turn is finished") + self._end_frame_pending_bot_turn_finished = frame + return + + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + await self.push_frame(frame, direction) + elif isinstance(frame, OpenAILLMContextFrame): + context: GeminiLiveContext = GeminiLiveContext.upgrade(frame.context) + # For now, we'll only trigger inference here when either: + # 1. We have not seen a context frame before + # 2. The last message is a tool call result + if not self._context: + self._context = context + if frame.context.tools: + self._tools = frame.context.tools + await self._create_initial_response() + elif context.messages and context.messages[-1].get("role") == "tool": + # Support just one tool call per context frame for now + tool_result_message = context.messages[-1] + await self._tool_result(tool_result_message) + elif isinstance(frame, LLMContextFrame): + raise NotImplementedError("Universal LLMContext is not yet supported for Gemini Live.") + elif isinstance(frame, InputTextRawFrame): + await self._send_user_text(frame.text) + await self.push_frame(frame, direction) + elif isinstance(frame, InputAudioRawFrame): + await self._send_user_audio(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, InputImageRawFrame): + await self._send_user_video(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, InterruptionFrame): + await self._handle_interruption() + await self.push_frame(frame, direction) + elif isinstance(frame, UserStartedSpeakingFrame): + await self._handle_user_started_speaking(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, UserStoppedSpeakingFrame): + await self._handle_user_stopped_speaking(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, BotStartedSpeakingFrame): + # Ignore this frame. Use the serverContent API message instead + await self.push_frame(frame, direction) + elif isinstance(frame, BotStoppedSpeakingFrame): + # ignore this frame. Use the serverContent.turnComplete API message + await self.push_frame(frame, direction) + elif isinstance(frame, LLMMessagesAppendFrame): + # NOTE: handling LLMMessagesAppendFrame here in the LLMService is + # unusual - typically this would be handled in the user context + # aggregator. Leaving this handling here so that user code that + # uses this frame *without* a user context aggregator still works + # (we have an example that does just that, actually). + await self._create_single_response(frame.messages) + elif isinstance(frame, LLMUpdateSettingsFrame): + await self._update_settings(frame.settings) + elif isinstance(frame, LLMSetToolsFrame): + await self._update_settings() + else: + await self.push_frame(frame, direction) + + async def _set_bot_is_speaking(self, speaking: bool): + if self._bot_is_speaking == speaking: + return + + self._bot_is_speaking = speaking + + if not self._bot_is_speaking and self._end_frame_pending_bot_turn_finished: + await self.queue_frame(self._end_frame_pending_bot_turn_finished) + self._end_frame_pending_bot_turn_finished = None + + async def _connect(self, session_resumption_handle: Optional[str] = None): + """Establish client connection to Gemini Live API.""" + if self._session: + # Here we assume that if we have a client, we are connected. We + # handle disconnections in the send/recv code paths. + return + + if session_resumption_handle: + logger.info( + f"Connecting to Gemini service with session_resumption_handle: {session_resumption_handle}" + ) + else: + logger.info("Connecting to Gemini service") + try: + # Assemble basic configuration + config = LiveConnectConfig( + generation_config=GenerationConfig( + frequency_penalty=self._settings["frequency_penalty"], + max_output_tokens=self._settings["max_tokens"], + presence_penalty=self._settings["presence_penalty"], + temperature=self._settings["temperature"], + top_k=self._settings["top_k"], + top_p=self._settings["top_p"], + response_modalities=[Modality(self._settings["modalities"].value)], + speech_config=SpeechConfig( + voice_config=VoiceConfig( + prebuilt_voice_config={"voice_name": self._voice_id} + ), + language_code=self._settings["language"], + ), + media_resolution=MediaResolution(self._settings["media_resolution"].value), + ), + input_audio_transcription=AudioTranscriptionConfig(), + output_audio_transcription=AudioTranscriptionConfig(), + session_resumption=SessionResumptionConfig(handle=session_resumption_handle), + ) + + # Add context window compression to configuration, if enabled + if self._settings.get("context_window_compression", {}).get("enabled", False): + compression_config = ContextWindowCompressionConfig() + + # Add sliding window (always true if compression is enabled) + compression_config.sliding_window = SlidingWindow() + + # Add trigger_tokens if specified + trigger_tokens = self._settings.get("context_window_compression", {}).get( + "trigger_tokens" + ) + if trigger_tokens is not None: + compression_config.trigger_tokens = trigger_tokens + + config.context_window_compression = compression_config + + # Add thinking configuration to configuration, if provided + if self._settings.get("thinking"): + config.thinking_config = self._settings["thinking"] + + # Add affective dialog setting, if provided + if self._settings.get("enable_affective_dialog", False): + config.enable_affective_dialog = self._settings["enable_affective_dialog"] + + # Add proactivity configuration to configuration, if provided + if self._settings.get("proactivity"): + config.proactivity = self._settings["proactivity"] + + # Add VAD configuration to configuration, if provided + if self._settings.get("vad"): + vad_config = AutomaticActivityDetection() + vad_params = self._settings["vad"] + has_vad_settings = False + + # Only add parameters that are explicitly set + if vad_params.disabled is not None: + vad_config.disabled = vad_params.disabled + has_vad_settings = True + + if vad_params.start_sensitivity: + vad_config.start_of_speech_sensitivity = vad_params.start_sensitivity + has_vad_settings = True + + if vad_params.end_sensitivity: + vad_config.end_of_speech_sensitivity = vad_params.end_sensitivity + has_vad_settings = True + + if vad_params.prefix_padding_ms is not None: + vad_config.prefix_padding_ms = vad_params.prefix_padding_ms + has_vad_settings = True + + if vad_params.silence_duration_ms is not None: + vad_config.silence_duration_ms = vad_params.silence_duration_ms + has_vad_settings = True + + # Only add automatic_activity_detection if we have VAD settings + if has_vad_settings: + config.realtime_input_config = RealtimeInputConfig( + automatic_activity_detection=vad_config + ) + + # Add system instruction to configuration, if provided + system_instruction = self._system_instruction or "" + if self._context and hasattr(self._context, "extract_system_instructions"): + system_instruction += "\n" + self._context.extract_system_instructions() + if system_instruction: + logger.debug(f"Setting system instruction: {system_instruction}") + config.system_instruction = system_instruction + + # Add tools to configuration, if provided + if self._tools: + logger.debug(f"Setting tools: {self._tools}") + config.tools = self.get_llm_adapter().from_standard_tools(self._tools) + + # Start the connection + self._connection_task = self.create_task(self._connection_task_handler(config=config)) + + except Exception as e: + await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}", fatal=True)) + + async def _connection_task_handler(self, config: LiveConnectConfig): + async with self._client.aio.live.connect(model=self._model_name, config=config) as session: + logger.info("Connected to Gemini service") + + # Mark connection start time + self._connection_start_time = time.time() + + await self._handle_session_ready(session) + + while True: + try: + turn = self._session.receive() + async for message in turn: + # Reset failure counter if connection has been stable + self._check_and_reset_failure_counter() + + if message.server_content and message.server_content.model_turn: + await self._handle_msg_model_turn(message) + elif ( + message.server_content + and message.server_content.turn_complete + and message.usage_metadata + ): + await self._handle_msg_turn_complete(message) + await self._handle_msg_usage_metadata(message) + elif message.server_content and message.server_content.input_transcription: + await self._handle_msg_input_transcription(message) + elif message.server_content and message.server_content.output_transcription: + await self._handle_msg_output_transcription(message) + elif message.server_content and message.server_content.grounding_metadata: + await self._handle_msg_grounding_metadata(message) + elif message.tool_call: + await self._handle_msg_tool_call(message) + elif message.session_resumption_update: + self._handle_msg_resumption_update(message) + except Exception as e: + if not self._disconnecting: + should_reconnect = await self._handle_connection_error(e) + if should_reconnect: + await self._reconnect() + return # Exit this connection handler, _reconnect will start a new one + break + + def _check_and_reset_failure_counter(self): + """Check if connection has been stable long enough to reset the failure counter. + + If the connection has been active for longer than the established threshold + and there are accumulated failures, reset the counter to 0. + """ + if ( + self._connection_start_time + and self._consecutive_failures > 0 + and time.time() - self._connection_start_time >= CONNECTION_ESTABLISHED_THRESHOLD + ): + logger.info( + f"Connection stable for {CONNECTION_ESTABLISHED_THRESHOLD}s, " + f"resetting failure counter from {self._consecutive_failures} to 0" + ) + self._consecutive_failures = 0 + + async def _handle_connection_error(self, error: Exception) -> bool: + """Handle a connection error and determine if reconnection should be attempted. + + Args: + error: The exception that caused the connection error. + + Returns: + True if reconnection should be attempted, False if a fatal error should be pushed. + """ + self._consecutive_failures += 1 + logger.warning( + f"Connection error (failure {self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES}): {error}" + ) + + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + logger.error( + f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, " + "treating as fatal error" + ) + await self.push_error( + ErrorFrame(error=f"{self} Error in receive loop: {error}", fatal=True) + ) + return False + else: + logger.info( + f"Attempting reconnection ({self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES})" + ) + return True + + async def _reconnect(self): + """Reconnect to Gemini Live API.""" + await self._disconnect() + await self._connect(session_resumption_handle=self._session_resumption_handle) + + async def _disconnect(self): + """Disconnect from Gemini Live API and clean up resources.""" + logger.info("Disconnecting from Gemini service") + try: + self._disconnecting = True + await self.stop_all_metrics() + if self._connection_task: + await self.cancel_task(self._connection_task, timeout=1.0) + self._connection_task = None + if self._session: + await self._session.close() + self._session = None + self._disconnecting = False + except Exception as e: + logger.error(f"{self} error disconnecting: {e}") + + async def _send_user_audio(self, frame): + """Send user audio frame to Gemini Live API.""" + if self._audio_input_paused or self._disconnecting or not self._session: + return + + # Send all audio to Gemini + try: + await self._session.send_realtime_input( + audio=Blob(data=frame.audio, mime_type=f"audio/pcm;rate={frame.sample_rate}") + ) + except Exception as e: + await self._handle_send_error(e) + + # Manage a buffer of audio to use for transcription + audio = frame.audio + if self._user_is_speaking: + self._user_audio_buffer.extend(audio) + else: + # Keep 1/2 second of audio in the buffer even when not speaking. + self._user_audio_buffer.extend(audio) + length = int((frame.sample_rate * frame.num_channels * 2) * 0.5) + self._user_audio_buffer = self._user_audio_buffer[-length:] + + async def _send_user_text(self, text: str): + """Send user text via Gemini Live API's realtime input stream. + + This method sends text through the realtimeInput stream (via TextInputMessage) + rather than the clientContent stream. This ensures text input is synchronized + with audio and video inputs, preventing temporal misalignment that can occur + when different modalities are processed through separate API pathways. + + For realtimeInput, turn completion is automatically inferred by the API based + on user activity, so no explicit turnComplete signal is needed. + + Args: + text: The text to send as user input. + """ + if self._disconnecting or not self._session: + return + + try: + await self._session.send_realtime_input(text=text) + except Exception as e: + await self._handle_send_error(e) + + async def _send_user_video(self, frame): + """Send user video frame to Gemini Live API.""" + if self._video_input_paused or self._disconnecting or not self._session: + return + + now = time.time() + if now - self._last_sent_time < 1: + return # Ignore if less than 1 second has passed + + self._last_sent_time = now # Update last sent time + logger.debug(f"Sending video frame to Gemini: {frame}") + + buffer = io.BytesIO() + Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") + data = base64.b64encode(buffer.getvalue()).decode("utf-8") + + try: + await self._session.send_realtime_input(video=Blob(data=data, mime_type="image/jpeg")) + except Exception as e: + await self._handle_send_error(e) + + async def _create_initial_response(self): + """Create initial response based on context history.""" + if self._disconnecting: + return + + if not self._session: + self._run_llm_when_session_ready = True + return + + messages = self._context.get_messages_for_initializing_history() + if not messages: + return + + logger.debug(f"Creating initial response: {messages}") + + await self.start_ttfb_metrics() + + try: + await self._session.send_client_content( + turns=messages, turn_complete=self._inference_on_context_initialization + ) + except Exception as e: + await self._handle_send_error(e) + + # If we're generating a response right away upon initializing + # conversation history, set a flag saying that we need a turn complete + # message when the user stops speaking. + if not self._inference_on_context_initialization: + self._needs_turn_complete_message = True + + async def _create_single_response(self, messages_list): + """Create a single response from a list of messages.""" + if self._disconnecting or not self._session: + return + + # Create a throwaway context just for the purpose of getting messages + # in the right format + context = GeminiLiveContext.upgrade(OpenAILLMContext(messages=messages_list)) + messages = context.get_messages_for_initializing_history() + + if not messages: + return + + logger.debug(f"Creating response: {messages}") + + await self.start_ttfb_metrics() + + try: + await self._session.send_client_content(turns=messages, turn_complete=True) + except Exception as e: + await self._handle_send_error(e) + + @traced_gemini_live(operation="llm_tool_result") + async def _tool_result(self, tool_result_message): + """Send tool result back to the API.""" + if self._disconnecting or not self._session: + return + + # For now we're shoving the name into the tool_call_id field, so this + # will work until we revisit that. + id = tool_result_message.get("tool_call_id") + name = tool_result_message.get("tool_call_name") + result = json.loads(tool_result_message.get("content") or "") + response = FunctionResponse(name=name, id=id, response=result) + + try: + await self._session.send_tool_response(function_responses=response) + except Exception as e: + await self._handle_send_error(e) + + @traced_gemini_live(operation="llm_setup") + async def _handle_session_ready(self, session: AsyncSession): + """Handle the session being ready.""" + self._session = session + # If we were just waititng for the session to be ready to run the LLM, + # do that now. + if self._run_llm_when_session_ready: + self._run_llm_when_session_ready = False + await self._create_initial_response() + + async def _handle_msg_model_turn(self, msg: LiveServerMessage): + """Handle the model turn message.""" + part = msg.server_content.model_turn.parts[0] + if not part: + return + + await self.stop_ttfb_metrics() + + # part.text is added when `modalities` is set to TEXT; otherwise, it's None + text = part.text + if text: + if not self._bot_text_buffer: + await self.push_frame(LLMFullResponseStartFrame()) + + self._bot_text_buffer += text + self._search_result_buffer += text # Also accumulate for grounding + await self.push_frame(LLMTextFrame(text=text)) + + # Check for grounding metadata in server content + if msg.server_content and msg.server_content.grounding_metadata: + self._accumulated_grounding_metadata = msg.server_content.grounding_metadata + + inline_data = part.inline_data + if not inline_data: + return + + # Check if mime type matches expected format + expected_mime_type = f"audio/pcm;rate={self._sample_rate}" + if inline_data.mime_type == expected_mime_type: + # Perfect match, continue processing + pass + elif inline_data.mime_type == "audio/pcm": + # Sample rate not provided in mime type, assume default + if not hasattr(self, "_sample_rate_warning_logged"): + logger.warning( + f"Sample rate not provided in mime type '{inline_data.mime_type}', assuming rate of {self._sample_rate}" + ) + self._sample_rate_warning_logged = True + else: + # Unrecognized format + logger.warning(f"Unrecognized server_content format {inline_data.mime_type}") + return + + audio = inline_data.data + if not audio: + return + + if not self._bot_is_speaking: + await self._set_bot_is_speaking(True) + await self.push_frame(TTSStartedFrame()) + await self.push_frame(LLMFullResponseStartFrame()) + + self._bot_audio_buffer.extend(audio) + frame = TTSAudioRawFrame( + audio=audio, + sample_rate=self._sample_rate, + num_channels=1, + ) + await self.push_frame(frame) + + @traced_gemini_live(operation="llm_tool_call") + async def _handle_msg_tool_call(self, message: LiveServerMessage): + """Handle tool call messages.""" + function_calls = message.tool_call.function_calls + if not function_calls: + return + if not self._context: + logger.error("Function calls are not supported without a context object.") + + function_calls_llm = [ + FunctionCallFromLLM( + context=self._context, + tool_call_id=( + # NOTE: when using Vertex AI we don't get server-provided + # tool call IDs here + f.id or str(uuid.uuid4()) + ), + function_name=f.name, + arguments=f.args, + ) + for f in function_calls + ] + + await self.run_function_calls(function_calls_llm) + + @traced_gemini_live(operation="llm_response") + async def _handle_msg_turn_complete(self, message: LiveServerMessage): + """Handle the turn complete message.""" + await self._set_bot_is_speaking(False) + text = self._bot_text_buffer + + # Trace the complete LLM response (this will be handled by the decorator) + # The decorator will extract the output text and usage metadata from the message + + self._bot_text_buffer = "" + self._llm_output_buffer = "" + + # Process grounding metadata if we have accumulated any + if self._accumulated_grounding_metadata: + await self._process_grounding_metadata( + self._accumulated_grounding_metadata, self._search_result_buffer + ) + + # Reset grounding tracking for next response + self._search_result_buffer = "" + self._accumulated_grounding_metadata = None + + # Only push the TTSStoppedFrame if the bot is outputting audio + # when text is found, modalities is set to TEXT and no audio + # is produced. + if not text: + await self.push_frame(TTSStoppedFrame()) + + await self.push_frame(LLMFullResponseEndFrame()) + + @traced_stt + async def _handle_user_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing.""" + pass + + async def _handle_msg_input_transcription(self, message: LiveServerMessage): + """Handle the input transcription message. + + Gemini Live sends user transcriptions in either single words or multi-word + phrases. As a result, we have to aggregate the input transcription. This handler + aggregates into sentences, splitting on the end of sentence markers. + """ + if not message.server_content.input_transcription: + return + + text = message.server_content.input_transcription.text + + if not text: + return + + # Strip leading space from sentence starts if buffer is empty + if text.startswith(" ") and not self._user_transcription_buffer: + text = text.lstrip() + + # Accumulate text in the buffer + self._user_transcription_buffer += text + + # Check for complete sentences + while True: + eos_end_marker = match_endofsentence(self._user_transcription_buffer) + if not eos_end_marker: + break + + # Extract the complete sentence + complete_sentence = self._user_transcription_buffer[:eos_end_marker] + # Keep the remainder for the next chunk + self._user_transcription_buffer = self._user_transcription_buffer[eos_end_marker:] + + # Send a TranscriptionFrame with the complete sentence + logger.debug(f"[Transcription:user] [{complete_sentence}]") + await self._handle_user_transcription( + complete_sentence, True, self._settings["language"] + ) + await self.push_frame( + TranscriptionFrame( + text=complete_sentence, + user_id="", + timestamp=time_now_iso8601(), + result=message, + ), + FrameDirection.UPSTREAM, + ) + + async def _handle_msg_output_transcription(self, message: LiveServerMessage): + """Handle the output transcription message.""" + if not message.server_content.output_transcription: + return + + # This is the output transcription text when modalities is set to AUDIO. + # In this case, we push LLMTextFrame and TTSTextFrame to be handled by the + # downstream assistant context aggregator. + text = message.server_content.output_transcription.text + + if not text: + return + + # Accumulate text for grounding as well + self._search_result_buffer += text + + # Check for grounding metadata in server content + if message.server_content and message.server_content.grounding_metadata: + self._accumulated_grounding_metadata = message.server_content.grounding_metadata + # Collect text for tracing + self._llm_output_buffer += text + + await self.push_frame(LLMTextFrame(text=text)) + await self.push_frame(TTSTextFrame(text=text)) + + async def _handle_msg_grounding_metadata(self, message: LiveServerMessage): + """Handle dedicated grounding metadata messages.""" + if message.server_content and message.server_content.grounding_metadata: + grounding_metadata = message.server_content.grounding_metadata + # Process the grounding metadata immediately + await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer) + + async def _process_grounding_metadata( + self, grounding_metadata: GroundingMetadata, search_result: str = "" + ): + """Process grounding metadata and emit LLMSearchResponseFrame.""" + if not grounding_metadata: + return + + # Extract rendered content for search suggestions + rendered_content = None + if ( + grounding_metadata.search_entry_point + and grounding_metadata.search_entry_point.rendered_content + ): + rendered_content = grounding_metadata.search_entry_point.rendered_content + + # Convert grounding chunks and supports to LLMSearchOrigin format + origins = [] + + if grounding_metadata.grounding_chunks and grounding_metadata.grounding_supports: + # Create a mapping of chunk indices to origins + chunk_to_origin: Dict[int, LLMSearchOrigin] = {} + + for index, chunk in enumerate(grounding_metadata.grounding_chunks): + if chunk.web: + origin = LLMSearchOrigin( + site_uri=chunk.web.uri, site_title=chunk.web.title, results=[] + ) + chunk_to_origin[index] = origin + origins.append(origin) + + # Add grounding support results to the appropriate origins + for support in grounding_metadata.grounding_supports: + if support.segment and support.grounding_chunk_indices: + text = support.segment.text or "" + confidence_scores = support.confidence_scores or [] + + # Add this result to all origins referenced by this support + for chunk_index in support.grounding_chunk_indices: + if chunk_index in chunk_to_origin: + result = LLMSearchResult(text=text, confidence=confidence_scores) + chunk_to_origin[chunk_index].results.append(result) + + # Create and push the search response frame + search_frame = LLMSearchResponseFrame( + search_result=search_result, origins=origins, rendered_content=rendered_content + ) + + await self.push_frame(search_frame) + + async def _handle_msg_usage_metadata(self, message: LiveServerMessage): + """Handle the usage metadata message.""" + if not message.usage_metadata: + return + + usage = message.usage_metadata + + # Ensure we have valid integers for all token counts + prompt_tokens = usage.prompt_token_count or 0 + completion_tokens = usage.response_token_count or 0 + total_tokens = usage.total_token_count or (prompt_tokens + completion_tokens) + + tokens = LLMTokenUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + + await self.start_llm_usage_metrics(tokens) + + def _handle_msg_resumption_update(self, message: LiveServerMessage): + update = message.session_resumption_update + if update.resumable and update.new_handle: + self._session_resumption_handle = update.new_handle + + async def _handle_send_error(self, error: Exception): + # In server-to-server contexts, a WebSocket error should be quite rare. + # Given how hard it is to recover from a send-side error with proper + # state management, and that exponential backoff for retries can have + # cost/stability implications for a service cluster, let's just treat a + # send-side error as fatal. + if not self._disconnecting: + await self.push_error(ErrorFrame(error=f"{self} Send error: {error}", fatal=True)) + + def create_context_aggregator( + self, + context: OpenAILLMContext, + *, + user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), + assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), + ) -> GeminiLiveContextAggregatorPair: + """Create an instance of GeminiLiveContextAggregatorPair from an OpenAILLMContext. + + Constructor keyword arguments for both the user and assistant aggregators can be provided. + + Args: + context: The LLM context to use. + user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams(). + assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams(). + + Returns: + GeminiLiveContextAggregatorPair: A pair of context + aggregators, one for the user and one for the assistant, + encapsulated in an GeminiLiveContextAggregatorPair. + """ + context.set_llm_adapter(self.get_llm_adapter()) + + GeminiLiveContext.upgrade(context) + user = GeminiLiveUserContextAggregator(context, params=user_params) + + assistant_params.expect_stripped_words = False + assistant = GeminiLiveAssistantContextAggregator(context, params=assistant_params) + return GeminiLiveContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/google/gemini_live/llm_vertex.py b/src/pipecat/services/google/gemini_live/llm_vertex.py new file mode 100644 index 000000000..a38154755 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/llm_vertex.py @@ -0,0 +1,184 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Service for accessing Gemini Live via Google Vertex AI. + +This module provides integration with Google's Gemini Live model via +Vertex AI, supporting both text and audio modalities with voice transcription, +streaming responses, and tool usage. +""" + +import json +from typing import List, Optional, Union + +from loguru import logger + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.services.google.gemini_live.llm import ( + GeminiLiveLLMService, + HttpOptions, + InputParams, +) + +try: + from google.auth import default + from google.auth.exceptions import GoogleAuthError + from google.auth.transport.requests import Request + from google.genai import Client + from google.oauth2 import service_account + +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Google Vertex AI, you need to `pip install pipecat-ai[google]`.") + raise Exception(f"Missing module: {e}") + + +class GeminiLiveVertexLLMService(GeminiLiveLLMService): + """Provides access to Google's Gemini Live model via Vertex AI. + + This service enables real-time conversations with Gemini, supporting both + text and audio modalities. It handles voice transcription, streaming audio + responses, and tool usage. + """ + + def __init__( + self, + *, + credentials: Optional[str] = None, + credentials_path: Optional[str] = None, + location: str, + project_id: str, + model="google/gemini-2.0-flash-live-preview-04-09", + voice_id: str = "Charon", + start_audio_paused: bool = False, + start_video_paused: bool = False, + system_instruction: Optional[str] = None, + tools: Optional[Union[List[dict], ToolsSchema]] = None, + params: Optional[InputParams] = None, + inference_on_context_initialization: bool = True, + file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files", + http_options: Optional[HttpOptions] = None, + **kwargs, + ): + """Initialize the service for accessing Gemini Live via Google Vertex AI. + + Args: + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. + location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + project_id: Google Cloud project ID. + model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-preview-04-09". + voice_id: TTS voice identifier. Defaults to "Charon". + start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_video_paused: Whether to start with video input paused. Defaults to False. + system_instruction: System prompt for the model. Defaults to None. + tools: Tools/functions available to the model. Defaults to None. + params: Configuration parameters for the model along with Vertex AI + location and project ID. + inference_on_context_initialization: Whether to generate a response when context + is first set. Defaults to True. + file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint. + http_options: HTTP options for the client. + **kwargs: Additional arguments passed to parent GeminiLiveLLMService. + """ + # Check if user incorrectly passed api_key, which is used by parent + # class but not here. + if "api_key" in kwargs: + logger.error( + "GeminiLiveVertexLLMService does not accept 'api_key' parameter. " + "Use 'credentials' or 'credentials_path' instead for Vertex AI authentication." + ) + raise ValueError( + "Invalid parameter 'api_key'. Use 'credentials' or 'credentials_path' for Vertex AI authentication." + ) + + # These need to be set before calling super().__init__() because + # super().__init__() invokes create_client(), which needs these. + self._credentials = self._get_credentials(credentials, credentials_path) + self._project_id = project_id + self._location = location + + # Call parent constructor with the obtained API key + super().__init__( + # api_key is required by parent class, but actually not used with + # Vertex + api_key="dummy", + model=model, + voice_id=voice_id, + start_audio_paused=start_audio_paused, + start_video_paused=start_video_paused, + system_instruction=system_instruction, + tools=tools, + params=params, + inference_on_context_initialization=inference_on_context_initialization, + file_api_base_url=file_api_base_url, + http_options=http_options, + **kwargs, + ) + + def create_client(self): + """Create the Gemini client instance.""" + self._client = Client( + vertexai=True, + credentials=self._credentials, + project=self._project_id, + location=self._location, + ) + + @property + def file_api(self): + """Gemini File API is not supported with Vertex AI.""" + raise NotImplementedError( + "When using Vertex AI, the recommended approach is to use Google Cloud Storage for file handling. The Gemini File API is not directly supported in this context." + ) + + @staticmethod + def _get_credentials(credentials: Optional[str], credentials_path: Optional[str]) -> str: + """Retrieve Credentials using Google service account credentials JSON. + + Supports multiple authentication methods: + 1. Direct JSON credentials string + 2. Path to service account JSON file + 3. Default application credentials (ADC) + + Args: + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. + + Returns: + OAuth token for API authentication. + + Raises: + ValueError: If no valid credentials are provided or found. + """ + creds: Optional[service_account.Credentials] = None + + if credentials: + # Parse and load credentials from JSON string + creds = service_account.Credentials.from_service_account_info( + json.loads(credentials), + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + elif credentials_path: + # Load credentials from JSON file + creds = service_account.Credentials.from_service_account_file( + credentials_path, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + else: + try: + creds, project_id = default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + except GoogleAuthError: + pass + + if not creds: + raise ValueError("No valid credentials provided.") + + creds.refresh(Request()) # Ensure token is up-to-date, lifetime is 1 hour. + + return creds diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 70d4ca2bf..c59cb41ae 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -35,6 +35,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, + OutputImageRawFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -72,6 +73,9 @@ try: HttpOptions, Part, ) + + # Temporary hack to be able to process Nano Banana returned images. + genai._api_client.READ_BUFFER_SIZE = 5 * 1024 * 1024 except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") @@ -682,7 +686,7 @@ class GoogleLLMService(LLMService): self, *, api_key: str, - model: str = "gemini-2.0-flash", + model: str = "gemini-2.5-flash", params: Optional[InputParams] = None, system_instruction: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, @@ -710,6 +714,7 @@ class GoogleLLMService(LLMService): self._api_key = api_key self._system_instruction = system_instruction self._http_options = http_options + self._create_client(api_key, http_options) self._settings = { "max_tokens": params.max_tokens, @@ -788,6 +793,9 @@ class GoogleLLMService(LLMService): # and can be configured to turn it off. if not self._model_name.startswith("gemini-2.5-flash"): return + # If we have an image model, we don't use a budget either. + if "image" in self._model_name: + return # If thinking_config is already set, don't override it. if "thinking_config" in generation_params: return @@ -927,6 +935,12 @@ class GoogleLLMService(LLMService): arguments=function_call.args or {}, ) ) + elif part.inline_data and part.inline_data.data: + image = Image.open(io.BytesIO(part.inline_data.data)) + frame = OutputImageRawFrame( + image=image.tobytes(), size=image.size, format="RGB" + ) + await self.push_frame(frame) if ( candidate.grounding_metadata @@ -1020,6 +1034,23 @@ class GoogleLLMService(LLMService): if context: await self._process_context(context) + async def stop(self, frame): + """Override stop to gracefully close the client.""" + await super().stop(frame) + await self._close_client() + + async def cancel(self, frame): + """Override cancel to gracefully close the client.""" + await super().cancel(frame) + await self._close_client() + + async def _close_client(self): + try: + await self._client.aio.aclose() + except Exception: + # Do nothing - we're shutting down anyway + pass + def create_context_aggregator( self, context: OpenAILLMContext, diff --git a/src/pipecat/services/google/llm_openai.py b/src/pipecat/services/google/llm_openai.py index 81d124cb7..fde62154e 100644 --- a/src/pipecat/services/google/llm_openai.py +++ b/src/pipecat/services/google/llm_openai.py @@ -94,9 +94,9 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): async for chunk in chunk_stream: if chunk.usage: tokens = LLMTokenUsage( - prompt_tokens=chunk.usage.prompt_tokens, - completion_tokens=chunk.usage.completion_tokens, - total_tokens=chunk.usage.total_tokens, + prompt_tokens=chunk.usage.prompt_tokens or 0, + completion_tokens=chunk.usage.completion_tokens or 0, + total_tokens=chunk.usage.total_tokens or 0, ) await self.start_llm_usage_metrics(tokens) diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py index ee66c1f20..49adb2e9b 100644 --- a/src/pipecat/services/google/llm_vertex.py +++ b/src/pipecat/services/google/llm_vertex.py @@ -53,12 +53,44 @@ class GoogleVertexLLMService(OpenAILLMService): Parameters: location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + + .. deprecated:: 0.0.90 + Use `location` as a direct argument to + `GoogleVertexLLMService.__init__()` instead. + project_id: Google Cloud project ID. + + .. deprecated:: 0.0.90 + Use `project_id` as a direct argument to + `GoogleVertexLLMService.__init__()` instead. """ # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations - location: str = "us-east4" - project_id: str + location: Optional[str] = None + project_id: Optional[str] = None + + def __init__(self, **kwargs): + """Initializes the InputParams.""" + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + if "location" in kwargs and kwargs["location"] is not None: + warnings.warn( + "GoogleVertexLLMService.InputParams.location is deprecated. " + "Please provide 'location' as a direct argument to GoogleVertexLLMService.__init__() instead.", + DeprecationWarning, + stacklevel=2, + ) + + if "project_id" in kwargs and kwargs["project_id"] is not None: + warnings.warn( + "GoogleVertexLLMService.InputParams.project_id is deprecated. " + "Please provide 'project_id' as a direct argument to GoogleVertexLLMService.__init__() instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(**kwargs) def __init__( self, @@ -66,7 +98,8 @@ class GoogleVertexLLMService(OpenAILLMService): credentials: Optional[str] = None, credentials_path: Optional[str] = None, model: str = "google/gemini-2.0-flash-001", - params: Optional[InputParams] = None, + location: Optional[str] = None, + project_id: Optional[str] = None, **kwargs, ): """Initializes the VertexLLMService. @@ -75,33 +108,60 @@ class GoogleVertexLLMService(OpenAILLMService): credentials: JSON string of service account credentials. credentials_path: Path to the service account JSON file. model: Model identifier (e.g., "google/gemini-2.0-flash-001"). - params: Vertex AI input parameters including location and project. + location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + project_id: Google Cloud project ID. **kwargs: Additional arguments passed to OpenAILLMService. """ - params = params or OpenAILLMService.InputParams() - base_url = self._get_base_url(params) + # Handle deprecated InputParams fields + if "params" in kwargs and isinstance(kwargs["params"], GoogleVertexLLMService.InputParams): + params = kwargs["params"] + # Extract location and project_id from params if not provided + # directly, for backward compatibility + if project_id is None: + project_id = params.project_id + if location is None: + location = params.location + # Convert to base InputParams + params = OpenAILLMService.InputParams( + **params.model_dump(exclude={"location", "project_id"}, exclude_unset=True) + ) + kwargs["params"] = params + + # Validate project_id and location parameters + # NOTE: once we remove Vertex-spcific InputParams class, we can update + # __init__() signature as follows: + # - location: str = "us-east4", + # - project_id: str, + # But for now, we need them as-is to maintain proper backward + # compatibility. + if project_id is None: + raise ValueError("project_id is required") + if location is None: + # If location is not provided, default to "us-east4". + # Note: this is legacy behavior; ideally location would be + # required. + logger.warning("location is not provided. Defaulting to 'us-east4'.") + location = "us-east4" # Default location if not provided + + base_url = self._get_base_url(location, project_id) self._api_key = self._get_api_token(credentials, credentials_path) super().__init__( api_key=self._api_key, base_url=base_url, model=model, - params=params, **kwargs, ) @staticmethod - def _get_base_url(params: InputParams) -> str: + def _get_base_url(location: str, project_id: str) -> str: """Construct the base URL for Vertex AI API.""" # Determine the correct API host based on location - if params.location == "global": + if location == "global": api_host = "aiplatform.googleapis.com" else: - api_host = f"{params.location}-aiplatform.googleapis.com" - return ( - f"https://{api_host}/v1/" - f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" - ) + api_host = f"{location}-aiplatform.googleapis.com" + return f"https://{api_host}/v1/projects/{project_id}/locations/{location}/endpoints/openapi" @staticmethod def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 31ae597f7..b9e56f55b 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -730,6 +730,8 @@ class GoogleSTTService(STTService): self._request_queue = asyncio.Queue() self._streaming_task = self.create_task(self._stream_audio()) + await self._call_event_handler("on_connected") + async def _disconnect(self): """Clean up streaming recognition resources.""" if self._streaming_task: @@ -737,6 +739,8 @@ class GoogleSTTService(STTService): await self.cancel_task(self._streaming_task) self._streaming_task = None + await self._call_event_handler("on_disconnected") + async def _request_generator(self): """Generates requests for the streaming recognize method.""" recognizer_path = f"projects/{self._project_id}/locations/{self._location}/recognizers/_" diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index c995f426d..34947fb44 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -42,7 +42,7 @@ class HumeTTSService(TTSService): """Hume Octave Text-to-Speech service. Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint - using the Python SDK and emits `TTSAudioRawFrame`s suitable for Pipecat transports. + using the Python SDK and emits ``TTSAudioRawFrame`` frames suitable for Pipecat transports. Supported features: @@ -78,7 +78,7 @@ class HumeTTSService(TTSService): Args: api_key: Hume API key. If omitted, reads the ``HUME_API_KEY`` environment variable. - voice_id: ID of the voice to use (ID-only; names are not supported here). + voice_id: ID of the voice to use. Only voice IDs are supported; voice names are not. params: Optional synthesis controls (acting instructions, speed, trailing silence). sample_rate: Output sample rate for emitted PCM frames. Defaults to 48_000 (Hume). **kwargs: Additional arguments passed to the parent class. diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index a602789fd..9f9fef5fc 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -222,6 +222,7 @@ class LmntTTSService(InterruptibleTTSService): # Send initialization message await self._websocket.send(json.dumps(init_msg)) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -243,6 +244,7 @@ class LmntTTSService(InterruptibleTTSService): finally: self._started = False self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): """Get the WebSocket connection if available.""" diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 46d805086..6ccdfe17f 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -293,6 +293,8 @@ class NeuphonicTTSService(InterruptibleTTSService): headers = {"x-api-key": self._api_key} self._websocket = await websocket_connect(url, additional_headers=headers) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -311,6 +313,7 @@ class NeuphonicTTSService(InterruptibleTTSService): finally: self._started = False self._websocket = None + await self._call_event_handler("on_disconnected") async def _receive_messages(self): """Receive and process messages from Neuphonic WebSocket.""" diff --git a/src/pipecat/services/openai/__init__.py b/src/pipecat/services/openai/__init__.py index 4decac126..d913c5ee0 100644 --- a/src/pipecat/services/openai/__init__.py +++ b/src/pipecat/services/openai/__init__.py @@ -10,6 +10,7 @@ from pipecat.services import DeprecatedModuleProxy from .image import * from .llm import * +from .realtime import * from .stt import * from .tts import * diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 5c48f86e9..d020e1106 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -66,6 +66,7 @@ class BaseOpenAILLMService(LLMService): top_p: Top-p (nucleus) sampling parameter (0.0 to 1.0). max_tokens: Maximum tokens in response (deprecated, use max_completion_tokens). max_completion_tokens: Maximum completion tokens to generate. + service_tier: Service tier to use (e.g., "auto", "flex", "priority"). extra: Additional model-specific parameters. """ @@ -83,6 +84,7 @@ class BaseOpenAILLMService(LLMService): top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0) max_tokens: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=1) max_completion_tokens: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=1) + service_tier: Optional[str] = Field(default_factory=lambda: NOT_GIVEN) extra: Optional[Dict[str, Any]] = Field(default_factory=dict) def __init__( @@ -125,6 +127,7 @@ class BaseOpenAILLMService(LLMService): "top_p": params.top_p, "max_tokens": params.max_tokens, "max_completion_tokens": params.max_completion_tokens, + "service_tier": params.service_tier, "extra": params.extra if isinstance(params.extra, dict) else {}, } self._retry_timeout_secs = retry_timeout_secs @@ -236,6 +239,7 @@ class BaseOpenAILLMService(LLMService): "top_p": self._settings["top_p"], "max_tokens": self._settings["max_tokens"], "max_completion_tokens": self._settings["max_completion_tokens"], + "service_tier": self._settings["service_tier"], } # Messages, tools, tool_choice diff --git a/src/pipecat/services/openai/realtime/__init__.py b/src/pipecat/services/openai/realtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py new file mode 100644 index 000000000..cb1c0a9f5 --- /dev/null +++ b/src/pipecat/services/openai/realtime/context.py @@ -0,0 +1,272 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""OpenAI Realtime LLM context and aggregator implementations.""" + +import copy +import json + +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + FunctionCallResultFrame, + InterimTranscriptionFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMTextFrame, + TranscriptionFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) + +from . import events +from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame + + +class OpenAIRealtimeLLMContext(OpenAILLMContext): + """OpenAI Realtime LLM context with session management and message conversion. + + Extends the standard OpenAI LLM context to support real-time session properties, + instruction management, and conversion between standard message formats and + realtime conversation items. + """ + + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize the OpenAIRealtimeLLMContext. + + Args: + messages: Initial conversation messages. Defaults to None. + tools: Available function tools. Defaults to None. + **kwargs: Additional arguments passed to parent OpenAILLMContext. + """ + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self): + self.llm_needs_settings_update = True + self.llm_needs_initial_messages = True + self._session_instructions = "" + + return + + @staticmethod + def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": + """Upgrade a standard OpenAI LLM context to a realtime context. + + Args: + obj: The OpenAILLMContext instance to upgrade. + + Returns: + The upgraded OpenAIRealtimeLLMContext instance. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): + obj.__class__ = OpenAIRealtimeLLMContext + obj.__setup_local() + return obj + + # todo + # - finish implementing all frames + + def from_standard_message(self, message): + """Convert a standard message format to a realtime conversation item. + + Args: + message: The standard message dictionary to convert. + + Returns: + A ConversationItem instance for the realtime API. + """ + if message.get("role") == "user": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + return events.ConversationItem( + role="user", + type="message", + content=[events.ItemContent(type="input_text", text=content)], + ) + if message.get("role") == "assistant" and message.get("tool_calls"): + tc = message.get("tool_calls")[0] + return events.ConversationItem( + type="function_call", + call_id=tc["id"], + name=tc["function"]["name"], + arguments=tc["function"]["arguments"], + ) + logger.error(f"Unhandled message type in from_standard_message: {message}") + + def get_messages_for_initializing_history(self): + """Get conversation items for initializing the realtime session history. + + Converts the context's messages to a format suitable for the realtime API, + handling system instructions and conversation history packaging. + + Returns: + List of conversation items for session initialization. + """ + # We can't load a long conversation history into the openai realtime api yet. (The API/model + # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So + # our general strategy until this is fixed is just to put everything into a first "user" + # message as a single input. + if not self.messages: + return [] + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into session + # "instructions" + if messages[0].get("role") == "system": + self.llm_needs_settings_update = True + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + self._session_instructions = content + elif isinstance(content, list): + self._session_instructions = content[0].get("text") + if not messages: + return [] + + # If we have just a single "user" item, we can just send it normally + if len(messages) == 1 and messages[0].get("role") == "user": + return [self.from_standard_message(messages[0])] + + # Otherwise, let's pack everything into a single "user" message with a bit of + # explanation for the LLM + intro_text = """ + This is a previously saved conversation. Please treat this conversation history as a + starting point for the current conversation.""" + + trailing_text = """ + This is the end of the previously saved conversation. Please continue the conversation + from here. If the last message is a user instruction or question, act on that instruction + or answer the question. If the last message is an assistant response, simple say that you + are ready to continue the conversation.""" + + return [ + { + "role": "user", + "type": "message", + "content": [ + { + "type": "input_text", + "text": "\n\n".join( + [intro_text, json.dumps(messages, indent=2), trailing_text] + ), + } + ], + } + ] + + def add_user_content_item_as_message(self, item): + """Add a user content item as a standard message to the context. + + Args: + item: The conversation item to add as a user message. + """ + message = { + "role": "user", + "content": [{"type": "text", "text": item.content[0].transcript}], + } + self.add_message(message) + + +class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): + """User context aggregator for OpenAI Realtime API. + + Handles user input frames and generates appropriate context updates + for the realtime conversation, including message updates and tool settings. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + + async def process_frame( + self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM + ): + """Process incoming frames and handle realtime-specific frame types. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + # Parent does not push LLMMessagesUpdateFrame. This ensures that in a typical pipeline, + # messages are only processed by the user context aggregator, which is generally what we want. But + # we also need to send new messages over the websocket, so the openai realtime API has them + # in its context. + if isinstance(frame, LLMMessagesUpdateFrame): + await self.push_frame(RealtimeMessagesUpdateFrame(context=self._context)) + + # Parent also doesn't push the LLMSetToolsFrame. + if isinstance(frame, LLMSetToolsFrame): + await self.push_frame(frame, direction) + + async def push_aggregation(self): + """Push user input aggregation. + + Currently ignores all user input coming into the pipeline as realtime + audio input is handled directly by the service. + """ + # for the moment, ignore all user input coming into the pipeline. + # todo: think about whether/how to fix this to allow for text input from + # upstream (transport/transcription, or other sources) + pass + + +class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Assistant context aggregator for OpenAI Realtime API. + + Handles assistant output frames from the realtime service, filtering + out duplicate text frames and managing function call results. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. + # OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames, + # so we need to ignore pushing those as well, as they're also TextFrames. + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process assistant frames, filtering out duplicate text content. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)): + await super().process_frame(frame, direction) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call result and notify the realtime service. + + Args: + frame: The function call result frame to handle. + """ + await super().handle_function_call_result(frame) + + # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, + # so we didn't have a chance to add the result to the openai realtime api context. Let's push a + # special frame to do that. + await self.push_frame( + RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py new file mode 100644 index 000000000..200e59d68 --- /dev/null +++ b/src/pipecat/services/openai/realtime/events.py @@ -0,0 +1,1106 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Event models and data structures for OpenAI Realtime API communication.""" + +import json +import uuid +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field + +# +# session properties +# + + +class AudioFormat(BaseModel): + """Base class for audio format configuration.""" + + type: str + + +class PCMAudioFormat(AudioFormat): + """PCM audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcm". + rate: Sample rate, always 24000 for PCM. + """ + + type: Literal["audio/pcm"] = "audio/pcm" + rate: Literal[24000] = 24000 + + +class PCMUAudioFormat(AudioFormat): + """PCMU (G.711 ΞΌ-law) audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcmu". + """ + + type: Literal["audio/pcmu"] = "audio/pcmu" + + +class PCMAAudioFormat(AudioFormat): + """PCMA (G.711 A-law) audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcma". + """ + + type: Literal["audio/pcma"] = "audio/pcma" + + +class InputAudioTranscription(BaseModel): + """Configuration for audio transcription settings.""" + + model: str = "gpt-4o-transcribe" + language: Optional[str] + prompt: Optional[str] + + def __init__( + self, + model: Optional[str] = "gpt-4o-transcribe", + language: Optional[str] = None, + prompt: Optional[str] = None, + ): + """Initialize InputAudioTranscription. + + Args: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ + super().__init__(model=model, language=language, prompt=prompt) + + +class TurnDetection(BaseModel): + """Server-side voice activity detection configuration. + + Parameters: + type: Detection type, must be "server_vad". + threshold: Voice activity detection threshold (0.0-1.0). Defaults to 0.5. + prefix_padding_ms: Padding before speech starts in milliseconds. Defaults to 300. + silence_duration_ms: Silence duration to detect speech end in milliseconds. Defaults to 500. + """ + + type: Optional[Literal["server_vad"]] = "server_vad" + threshold: Optional[float] = 0.5 + prefix_padding_ms: Optional[int] = 300 + silence_duration_ms: Optional[int] = 500 + + +class SemanticTurnDetection(BaseModel): + """Semantic-based turn detection configuration. + + Parameters: + type: Detection type, must be "semantic_vad". + eagerness: Turn detection eagerness level. Can be "low", "medium", "high", or "auto". + create_response: Whether to automatically create responses on turn detection. + interrupt_response: Whether to interrupt ongoing responses on turn detection. + """ + + type: Optional[Literal["semantic_vad"]] = "semantic_vad" + eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None + create_response: Optional[bool] = None + interrupt_response: Optional[bool] = None + + +class InputAudioNoiseReduction(BaseModel): + """Input audio noise reduction configuration. + + Parameters: + type: Noise reduction type for different microphone scenarios. + """ + + type: Optional[Literal["near_field", "far_field"]] + + +class AudioInput(BaseModel): + """Audio input configuration. + + Parameters: + format: The format of the input audio. + transcription: Configuration for input audio transcription. + noise_reduction: Configuration for input audio noise reduction. + turn_detection: Configuration for turn detection, or False to disable. + """ + + format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None + transcription: Optional[InputAudioTranscription] = None + noise_reduction: Optional[InputAudioNoiseReduction] = None + turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = None + + +class AudioOutput(BaseModel): + """Audio output configuration. + + Parameters: + format: The format of the output audio. + voice: The voice the model uses to respond. + speed: The speed of the model's spoken response. + """ + + format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None + voice: Optional[str] = None + speed: Optional[float] = None + + +class AudioConfiguration(BaseModel): + """Audio configuration for input and output. + + Parameters: + input: Configuration for input audio. + output: Configuration for output audio. + """ + + input: Optional[AudioInput] = None + output: Optional[AudioOutput] = None + + +class SessionProperties(BaseModel): + """Configuration properties for an OpenAI Realtime session. + + Parameters: + type: The type of session, always "realtime". + object: Object type identifier, always "realtime.session". + id: Unique identifier for the session. + model: The Realtime model used for this session. + output_modalities: The set of modalities the model can respond with. + instructions: System instructions for the assistant. + audio: Configuration for input and output audio. + tools: Available function tools for the assistant. + tool_choice: Tool usage strategy ("auto", "none", or "required"). + max_output_tokens: Maximum tokens in response or "inf" for unlimited. + tracing: Configuration options for tracing. + prompt: Reference to a prompt template and its variables. + expires_at: Session expiration timestamp. + include: Additional fields to include in server outputs. + """ + + type: Optional[Literal["realtime"]] = "realtime" + object: Optional[Literal["realtime.session"]] = None + id: Optional[str] = None + model: Optional[str] = None + output_modalities: Optional[List[Literal["text", "audio"]]] = None + instructions: Optional[str] = None + audio: Optional[AudioConfiguration] = None + tools: Optional[List[Dict]] = None + tool_choice: Optional[Literal["auto", "none", "required"]] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + tracing: Optional[Union[Literal["auto"], Dict]] = None + prompt: Optional[Dict] = None + expires_at: Optional[int] = None + include: Optional[List[str]] = None + + +# +# context +# + + +class ItemContent(BaseModel): + """Content within a conversation item. + + Parameters: + type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). + text: Text content for text-based items. + audio: Base64-encoded audio data for audio items. + transcript: Transcribed text for audio items. + """ + + type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] + text: Optional[str] = None + audio: Optional[str] = None # base64-encoded audio + transcript: Optional[str] = None + + +class ConversationItem(BaseModel): + """A conversation item in the realtime session. + + Parameters: + id: Unique identifier for the item, auto-generated if not provided. + object: Object type identifier for the realtime API. + type: Item type (message, function_call, or function_call_output). + status: Current status of the item. + role: Speaker role for message items (user, assistant, or system). + content: Content list for message items. + call_id: Function call identifier for function_call items. + name: Function name for function_call items. + arguments: Function arguments as JSON string for function_call items. + output: Function output as JSON string for function_call_output items. + """ + + id: str = Field(default_factory=lambda: str(uuid.uuid4().hex)) + object: Optional[Literal["realtime.item"]] = None + type: Literal["message", "function_call", "function_call_output"] + status: Optional[Literal["completed", "in_progress", "incomplete"]] = None + # role and content are present for message items + role: Optional[Literal["user", "assistant", "system"]] = None + content: Optional[List[ItemContent]] = None + # these four fields are present for function_call items + call_id: Optional[str] = None + name: Optional[str] = None + arguments: Optional[str] = None + output: Optional[str] = None + + +class RealtimeConversation(BaseModel): + """A realtime conversation session. + + Parameters: + id: Unique identifier for the conversation. + object: Object type identifier, always "realtime.conversation". + """ + + id: str + object: Literal["realtime.conversation"] + + +class ResponseProperties(BaseModel): + """Properties for configuring assistant responses. + + Parameters: + output_modalities: Output modalities for the response. Must be either ["text"] or ["audio"]. Defaults to ["audio"]. + instructions: Specific instructions for this response. + audio: Audio configuration for this response. + tools: Available tools for this response. + tool_choice: Tool usage strategy for this response. + temperature: Sampling temperature for this response. + max_output_tokens: Maximum tokens for this response. + """ + + output_modalities: Optional[List[Literal["text", "audio"]]] = ["audio"] + instructions: Optional[str] = None + audio: Optional[AudioConfiguration] = None + tools: Optional[List[Dict]] = None + tool_choice: Optional[Literal["auto", "none", "required"]] = None + temperature: Optional[float] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + + +# +# error class +# +class RealtimeError(BaseModel): + """Error information from the realtime API. + + Parameters: + type: Error type identifier. + code: Specific error code. + message: Human-readable error message. + param: Parameter name that caused the error, if applicable. + event_id: Event ID associated with the error, if applicable. + """ + + type: str + code: Optional[str] = "" + message: str + param: Optional[str] = None + event_id: Optional[str] = None + + +# +# client events +# + + +class ClientEvent(BaseModel): + """Base class for client events sent to the realtime API. + + Parameters: + event_id: Unique identifier for the event, auto-generated if not provided. + """ + + event_id: str = Field(default_factory=lambda: str(uuid.uuid4())) + + +class SessionUpdateEvent(ClientEvent): + """Event to update session properties. + + Parameters: + type: Event type, always "session.update". + session: Updated session properties. + """ + + type: Literal["session.update"] = "session.update" + session: SessionProperties + + def model_dump(self, *args, **kwargs) -> Dict[str, Any]: + """Serialize the event to a dictionary. + + Handles special serialization for turn_detection where False becomes null. + + Args: + *args: Positional arguments passed to parent model_dump. + **kwargs: Keyword arguments passed to parent model_dump. + + Returns: + Dictionary representation of the event. + """ + dump = super().model_dump(*args, **kwargs) + + # Handle turn_detection in audio.input so that False becomes null + if "audio" in dump["session"] and dump["session"]["audio"]: + if "input" in dump["session"]["audio"] and dump["session"]["audio"]["input"]: + if "turn_detection" in dump["session"]["audio"]["input"]: + if dump["session"]["audio"]["input"]["turn_detection"] is False: + dump["session"]["audio"]["input"]["turn_detection"] = None + + return dump + + +class InputAudioBufferAppendEvent(ClientEvent): + """Event to append audio data to the input buffer. + + Parameters: + type: Event type, always "input_audio_buffer.append". + audio: Base64-encoded audio data to append. + """ + + type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" + audio: str # base64-encoded audio + + +class InputAudioBufferCommitEvent(ClientEvent): + """Event to commit the current input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.commit". + """ + + type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" + + +class InputAudioBufferClearEvent(ClientEvent): + """Event to clear the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.clear". + """ + + type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" + + +class ConversationItemCreateEvent(ClientEvent): + """Event to create a new conversation item. + + Parameters: + type: Event type, always "conversation.item.create". + previous_item_id: ID of the item to insert after, if any. + item: The conversation item to create. + """ + + type: Literal["conversation.item.create"] = "conversation.item.create" + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemTruncateEvent(ClientEvent): + """Event to truncate a conversation item's audio content. + + Parameters: + type: Event type, always "conversation.item.truncate". + item_id: ID of the item to truncate. + content_index: Index of the content to truncate within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + + type: Literal["conversation.item.truncate"] = "conversation.item.truncate" + item_id: str + content_index: int + audio_end_ms: int + + +class ConversationItemDeleteEvent(ClientEvent): + """Event to delete a conversation item. + + Parameters: + type: Event type, always "conversation.item.delete". + item_id: ID of the item to delete. + """ + + type: Literal["conversation.item.delete"] = "conversation.item.delete" + item_id: str + + +class ConversationItemRetrieveEvent(ClientEvent): + """Event to retrieve a conversation item by ID. + + Parameters: + type: Event type, always "conversation.item.retrieve". + item_id: ID of the item to retrieve. + """ + + type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" + item_id: str + + +class ResponseCreateEvent(ClientEvent): + """Event to create a new assistant response. + + Parameters: + type: Event type, always "response.create". + response: Optional response configuration properties. + """ + + type: Literal["response.create"] = "response.create" + response: Optional[ResponseProperties] = None + + +class ResponseCancelEvent(ClientEvent): + """Event to cancel the current assistant response. + + Parameters: + type: Event type, always "response.cancel". + """ + + type: Literal["response.cancel"] = "response.cancel" + + +# +# server events +# + + +class ServerEvent(BaseModel): + """Base class for server events received from the realtime API. + + Parameters: + event_id: Unique identifier for the event. + type: Type of the server event. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + event_id: str + type: str + + +class SessionCreatedEvent(ServerEvent): + """Event indicating a session has been created. + + Parameters: + type: Event type, always "session.created". + session: The created session properties. + """ + + type: Literal["session.created"] + session: SessionProperties + + +class SessionUpdatedEvent(ServerEvent): + """Event indicating a session has been updated. + + Parameters: + type: Event type, always "session.updated". + session: The updated session properties. + """ + + type: Literal["session.updated"] + session: SessionProperties + + +class ConversationCreated(ServerEvent): + """Event indicating a conversation has been created. + + Parameters: + type: Event type, always "conversation.created". + conversation: The created conversation. + """ + + type: Literal["conversation.created"] + conversation: RealtimeConversation + + +class ConversationItemAdded(ServerEvent): + """Event indicating a conversation item has been added. + + Parameters: + type: Event type, always "conversation.item.added". + previous_item_id: ID of the previous item, if any. + item: The added conversation item. + """ + + type: Literal["conversation.item.added"] + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemDone(ServerEvent): + """Event indicating a conversation item is done processing. + + Parameters: + type: Event type, always "conversation.item.done". + previous_item_id: ID of the previous item, if any. + item: The completed conversation item. + """ + + type: Literal["conversation.item.done"] + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemInputAudioTranscriptionDelta(ServerEvent): + """Event containing incremental input audio transcription. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.delta". + item_id: ID of the conversation item being transcribed. + content_index: Index of the content within the item. + delta: Incremental transcription text. + """ + + type: Literal["conversation.item.input_audio_transcription.delta"] + item_id: str + content_index: int + delta: str + + +class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): + """Event indicating input audio transcription is complete. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.completed". + item_id: ID of the conversation item that was transcribed. + content_index: Index of the content within the item. + transcript: Complete transcription text. + """ + + type: Literal["conversation.item.input_audio_transcription.completed"] + item_id: str + content_index: int + transcript: str + + +class ConversationItemInputAudioTranscriptionFailed(ServerEvent): + """Event indicating input audio transcription failed. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.failed". + item_id: ID of the conversation item that failed transcription. + content_index: Index of the content within the item. + error: Error details for the transcription failure. + """ + + type: Literal["conversation.item.input_audio_transcription.failed"] + item_id: str + content_index: int + error: RealtimeError + + +class ConversationItemTruncated(ServerEvent): + """Event indicating a conversation item has been truncated. + + Parameters: + type: Event type, always "conversation.item.truncated". + item_id: ID of the truncated conversation item. + content_index: Index of the content within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + + type: Literal["conversation.item.truncated"] + item_id: str + content_index: int + audio_end_ms: int + + +class ConversationItemDeleted(ServerEvent): + """Event indicating a conversation item has been deleted. + + Parameters: + type: Event type, always "conversation.item.deleted". + item_id: ID of the deleted conversation item. + """ + + type: Literal["conversation.item.deleted"] + item_id: str + + +class ConversationItemRetrieved(ServerEvent): + """Event containing a retrieved conversation item. + + Parameters: + type: Event type, always "conversation.item.retrieved". + item: The retrieved conversation item. + """ + + type: Literal["conversation.item.retrieved"] + item: ConversationItem + + +class ResponseCreated(ServerEvent): + """Event indicating an assistant response has been created. + + Parameters: + type: Event type, always "response.created". + response: The created response object. + """ + + type: Literal["response.created"] + response: "Response" + + +class ResponseDone(ServerEvent): + """Event indicating an assistant response is complete. + + Parameters: + type: Event type, always "response.done". + response: The completed response object. + """ + + type: Literal["response.done"] + response: "Response" + + +class ResponseOutputItemAdded(ServerEvent): + """Event indicating an output item has been added to a response. + + Parameters: + type: Event type, always "response.output_item.added". + response_id: ID of the response. + output_index: Index of the output item. + item: The added conversation item. + """ + + type: Literal["response.output_item.added"] + response_id: str + output_index: int + item: ConversationItem + + +class ResponseOutputItemDone(ServerEvent): + """Event indicating an output item is complete. + + Parameters: + type: Event type, always "response.output_item.done". + response_id: ID of the response. + output_index: Index of the output item. + item: The completed conversation item. + """ + + type: Literal["response.output_item.done"] + response_id: str + output_index: int + item: ConversationItem + + +class ResponseContentPartAdded(ServerEvent): + """Event indicating a content part has been added to a response. + + Parameters: + type: Event type, always "response.content_part.added". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The added content part. + """ + + type: Literal["response.content_part.added"] + response_id: str + item_id: str + output_index: int + content_index: int + part: ItemContent + + +class ResponseContentPartDone(ServerEvent): + """Event indicating a content part is complete. + + Parameters: + type: Event type, always "response.content_part.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The completed content part. + """ + + type: Literal["response.content_part.done"] + response_id: str + item_id: str + output_index: int + content_index: int + part: ItemContent + + +class ResponseTextDelta(ServerEvent): + """Event containing incremental text from a response. + + Parameters: + type: Event type, always "response.output_text.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental text content. + """ + + type: Literal["response.output_text.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseTextDone(ServerEvent): + """Event indicating text content is complete. + + Parameters: + type: Event type, always "response.output_text.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + text: Complete text content. + """ + + type: Literal["response.output_text.done"] + response_id: str + item_id: str + output_index: int + content_index: int + text: str + + +class ResponseAudioTranscriptDelta(ServerEvent): + """Event containing incremental audio transcript from a response. + + Parameters: + type: Event type, always "response.output_audio_transcript.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental transcript text. + """ + + type: Literal["response.output_audio_transcript.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseAudioTranscriptDone(ServerEvent): + """Event indicating audio transcript is complete. + + Parameters: + type: Event type, always "response.output_audio_transcript.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + transcript: Complete transcript text. + """ + + type: Literal["response.output_audio_transcript.done"] + response_id: str + item_id: str + output_index: int + content_index: int + transcript: str + + +class ResponseAudioDelta(ServerEvent): + """Event containing incremental audio data from a response. + + Parameters: + type: Event type, always "response.output_audio.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Base64-encoded incremental audio data. + """ + + type: Literal["response.output_audio.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str # base64-encoded audio + + +class ResponseAudioDone(ServerEvent): + """Event indicating audio content is complete. + + Parameters: + type: Event type, always "response.output_audio.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + """ + + type: Literal["response.output_audio.done"] + response_id: str + item_id: str + output_index: int + content_index: int + + +class ResponseFunctionCallArgumentsDelta(ServerEvent): + """Event containing incremental function call arguments. + + Parameters: + type: Event type, always "response.function_call_arguments.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + delta: Incremental function arguments as JSON. + """ + + type: Literal["response.function_call_arguments.delta"] + response_id: str + item_id: str + output_index: int + call_id: str + delta: str + + +class ResponseFunctionCallArgumentsDone(ServerEvent): + """Event indicating function call arguments are complete. + + Parameters: + type: Event type, always "response.function_call_arguments.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + arguments: Complete function arguments as JSON string. + """ + + type: Literal["response.function_call_arguments.done"] + response_id: str + item_id: str + output_index: int + call_id: str + arguments: str + + +class InputAudioBufferSpeechStarted(ServerEvent): + """Event indicating speech has started in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_started". + audio_start_ms: Start time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + + type: Literal["input_audio_buffer.speech_started"] + audio_start_ms: int + item_id: str + + +class InputAudioBufferSpeechStopped(ServerEvent): + """Event indicating speech has stopped in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_stopped". + audio_end_ms: End time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + + type: Literal["input_audio_buffer.speech_stopped"] + audio_end_ms: int + item_id: str + + +class InputAudioBufferCommitted(ServerEvent): + """Event indicating the input audio buffer has been committed. + + Parameters: + type: Event type, always "input_audio_buffer.committed". + previous_item_id: ID of the previous item, if any. + item_id: ID of the committed conversation item. + """ + + type: Literal["input_audio_buffer.committed"] + previous_item_id: Optional[str] = None + item_id: str + + +class InputAudioBufferCleared(ServerEvent): + """Event indicating the input audio buffer has been cleared. + + Parameters: + type: Event type, always "input_audio_buffer.cleared". + """ + + type: Literal["input_audio_buffer.cleared"] + + +class ErrorEvent(ServerEvent): + """Event indicating an error occurred. + + Parameters: + type: Event type, always "error". + error: Error details. + """ + + type: Literal["error"] + error: RealtimeError + + +class RateLimitsUpdated(ServerEvent): + """Event indicating rate limits have been updated. + + Parameters: + type: Event type, always "rate_limits.updated". + rate_limits: List of rate limit information. + """ + + type: Literal["rate_limits.updated"] + rate_limits: List[Dict[str, Any]] + + +class CachedTokensDetails(BaseModel): + """Details about cached tokens. + + Parameters: + text_tokens: Number of cached text tokens. + audio_tokens: Number of cached audio tokens. + """ + + text_tokens: Optional[int] = 0 + audio_tokens: Optional[int] = 0 + + +class TokenDetails(BaseModel): + """Detailed token usage information. + + Parameters: + cached_tokens: Number of cached tokens used. Defaults to 0. + text_tokens: Number of text tokens used. Defaults to 0. + audio_tokens: Number of audio tokens used. Defaults to 0. + cached_tokens_details: Detailed breakdown of cached tokens. + image_tokens: Number of image tokens used (for input only). + """ + + cached_tokens: Optional[int] = 0 + text_tokens: Optional[int] = 0 + audio_tokens: Optional[int] = 0 + cached_tokens_details: Optional[CachedTokensDetails] = None + image_tokens: Optional[int] = 0 + + class Config: + """Pydantic configuration for TokenDetails.""" + + extra = "allow" + + +class Usage(BaseModel): + """Token usage statistics for a response. + + Parameters: + total_tokens: Total number of tokens used. + input_tokens: Number of input tokens used. + output_tokens: Number of output tokens used. + input_token_details: Detailed breakdown of input token usage. + output_token_details: Detailed breakdown of output token usage. + """ + + total_tokens: int + input_tokens: int + output_tokens: int + input_token_details: TokenDetails + output_token_details: TokenDetails + + +class Response(BaseModel): + """A complete assistant response. + + Parameters: + id: Unique identifier for the response. + object: Object type, always "realtime.response". + status: Current status of the response. + status_details: Additional status information. + output: List of conversation items in the response. + conversation_id: Which conversation the response is added to. + output_modalities: The set of modalities the model used to respond. + max_output_tokens: Maximum number of output tokens used. + audio: Audio configuration for the response. + usage: Token usage statistics for the response. + voice: The voice the model used to respond. + temperature: Sampling temperature used for the response. + output_audio_format: The format of output audio. + """ + + id: str + object: Literal["realtime.response"] + status: Literal["completed", "in_progress", "incomplete", "cancelled", "failed"] + status_details: Any + output: List[ConversationItem] + output_modalities: Optional[List[Literal["text", "audio"]]] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + audio: Optional[AudioConfiguration] = None + usage: Optional[Usage] = None + voice: Optional[str] = None + temperature: Optional[float] = None + output_audio_format: Optional[str] = None + + +_server_event_types = { + "error": ErrorEvent, + "session.created": SessionCreatedEvent, + "session.updated": SessionUpdatedEvent, + "conversation.created": ConversationCreated, + "input_audio_buffer.committed": InputAudioBufferCommitted, + "input_audio_buffer.cleared": InputAudioBufferCleared, + "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, + "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, + "conversation.item.added": ConversationItemAdded, + "conversation.item.done": ConversationItemDone, + "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, + "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, + "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, + "conversation.item.truncated": ConversationItemTruncated, + "conversation.item.deleted": ConversationItemDeleted, + "conversation.item.retrieved": ConversationItemRetrieved, + "response.created": ResponseCreated, + "response.done": ResponseDone, + "response.output_item.added": ResponseOutputItemAdded, + "response.output_item.done": ResponseOutputItemDone, + "response.content_part.added": ResponseContentPartAdded, + "response.content_part.done": ResponseContentPartDone, + "response.output_text.delta": ResponseTextDelta, + "response.output_text.done": ResponseTextDone, + "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta, + "response.output_audio_transcript.done": ResponseAudioTranscriptDone, + "response.output_audio.delta": ResponseAudioDelta, + "response.output_audio.done": ResponseAudioDone, + "response.function_call_arguments.delta": ResponseFunctionCallArgumentsDelta, + "response.function_call_arguments.done": ResponseFunctionCallArgumentsDone, + "rate_limits.updated": RateLimitsUpdated, +} + + +def parse_server_event(str): + """Parse a server event from JSON string. + + Args: + str: JSON string containing the server event. + + Returns: + Parsed server event object of the appropriate type. + + Raises: + Exception: If the event type is unimplemented or parsing fails. + """ + try: + event = json.loads(str) + event_type = event["type"] + if event_type not in _server_event_types: + raise Exception(f"Unimplemented server event type: {event_type}") + return _server_event_types[event_type].model_validate(event) + except Exception as e: + raise Exception(f"{e} \n\n{str}") diff --git a/src/pipecat/services/openai/realtime/frames.py b/src/pipecat/services/openai/realtime/frames.py new file mode 100644 index 000000000..8617c6efd --- /dev/null +++ b/src/pipecat/services/openai/realtime/frames.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Custom frame types for OpenAI Realtime API integration.""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from pipecat.frames.frames import DataFrame, FunctionCallResultFrame + +if TYPE_CHECKING: + from pipecat.services.openai.realtime.context import OpenAIRealtimeLLMContext + + +@dataclass +class RealtimeMessagesUpdateFrame(DataFrame): + """Frame indicating that the realtime context messages have been updated. + + Parameters: + context: The updated OpenAI realtime LLM context. + """ + + context: "OpenAIRealtimeLLMContext" + + +@dataclass +class RealtimeFunctionCallResultFrame(DataFrame): + """Frame containing function call results for the realtime service. + + Parameters: + result_frame: The function call result frame to send to the realtime API. + """ + + result_frame: FunctionCallResultFrame diff --git a/src/pipecat/services/openai_realtime/openai.py b/src/pipecat/services/openai/realtime/llm.py similarity index 100% rename from src/pipecat/services/openai_realtime/openai.py rename to src/pipecat/services/openai/realtime/llm.py diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index a67392caf..cdf0d11ac 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -14,6 +14,7 @@ from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger from openai import AsyncOpenAI, BadRequestError +from pydantic import BaseModel from pipecat.frames.frames import ( ErrorFrame, @@ -55,6 +56,17 @@ class OpenAITTSService(TTSService): OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz + class InputParams(BaseModel): + """Input parameters for OpenAI TTS configuration. + + Parameters: + instructions: Instructions to guide voice synthesis behavior. + speed: Voice speed control (0.25 to 4.0, default 1.0). + """ + + instructions: Optional[str] = None + speed: Optional[float] = None + def __init__( self, *, @@ -65,6 +77,7 @@ class OpenAITTSService(TTSService): sample_rate: Optional[int] = None, instructions: Optional[str] = None, speed: Optional[float] = None, + params: Optional[InputParams] = None, **kwargs, ): """Initialize OpenAI TTS service. @@ -77,7 +90,11 @@ class OpenAITTSService(TTSService): sample_rate: Output audio sample rate in Hz. If None, uses OpenAI's default 24kHz. instructions: Optional instructions to guide voice synthesis behavior. speed: Voice speed control (0.25 to 4.0, default 1.0). + params: Optional synthesis controls (acting instructions, speed, ...). **kwargs: Additional keyword arguments passed to TTSService. + + .. deprecated:: 0.0.91 + The `instructions` and `speed` parameters are deprecated, use `InputParams` instead. """ if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE: logger.warning( @@ -86,12 +103,26 @@ class OpenAITTSService(TTSService): ) super().__init__(sample_rate=sample_rate, **kwargs) - self._speed = speed self.set_model_name(model) self.set_voice(voice) - self._instructions = instructions self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) + if instructions or speed: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "The `instructions` and `speed` parameters are deprecated, use `InputParams` instead.", + DeprecationWarning, + stacklevel=2, + ) + + self._settings = { + "instructions": params.instructions if params else instructions, + "speed": params.speed if params else speed, + } + def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -144,11 +175,11 @@ class OpenAITTSService(TTSService): "response_format": "pcm", } - if self._instructions: - create_params["instructions"] = self._instructions + if self._settings["instructions"]: + create_params["instructions"] = self._settings["instructions"] - if self._speed: - create_params["speed"] = self._speed + if self._settings["speed"]: + create_params["speed"] = self._settings["speed"] async with self._client.audio.speech.with_streaming_response.create( **create_params diff --git a/src/pipecat/services/openai_realtime/__init__.py b/src/pipecat/services/openai_realtime/__init__.py index 6f3154f1c..f1fbefeda 100644 --- a/src/pipecat/services/openai_realtime/__init__.py +++ b/src/pipecat/services/openai_realtime/__init__.py @@ -1,9 +1,27 @@ -from .azure import AzureRealtimeLLMService -from .events import ( +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import warnings + +from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService +from pipecat.services.openai.realtime.events import ( InputAudioNoiseReduction, InputAudioTranscription, SemanticTurnDetection, SessionProperties, TurnDetection, ) -from .openai import OpenAIRealtimeLLMService +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/azure.py b/src/pipecat/services/openai_realtime/azure.py index aed53b94c..dae6ef496 100644 --- a/src/pipecat/services/openai_realtime/azure.py +++ b/src/pipecat/services/openai_realtime/azure.py @@ -1,67 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Azure OpenAI Realtime LLM service implementation.""" -from loguru import logger +import warnings -from .openai import OpenAIRealtimeLLMService +from pipecat.services.azure.realtime.llm import * -try: - from websockets.asyncio.client import connect as websocket_connect -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.azure are deprecated. " + "Please use the equivalent types from " + "pipecat.services.azure.realtime.llm instead.", + DeprecationWarning, + stacklevel=2, ) - raise Exception(f"Missing module: {e}") - - -class AzureRealtimeLLMService(OpenAIRealtimeLLMService): - """Azure OpenAI Realtime LLM service with Azure-specific authentication. - - Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, - using Azure's authentication headers and endpoint format. Provides the same - real-time audio and text communication capabilities as the base OpenAI service. - """ - - def __init__( - self, - *, - api_key: str, - base_url: str, - **kwargs, - ): - """Initialize Azure Realtime LLM service. - - Args: - api_key: The API key for the Azure OpenAI service. - base_url: The full Azure WebSocket endpoint URL including api-version and deployment. - Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" - **kwargs: Additional arguments passed to parent OpenAIRealtimeLLMService. - """ - super().__init__(base_url=base_url, api_key=api_key, **kwargs) - self.api_key = api_key - self.base_url = base_url - - async def _connect(self): - try: - if self._websocket: - # Here we assume that if we have a websocket, we are connected. We - # handle disconnections in the send/recv code paths. - return - - logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") - self._websocket = await websocket_connect( - uri=self.base_url, - additional_headers={ - "api-key": self.api_key, - }, - ) - self._receive_task = self.create_task(self._receive_task_handler()) - except Exception as e: - logger.error(f"{self} initialization error: {e}") - self._websocket = None diff --git a/src/pipecat/services/openai_realtime/context.py b/src/pipecat/services/openai_realtime/context.py index cb1c0a9f5..58f1cfe75 100644 --- a/src/pipecat/services/openai_realtime/context.py +++ b/src/pipecat/services/openai_realtime/context.py @@ -1,272 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """OpenAI Realtime LLM context and aggregator implementations.""" -import copy -import json +import warnings -from loguru import logger +from pipecat.services.openai.realtime.context import * -from pipecat.frames.frames import ( - Frame, - FunctionCallResultFrame, - InterimTranscriptionFrame, - LLMMessagesUpdateFrame, - LLMSetToolsFrame, - LLMTextFrame, - TranscriptionFrame, -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) - -from . import events -from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame - - -class OpenAIRealtimeLLMContext(OpenAILLMContext): - """OpenAI Realtime LLM context with session management and message conversion. - - Extends the standard OpenAI LLM context to support real-time session properties, - instruction management, and conversion between standard message formats and - realtime conversation items. - """ - - def __init__(self, messages=None, tools=None, **kwargs): - """Initialize the OpenAIRealtimeLLMContext. - - Args: - messages: Initial conversation messages. Defaults to None. - tools: Available function tools. Defaults to None. - **kwargs: Additional arguments passed to parent OpenAILLMContext. - """ - super().__init__(messages=messages, tools=tools, **kwargs) - self.__setup_local() - - def __setup_local(self): - self.llm_needs_settings_update = True - self.llm_needs_initial_messages = True - self._session_instructions = "" - - return - - @staticmethod - def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": - """Upgrade a standard OpenAI LLM context to a realtime context. - - Args: - obj: The OpenAILLMContext instance to upgrade. - - Returns: - The upgraded OpenAIRealtimeLLMContext instance. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): - obj.__class__ = OpenAIRealtimeLLMContext - obj.__setup_local() - return obj - - # todo - # - finish implementing all frames - - def from_standard_message(self, message): - """Convert a standard message format to a realtime conversation item. - - Args: - message: The standard message dictionary to convert. - - Returns: - A ConversationItem instance for the realtime API. - """ - if message.get("role") == "user": - content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): - if c.get("type") == "text": - content += " " + c.get("text") - else: - logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" - ) - return events.ConversationItem( - role="user", - type="message", - content=[events.ItemContent(type="input_text", text=content)], - ) - if message.get("role") == "assistant" and message.get("tool_calls"): - tc = message.get("tool_calls")[0] - return events.ConversationItem( - type="function_call", - call_id=tc["id"], - name=tc["function"]["name"], - arguments=tc["function"]["arguments"], - ) - logger.error(f"Unhandled message type in from_standard_message: {message}") - - def get_messages_for_initializing_history(self): - """Get conversation items for initializing the realtime session history. - - Converts the context's messages to a format suitable for the realtime API, - handling system instructions and conversation history packaging. - - Returns: - List of conversation items for session initialization. - """ - # We can't load a long conversation history into the openai realtime api yet. (The API/model - # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So - # our general strategy until this is fixed is just to put everything into a first "user" - # message as a single input. - if not self.messages: - return [] - - messages = copy.deepcopy(self.messages) - - # If we have a "system" message as our first message, let's pull that out into session - # "instructions" - if messages[0].get("role") == "system": - self.llm_needs_settings_update = True - system = messages.pop(0) - content = system.get("content") - if isinstance(content, str): - self._session_instructions = content - elif isinstance(content, list): - self._session_instructions = content[0].get("text") - if not messages: - return [] - - # If we have just a single "user" item, we can just send it normally - if len(messages) == 1 and messages[0].get("role") == "user": - return [self.from_standard_message(messages[0])] - - # Otherwise, let's pack everything into a single "user" message with a bit of - # explanation for the LLM - intro_text = """ - This is a previously saved conversation. Please treat this conversation history as a - starting point for the current conversation.""" - - trailing_text = """ - This is the end of the previously saved conversation. Please continue the conversation - from here. If the last message is a user instruction or question, act on that instruction - or answer the question. If the last message is an assistant response, simple say that you - are ready to continue the conversation.""" - - return [ - { - "role": "user", - "type": "message", - "content": [ - { - "type": "input_text", - "text": "\n\n".join( - [intro_text, json.dumps(messages, indent=2), trailing_text] - ), - } - ], - } - ] - - def add_user_content_item_as_message(self, item): - """Add a user content item as a standard message to the context. - - Args: - item: The conversation item to add as a user message. - """ - message = { - "role": "user", - "content": [{"type": "text", "text": item.content[0].transcript}], - } - self.add_message(message) - - -class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): - """User context aggregator for OpenAI Realtime API. - - Handles user input frames and generates appropriate context updates - for the realtime conversation, including message updates and tool settings. - - Args: - context: The OpenAI realtime LLM context. - **kwargs: Additional arguments passed to parent aggregator. - """ - - async def process_frame( - self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM - ): - """Process incoming frames and handle realtime-specific frame types. - - Args: - frame: The frame to process. - direction: The direction of frame flow in the pipeline. - """ - await super().process_frame(frame, direction) - # Parent does not push LLMMessagesUpdateFrame. This ensures that in a typical pipeline, - # messages are only processed by the user context aggregator, which is generally what we want. But - # we also need to send new messages over the websocket, so the openai realtime API has them - # in its context. - if isinstance(frame, LLMMessagesUpdateFrame): - await self.push_frame(RealtimeMessagesUpdateFrame(context=self._context)) - - # Parent also doesn't push the LLMSetToolsFrame. - if isinstance(frame, LLMSetToolsFrame): - await self.push_frame(frame, direction) - - async def push_aggregation(self): - """Push user input aggregation. - - Currently ignores all user input coming into the pipeline as realtime - audio input is handled directly by the service. - """ - # for the moment, ignore all user input coming into the pipeline. - # todo: think about whether/how to fix this to allow for text input from - # upstream (transport/transcription, or other sources) - pass - - -class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Assistant context aggregator for OpenAI Realtime API. - - Handles assistant output frames from the realtime service, filtering - out duplicate text frames and managing function call results. - - Args: - context: The OpenAI realtime LLM context. - **kwargs: Additional arguments passed to parent aggregator. - """ - - # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, - # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We - # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames - # are process. This ensures that the context gets only one set of messages. - # OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames, - # so we need to ignore pushing those as well, as they're also TextFrames. - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process assistant frames, filtering out duplicate text content. - - Args: - frame: The frame to process. - direction: The direction of frame flow in the pipeline. - """ - if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)): - await super().process_frame(frame, direction) - - async def handle_function_call_result(self, frame: FunctionCallResultFrame): - """Handle function call result and notify the realtime service. - - Args: - frame: The function call result frame to handle. - """ - await super().handle_function_call_result(frame) - - # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, - # so we didn't have a chance to add the result to the openai realtime api context. Let's push a - # special frame to do that. - await self.push_frame( - RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.context are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.context instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/events.py b/src/pipecat/services/openai_realtime/events.py index 200e59d68..53b4b0dff 100644 --- a/src/pipecat/services/openai_realtime/events.py +++ b/src/pipecat/services/openai_realtime/events.py @@ -1,1106 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Event models and data structures for OpenAI Realtime API communication.""" -import json -import uuid -from typing import Any, Dict, List, Literal, Optional, Union - -from pydantic import BaseModel, ConfigDict, Field - -# -# session properties -# - - -class AudioFormat(BaseModel): - """Base class for audio format configuration.""" - - type: str - - -class PCMAudioFormat(AudioFormat): - """PCM audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcm". - rate: Sample rate, always 24000 for PCM. - """ - - type: Literal["audio/pcm"] = "audio/pcm" - rate: Literal[24000] = 24000 - - -class PCMUAudioFormat(AudioFormat): - """PCMU (G.711 ΞΌ-law) audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcmu". - """ - - type: Literal["audio/pcmu"] = "audio/pcmu" - - -class PCMAAudioFormat(AudioFormat): - """PCMA (G.711 A-law) audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcma". - """ - - type: Literal["audio/pcma"] = "audio/pcma" - - -class InputAudioTranscription(BaseModel): - """Configuration for audio transcription settings.""" - - model: str = "gpt-4o-transcribe" - language: Optional[str] - prompt: Optional[str] - - def __init__( - self, - model: Optional[str] = "gpt-4o-transcribe", - language: Optional[str] = None, - prompt: Optional[str] = None, - ): - """Initialize InputAudioTranscription. - - Args: - model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). - language: Optional language code for transcription. - prompt: Optional transcription hint text. - """ - super().__init__(model=model, language=language, prompt=prompt) - - -class TurnDetection(BaseModel): - """Server-side voice activity detection configuration. - - Parameters: - type: Detection type, must be "server_vad". - threshold: Voice activity detection threshold (0.0-1.0). Defaults to 0.5. - prefix_padding_ms: Padding before speech starts in milliseconds. Defaults to 300. - silence_duration_ms: Silence duration to detect speech end in milliseconds. Defaults to 500. - """ - - type: Optional[Literal["server_vad"]] = "server_vad" - threshold: Optional[float] = 0.5 - prefix_padding_ms: Optional[int] = 300 - silence_duration_ms: Optional[int] = 500 - - -class SemanticTurnDetection(BaseModel): - """Semantic-based turn detection configuration. - - Parameters: - type: Detection type, must be "semantic_vad". - eagerness: Turn detection eagerness level. Can be "low", "medium", "high", or "auto". - create_response: Whether to automatically create responses on turn detection. - interrupt_response: Whether to interrupt ongoing responses on turn detection. - """ - - type: Optional[Literal["semantic_vad"]] = "semantic_vad" - eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None - create_response: Optional[bool] = None - interrupt_response: Optional[bool] = None - - -class InputAudioNoiseReduction(BaseModel): - """Input audio noise reduction configuration. - - Parameters: - type: Noise reduction type for different microphone scenarios. - """ - - type: Optional[Literal["near_field", "far_field"]] - - -class AudioInput(BaseModel): - """Audio input configuration. - - Parameters: - format: The format of the input audio. - transcription: Configuration for input audio transcription. - noise_reduction: Configuration for input audio noise reduction. - turn_detection: Configuration for turn detection, or False to disable. - """ - - format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None - transcription: Optional[InputAudioTranscription] = None - noise_reduction: Optional[InputAudioNoiseReduction] = None - turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = None - - -class AudioOutput(BaseModel): - """Audio output configuration. - - Parameters: - format: The format of the output audio. - voice: The voice the model uses to respond. - speed: The speed of the model's spoken response. - """ - - format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None - voice: Optional[str] = None - speed: Optional[float] = None - - -class AudioConfiguration(BaseModel): - """Audio configuration for input and output. - - Parameters: - input: Configuration for input audio. - output: Configuration for output audio. - """ - - input: Optional[AudioInput] = None - output: Optional[AudioOutput] = None - - -class SessionProperties(BaseModel): - """Configuration properties for an OpenAI Realtime session. - - Parameters: - type: The type of session, always "realtime". - object: Object type identifier, always "realtime.session". - id: Unique identifier for the session. - model: The Realtime model used for this session. - output_modalities: The set of modalities the model can respond with. - instructions: System instructions for the assistant. - audio: Configuration for input and output audio. - tools: Available function tools for the assistant. - tool_choice: Tool usage strategy ("auto", "none", or "required"). - max_output_tokens: Maximum tokens in response or "inf" for unlimited. - tracing: Configuration options for tracing. - prompt: Reference to a prompt template and its variables. - expires_at: Session expiration timestamp. - include: Additional fields to include in server outputs. - """ - - type: Optional[Literal["realtime"]] = "realtime" - object: Optional[Literal["realtime.session"]] = None - id: Optional[str] = None - model: Optional[str] = None - output_modalities: Optional[List[Literal["text", "audio"]]] = None - instructions: Optional[str] = None - audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None - tool_choice: Optional[Literal["auto", "none", "required"]] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - tracing: Optional[Union[Literal["auto"], Dict]] = None - prompt: Optional[Dict] = None - expires_at: Optional[int] = None - include: Optional[List[str]] = None - - -# -# context -# - - -class ItemContent(BaseModel): - """Content within a conversation item. - - Parameters: - type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). - text: Text content for text-based items. - audio: Base64-encoded audio data for audio items. - transcript: Transcribed text for audio items. - """ - - type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] - text: Optional[str] = None - audio: Optional[str] = None # base64-encoded audio - transcript: Optional[str] = None - - -class ConversationItem(BaseModel): - """A conversation item in the realtime session. - - Parameters: - id: Unique identifier for the item, auto-generated if not provided. - object: Object type identifier for the realtime API. - type: Item type (message, function_call, or function_call_output). - status: Current status of the item. - role: Speaker role for message items (user, assistant, or system). - content: Content list for message items. - call_id: Function call identifier for function_call items. - name: Function name for function_call items. - arguments: Function arguments as JSON string for function_call items. - output: Function output as JSON string for function_call_output items. - """ - - id: str = Field(default_factory=lambda: str(uuid.uuid4().hex)) - object: Optional[Literal["realtime.item"]] = None - type: Literal["message", "function_call", "function_call_output"] - status: Optional[Literal["completed", "in_progress", "incomplete"]] = None - # role and content are present for message items - role: Optional[Literal["user", "assistant", "system"]] = None - content: Optional[List[ItemContent]] = None - # these four fields are present for function_call items - call_id: Optional[str] = None - name: Optional[str] = None - arguments: Optional[str] = None - output: Optional[str] = None - - -class RealtimeConversation(BaseModel): - """A realtime conversation session. - - Parameters: - id: Unique identifier for the conversation. - object: Object type identifier, always "realtime.conversation". - """ - - id: str - object: Literal["realtime.conversation"] - - -class ResponseProperties(BaseModel): - """Properties for configuring assistant responses. - - Parameters: - output_modalities: Output modalities for the response. Must be either ["text"] or ["audio"]. Defaults to ["audio"]. - instructions: Specific instructions for this response. - audio: Audio configuration for this response. - tools: Available tools for this response. - tool_choice: Tool usage strategy for this response. - temperature: Sampling temperature for this response. - max_output_tokens: Maximum tokens for this response. - """ - - output_modalities: Optional[List[Literal["text", "audio"]]] = ["audio"] - instructions: Optional[str] = None - audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None - tool_choice: Optional[Literal["auto", "none", "required"]] = None - temperature: Optional[float] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - - -# -# error class -# -class RealtimeError(BaseModel): - """Error information from the realtime API. - - Parameters: - type: Error type identifier. - code: Specific error code. - message: Human-readable error message. - param: Parameter name that caused the error, if applicable. - event_id: Event ID associated with the error, if applicable. - """ - - type: str - code: Optional[str] = "" - message: str - param: Optional[str] = None - event_id: Optional[str] = None - - -# -# client events -# - - -class ClientEvent(BaseModel): - """Base class for client events sent to the realtime API. - - Parameters: - event_id: Unique identifier for the event, auto-generated if not provided. - """ - - event_id: str = Field(default_factory=lambda: str(uuid.uuid4())) - - -class SessionUpdateEvent(ClientEvent): - """Event to update session properties. - - Parameters: - type: Event type, always "session.update". - session: Updated session properties. - """ - - type: Literal["session.update"] = "session.update" - session: SessionProperties - - def model_dump(self, *args, **kwargs) -> Dict[str, Any]: - """Serialize the event to a dictionary. - - Handles special serialization for turn_detection where False becomes null. - - Args: - *args: Positional arguments passed to parent model_dump. - **kwargs: Keyword arguments passed to parent model_dump. - - Returns: - Dictionary representation of the event. - """ - dump = super().model_dump(*args, **kwargs) - - # Handle turn_detection in audio.input so that False becomes null - if "audio" in dump["session"] and dump["session"]["audio"]: - if "input" in dump["session"]["audio"] and dump["session"]["audio"]["input"]: - if "turn_detection" in dump["session"]["audio"]["input"]: - if dump["session"]["audio"]["input"]["turn_detection"] is False: - dump["session"]["audio"]["input"]["turn_detection"] = None - - return dump - - -class InputAudioBufferAppendEvent(ClientEvent): - """Event to append audio data to the input buffer. - - Parameters: - type: Event type, always "input_audio_buffer.append". - audio: Base64-encoded audio data to append. - """ - - type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" - audio: str # base64-encoded audio - - -class InputAudioBufferCommitEvent(ClientEvent): - """Event to commit the current input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.commit". - """ - - type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" - - -class InputAudioBufferClearEvent(ClientEvent): - """Event to clear the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.clear". - """ - - type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" - - -class ConversationItemCreateEvent(ClientEvent): - """Event to create a new conversation item. - - Parameters: - type: Event type, always "conversation.item.create". - previous_item_id: ID of the item to insert after, if any. - item: The conversation item to create. - """ - - type: Literal["conversation.item.create"] = "conversation.item.create" - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemTruncateEvent(ClientEvent): - """Event to truncate a conversation item's audio content. - - Parameters: - type: Event type, always "conversation.item.truncate". - item_id: ID of the item to truncate. - content_index: Index of the content to truncate within the item. - audio_end_ms: End time in milliseconds for the truncated audio. - """ - - type: Literal["conversation.item.truncate"] = "conversation.item.truncate" - item_id: str - content_index: int - audio_end_ms: int - - -class ConversationItemDeleteEvent(ClientEvent): - """Event to delete a conversation item. - - Parameters: - type: Event type, always "conversation.item.delete". - item_id: ID of the item to delete. - """ - - type: Literal["conversation.item.delete"] = "conversation.item.delete" - item_id: str - - -class ConversationItemRetrieveEvent(ClientEvent): - """Event to retrieve a conversation item by ID. - - Parameters: - type: Event type, always "conversation.item.retrieve". - item_id: ID of the item to retrieve. - """ - - type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" - item_id: str - - -class ResponseCreateEvent(ClientEvent): - """Event to create a new assistant response. - - Parameters: - type: Event type, always "response.create". - response: Optional response configuration properties. - """ - - type: Literal["response.create"] = "response.create" - response: Optional[ResponseProperties] = None - - -class ResponseCancelEvent(ClientEvent): - """Event to cancel the current assistant response. - - Parameters: - type: Event type, always "response.cancel". - """ - - type: Literal["response.cancel"] = "response.cancel" - - -# -# server events -# - - -class ServerEvent(BaseModel): - """Base class for server events received from the realtime API. - - Parameters: - event_id: Unique identifier for the event. - type: Type of the server event. - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - - event_id: str - type: str - - -class SessionCreatedEvent(ServerEvent): - """Event indicating a session has been created. - - Parameters: - type: Event type, always "session.created". - session: The created session properties. - """ - - type: Literal["session.created"] - session: SessionProperties - - -class SessionUpdatedEvent(ServerEvent): - """Event indicating a session has been updated. - - Parameters: - type: Event type, always "session.updated". - session: The updated session properties. - """ - - type: Literal["session.updated"] - session: SessionProperties - - -class ConversationCreated(ServerEvent): - """Event indicating a conversation has been created. - - Parameters: - type: Event type, always "conversation.created". - conversation: The created conversation. - """ - - type: Literal["conversation.created"] - conversation: RealtimeConversation - - -class ConversationItemAdded(ServerEvent): - """Event indicating a conversation item has been added. - - Parameters: - type: Event type, always "conversation.item.added". - previous_item_id: ID of the previous item, if any. - item: The added conversation item. - """ - - type: Literal["conversation.item.added"] - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemDone(ServerEvent): - """Event indicating a conversation item is done processing. - - Parameters: - type: Event type, always "conversation.item.done". - previous_item_id: ID of the previous item, if any. - item: The completed conversation item. - """ - - type: Literal["conversation.item.done"] - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemInputAudioTranscriptionDelta(ServerEvent): - """Event containing incremental input audio transcription. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.delta". - item_id: ID of the conversation item being transcribed. - content_index: Index of the content within the item. - delta: Incremental transcription text. - """ - - type: Literal["conversation.item.input_audio_transcription.delta"] - item_id: str - content_index: int - delta: str - - -class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): - """Event indicating input audio transcription is complete. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.completed". - item_id: ID of the conversation item that was transcribed. - content_index: Index of the content within the item. - transcript: Complete transcription text. - """ - - type: Literal["conversation.item.input_audio_transcription.completed"] - item_id: str - content_index: int - transcript: str - - -class ConversationItemInputAudioTranscriptionFailed(ServerEvent): - """Event indicating input audio transcription failed. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.failed". - item_id: ID of the conversation item that failed transcription. - content_index: Index of the content within the item. - error: Error details for the transcription failure. - """ - - type: Literal["conversation.item.input_audio_transcription.failed"] - item_id: str - content_index: int - error: RealtimeError - - -class ConversationItemTruncated(ServerEvent): - """Event indicating a conversation item has been truncated. - - Parameters: - type: Event type, always "conversation.item.truncated". - item_id: ID of the truncated conversation item. - content_index: Index of the content within the item. - audio_end_ms: End time in milliseconds for the truncated audio. - """ - - type: Literal["conversation.item.truncated"] - item_id: str - content_index: int - audio_end_ms: int - - -class ConversationItemDeleted(ServerEvent): - """Event indicating a conversation item has been deleted. - - Parameters: - type: Event type, always "conversation.item.deleted". - item_id: ID of the deleted conversation item. - """ - - type: Literal["conversation.item.deleted"] - item_id: str - - -class ConversationItemRetrieved(ServerEvent): - """Event containing a retrieved conversation item. - - Parameters: - type: Event type, always "conversation.item.retrieved". - item: The retrieved conversation item. - """ - - type: Literal["conversation.item.retrieved"] - item: ConversationItem - - -class ResponseCreated(ServerEvent): - """Event indicating an assistant response has been created. - - Parameters: - type: Event type, always "response.created". - response: The created response object. - """ - - type: Literal["response.created"] - response: "Response" - - -class ResponseDone(ServerEvent): - """Event indicating an assistant response is complete. - - Parameters: - type: Event type, always "response.done". - response: The completed response object. - """ - - type: Literal["response.done"] - response: "Response" - - -class ResponseOutputItemAdded(ServerEvent): - """Event indicating an output item has been added to a response. - - Parameters: - type: Event type, always "response.output_item.added". - response_id: ID of the response. - output_index: Index of the output item. - item: The added conversation item. - """ - - type: Literal["response.output_item.added"] - response_id: str - output_index: int - item: ConversationItem - - -class ResponseOutputItemDone(ServerEvent): - """Event indicating an output item is complete. - - Parameters: - type: Event type, always "response.output_item.done". - response_id: ID of the response. - output_index: Index of the output item. - item: The completed conversation item. - """ - - type: Literal["response.output_item.done"] - response_id: str - output_index: int - item: ConversationItem - - -class ResponseContentPartAdded(ServerEvent): - """Event indicating a content part has been added to a response. - - Parameters: - type: Event type, always "response.content_part.added". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - part: The added content part. - """ - - type: Literal["response.content_part.added"] - response_id: str - item_id: str - output_index: int - content_index: int - part: ItemContent - - -class ResponseContentPartDone(ServerEvent): - """Event indicating a content part is complete. - - Parameters: - type: Event type, always "response.content_part.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - part: The completed content part. - """ - - type: Literal["response.content_part.done"] - response_id: str - item_id: str - output_index: int - content_index: int - part: ItemContent - - -class ResponseTextDelta(ServerEvent): - """Event containing incremental text from a response. - - Parameters: - type: Event type, always "response.output_text.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Incremental text content. - """ - - type: Literal["response.output_text.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str - - -class ResponseTextDone(ServerEvent): - """Event indicating text content is complete. - - Parameters: - type: Event type, always "response.output_text.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - text: Complete text content. - """ - - type: Literal["response.output_text.done"] - response_id: str - item_id: str - output_index: int - content_index: int - text: str - - -class ResponseAudioTranscriptDelta(ServerEvent): - """Event containing incremental audio transcript from a response. - - Parameters: - type: Event type, always "response.output_audio_transcript.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Incremental transcript text. - """ - - type: Literal["response.output_audio_transcript.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str - - -class ResponseAudioTranscriptDone(ServerEvent): - """Event indicating audio transcript is complete. - - Parameters: - type: Event type, always "response.output_audio_transcript.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - transcript: Complete transcript text. - """ - - type: Literal["response.output_audio_transcript.done"] - response_id: str - item_id: str - output_index: int - content_index: int - transcript: str - - -class ResponseAudioDelta(ServerEvent): - """Event containing incremental audio data from a response. - - Parameters: - type: Event type, always "response.output_audio.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Base64-encoded incremental audio data. - """ - - type: Literal["response.output_audio.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str # base64-encoded audio - - -class ResponseAudioDone(ServerEvent): - """Event indicating audio content is complete. - - Parameters: - type: Event type, always "response.output_audio.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - """ - - type: Literal["response.output_audio.done"] - response_id: str - item_id: str - output_index: int - content_index: int - - -class ResponseFunctionCallArgumentsDelta(ServerEvent): - """Event containing incremental function call arguments. - - Parameters: - type: Event type, always "response.function_call_arguments.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - call_id: ID of the function call. - delta: Incremental function arguments as JSON. - """ - - type: Literal["response.function_call_arguments.delta"] - response_id: str - item_id: str - output_index: int - call_id: str - delta: str - - -class ResponseFunctionCallArgumentsDone(ServerEvent): - """Event indicating function call arguments are complete. - - Parameters: - type: Event type, always "response.function_call_arguments.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - call_id: ID of the function call. - arguments: Complete function arguments as JSON string. - """ - - type: Literal["response.function_call_arguments.done"] - response_id: str - item_id: str - output_index: int - call_id: str - arguments: str - - -class InputAudioBufferSpeechStarted(ServerEvent): - """Event indicating speech has started in the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.speech_started". - audio_start_ms: Start time of speech in milliseconds. - item_id: ID of the associated conversation item. - """ - - type: Literal["input_audio_buffer.speech_started"] - audio_start_ms: int - item_id: str - - -class InputAudioBufferSpeechStopped(ServerEvent): - """Event indicating speech has stopped in the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.speech_stopped". - audio_end_ms: End time of speech in milliseconds. - item_id: ID of the associated conversation item. - """ - - type: Literal["input_audio_buffer.speech_stopped"] - audio_end_ms: int - item_id: str - - -class InputAudioBufferCommitted(ServerEvent): - """Event indicating the input audio buffer has been committed. - - Parameters: - type: Event type, always "input_audio_buffer.committed". - previous_item_id: ID of the previous item, if any. - item_id: ID of the committed conversation item. - """ - - type: Literal["input_audio_buffer.committed"] - previous_item_id: Optional[str] = None - item_id: str - - -class InputAudioBufferCleared(ServerEvent): - """Event indicating the input audio buffer has been cleared. - - Parameters: - type: Event type, always "input_audio_buffer.cleared". - """ - - type: Literal["input_audio_buffer.cleared"] - - -class ErrorEvent(ServerEvent): - """Event indicating an error occurred. - - Parameters: - type: Event type, always "error". - error: Error details. - """ - - type: Literal["error"] - error: RealtimeError - - -class RateLimitsUpdated(ServerEvent): - """Event indicating rate limits have been updated. - - Parameters: - type: Event type, always "rate_limits.updated". - rate_limits: List of rate limit information. - """ - - type: Literal["rate_limits.updated"] - rate_limits: List[Dict[str, Any]] - - -class CachedTokensDetails(BaseModel): - """Details about cached tokens. - - Parameters: - text_tokens: Number of cached text tokens. - audio_tokens: Number of cached audio tokens. - """ - - text_tokens: Optional[int] = 0 - audio_tokens: Optional[int] = 0 - - -class TokenDetails(BaseModel): - """Detailed token usage information. - - Parameters: - cached_tokens: Number of cached tokens used. Defaults to 0. - text_tokens: Number of text tokens used. Defaults to 0. - audio_tokens: Number of audio tokens used. Defaults to 0. - cached_tokens_details: Detailed breakdown of cached tokens. - image_tokens: Number of image tokens used (for input only). - """ - - cached_tokens: Optional[int] = 0 - text_tokens: Optional[int] = 0 - audio_tokens: Optional[int] = 0 - cached_tokens_details: Optional[CachedTokensDetails] = None - image_tokens: Optional[int] = 0 - - class Config: - """Pydantic configuration for TokenDetails.""" - - extra = "allow" - - -class Usage(BaseModel): - """Token usage statistics for a response. - - Parameters: - total_tokens: Total number of tokens used. - input_tokens: Number of input tokens used. - output_tokens: Number of output tokens used. - input_token_details: Detailed breakdown of input token usage. - output_token_details: Detailed breakdown of output token usage. - """ - - total_tokens: int - input_tokens: int - output_tokens: int - input_token_details: TokenDetails - output_token_details: TokenDetails - - -class Response(BaseModel): - """A complete assistant response. - - Parameters: - id: Unique identifier for the response. - object: Object type, always "realtime.response". - status: Current status of the response. - status_details: Additional status information. - output: List of conversation items in the response. - conversation_id: Which conversation the response is added to. - output_modalities: The set of modalities the model used to respond. - max_output_tokens: Maximum number of output tokens used. - audio: Audio configuration for the response. - usage: Token usage statistics for the response. - voice: The voice the model used to respond. - temperature: Sampling temperature used for the response. - output_audio_format: The format of output audio. - """ - - id: str - object: Literal["realtime.response"] - status: Literal["completed", "in_progress", "incomplete", "cancelled", "failed"] - status_details: Any - output: List[ConversationItem] - output_modalities: Optional[List[Literal["text", "audio"]]] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - audio: Optional[AudioConfiguration] = None - usage: Optional[Usage] = None - voice: Optional[str] = None - temperature: Optional[float] = None - output_audio_format: Optional[str] = None - - -_server_event_types = { - "error": ErrorEvent, - "session.created": SessionCreatedEvent, - "session.updated": SessionUpdatedEvent, - "conversation.created": ConversationCreated, - "input_audio_buffer.committed": InputAudioBufferCommitted, - "input_audio_buffer.cleared": InputAudioBufferCleared, - "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, - "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, - "conversation.item.added": ConversationItemAdded, - "conversation.item.done": ConversationItemDone, - "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, - "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, - "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, - "conversation.item.truncated": ConversationItemTruncated, - "conversation.item.deleted": ConversationItemDeleted, - "conversation.item.retrieved": ConversationItemRetrieved, - "response.created": ResponseCreated, - "response.done": ResponseDone, - "response.output_item.added": ResponseOutputItemAdded, - "response.output_item.done": ResponseOutputItemDone, - "response.content_part.added": ResponseContentPartAdded, - "response.content_part.done": ResponseContentPartDone, - "response.output_text.delta": ResponseTextDelta, - "response.output_text.done": ResponseTextDone, - "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta, - "response.output_audio_transcript.done": ResponseAudioTranscriptDone, - "response.output_audio.delta": ResponseAudioDelta, - "response.output_audio.done": ResponseAudioDone, - "response.function_call_arguments.delta": ResponseFunctionCallArgumentsDelta, - "response.function_call_arguments.done": ResponseFunctionCallArgumentsDone, - "rate_limits.updated": RateLimitsUpdated, -} - - -def parse_server_event(str): - """Parse a server event from JSON string. - - Args: - str: JSON string containing the server event. - - Returns: - Parsed server event object of the appropriate type. - - Raises: - Exception: If the event type is unimplemented or parsing fails. - """ - try: - event = json.loads(str) - event_type = event["type"] - if event_type not in _server_event_types: - raise Exception(f"Unimplemented server event type: {event_type}") - return _server_event_types[event_type].model_validate(event) - except Exception as e: - raise Exception(f"{e} \n\n{str}") +import warnings + +from pipecat.services.openai.realtime.events import * + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.events are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.events instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/frames.py b/src/pipecat/services/openai_realtime/frames.py index 290e025f9..e7e4d7d9f 100644 --- a/src/pipecat/services/openai_realtime/frames.py +++ b/src/pipecat/services/openai_realtime/frames.py @@ -1,37 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Custom frame types for OpenAI Realtime API integration.""" -from dataclasses import dataclass -from typing import TYPE_CHECKING +import warnings -from pipecat.frames.frames import DataFrame, FunctionCallResultFrame +from pipecat.services.openai.realtime.frames import * -if TYPE_CHECKING: - from pipecat.services.openai_realtime.context import OpenAIRealtimeLLMContext - - -@dataclass -class RealtimeMessagesUpdateFrame(DataFrame): - """Frame indicating that the realtime context messages have been updated. - - Parameters: - context: The updated OpenAI realtime LLM context. - """ - - context: "OpenAIRealtimeLLMContext" - - -@dataclass -class RealtimeFunctionCallResultFrame(DataFrame): - """Frame containing function call results for the realtime service. - - Parameters: - result_frame: The function call result frame to send to the realtime API. - """ - - result_frame: FunctionCallResultFrame +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.frames are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.frames instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime_beta/azure.py b/src/pipecat/services/openai_realtime_beta/azure.py index 784438e81..2ec556ee3 100644 --- a/src/pipecat/services/openai_realtime_beta/azure.py +++ b/src/pipecat/services/openai_realtime_beta/azure.py @@ -70,7 +70,7 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): # handle disconnections in the send/recv code paths. return - logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") + logger.info(f"Connecting to {self.base_url}") self._websocket = await websocket_connect( uri=self.base_url, additional_headers={ diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index d5b663c77..fa43a720c 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -14,7 +14,6 @@ from loguru import logger from pipecat.frames.frames import ( ErrorFrame, Frame, - TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) @@ -99,16 +98,15 @@ class PiperTTSService(TTSService): await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + CHUNK_SIZE = self.chunk_size - yield TTSStartedFrame() - async for chunk in response.content.iter_chunked(CHUNK_SIZE): - # remove wav header if present - if chunk.startswith(b"RIFF"): - chunk = chunk[44:] - if len(chunk) > 0: - await self.stop_ttfb_metrics() - yield TTSAudioRawFrame(chunk, self.sample_rate, 1) + async for frame in self._stream_audio_frames_from_iterator( + response.content.iter_chunked(CHUNK_SIZE), strip_wav_header=True + ): + await self.stop_ttfb_metrics() + yield frame except Exception as e: logger.error(f"Error in run_tts: {e}") yield ErrorFrame(error=str(e)) diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 0f23b7b5e..925480794 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -14,6 +14,7 @@ import io import json import struct import uuid +import warnings from typing import AsyncGenerator, Optional import aiohttp @@ -110,6 +111,11 @@ def language_to_playht_language(language: Language) -> Optional[str]: class PlayHTTTSService(InterruptibleTTSService): """PlayHT WebSocket-based text-to-speech service. + .. deprecated:: 0.0.88 + + This class is deprecated and will be removed in a future version. + PlayHT is shutting down their API on December 31st, 2025. + Provides real-time text-to-speech synthesis using PlayHT's WebSocket API. Supports streaming audio generation with configurable voice engines and language settings. @@ -158,6 +164,15 @@ class PlayHTTTSService(InterruptibleTTSService): **kwargs, ) + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "PlayHT is shutting down their API on December 31st, 2025. " + "'PlayHTTTSService' is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + params = params or PlayHTTTSService.InputParams() self._api_key = api_key @@ -254,6 +269,8 @@ class PlayHTTTSService(InterruptibleTTSService): raise ValueError("WebSocket URL is not a string") self._websocket = await websocket_connect(self._websocket_url) + + await self._call_event_handler("on_connected") except ValueError as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -276,6 +293,7 @@ class PlayHTTTSService(InterruptibleTTSService): finally: self._request_id = None self._websocket = None + await self._call_event_handler("on_disconnected") async def _get_websocket_url(self): """Retrieve WebSocket URL from PlayHT API.""" @@ -401,6 +419,11 @@ class PlayHTTTSService(InterruptibleTTSService): class PlayHTHttpTTSService(TTSService): """PlayHT HTTP-based text-to-speech service. + .. deprecated:: 0.0.88 + + This class is deprecated and will be removed in a future version. + PlayHT is shutting down their API on December 31st, 2025. + Provides text-to-speech synthesis using PlayHT's HTTP API for simpler, non-streaming synthesis. Suitable for use cases where streaming is not required and simpler integration is preferred. @@ -454,8 +477,6 @@ class PlayHTHttpTTSService(TTSService): # Warn about deprecated protocol parameter if explicitly provided if protocol: - import warnings - with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( @@ -464,6 +485,15 @@ class PlayHTHttpTTSService(TTSService): stacklevel=2, ) + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "PlayHT is shutting down their API on December 31st, 2025. " + "'PlayHTHttpTTSService' is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + params = params or PlayHTHttpTTSService.InputParams() self._user_id = user_id diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 917716545..fa3fa447d 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -255,6 +255,8 @@ class RimeTTSService(AudioContextWordTTSService): url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect(url, additional_headers=headers) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -272,6 +274,7 @@ class RimeTTSService(AudioContextWordTTSService): finally: self._context_id = None self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): """Get active websocket connection or raise exception.""" @@ -553,15 +556,13 @@ class RimeHttpTTSService(TTSService): CHUNK_SIZE = self.chunk_size - async for chunk in response.content.iter_chunked(CHUNK_SIZE): - if need_to_strip_wav_header and chunk.startswith(b"RIFF"): - chunk = chunk[44:] - need_to_strip_wav_header = False + async for frame in self._stream_audio_frames_from_iterator( + response.content.iter_chunked(CHUNK_SIZE), + strip_wav_header=need_to_strip_wav_header, + ): + await self.stop_ttfb_metrics() + yield frame - if len(chunk) > 0: - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) - yield frame except Exception as e: logger.exception(f"Error generating TTS: {e}") yield ErrorFrame(error=f"Rime TTS error: {str(e)}") diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py index d00eb4f42..eddd3da9e 100644 --- a/src/pipecat/services/riva/stt.py +++ b/src/pipecat/services/riva/stt.py @@ -583,7 +583,9 @@ class RivaSegmentedSTTService(SegmentedSTTService): self._config.language_code = self._language @traced_stt - async def _handle_transcription(self, transcript: str, language: Optional[Language] = None): + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): """Handle a transcription result with tracing.""" pass diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index a9fedcc58..7096683eb 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -76,17 +76,29 @@ class SarvamHttpTTSService(TTSService): Example:: - tts = SarvamTTSService( + tts = SarvamHttpTTSService( api_key="your-api-key", voice_id="anushka", model="bulbul:v2", aiohttp_session=session, - params=SarvamTTSService.InputParams( + params=SarvamHttpTTSService.InputParams( language=Language.HI, pitch=0.1, pace=1.2 ) ) + + # For bulbul v3 beta with any speaker: + tts_v3 = SarvamHttpTTSService( + api_key="your-api-key", + voice_id="speaker_name", + model="bulbul:v3, + aiohttp_session=session, + params=SarvamHttpTTSService.InputParams( + language=Language.HI, + temperature=0.8 + ) + ) """ class InputParams(BaseModel): @@ -105,6 +117,14 @@ class SarvamHttpTTSService(TTSService): pace: Optional[float] = Field(default=1.0, ge=0.3, le=3.0) loudness: Optional[float] = Field(default=1.0, ge=0.1, le=3.0) enable_preprocessing: Optional[bool] = False + temperature: Optional[float] = Field( + default=0.6, + ge=0.01, + le=1.0, + description="Controls the randomness of the output for bulbul v3 beta. " + "Lower values make the output more focused and deterministic, while " + "higher values make it more random. Range: 0.01 to 1.0. Default: 0.6.", + ) def __init__( self, @@ -124,7 +144,7 @@ class SarvamHttpTTSService(TTSService): api_key: Sarvam AI API subscription key. aiohttp_session: Shared aiohttp session for making requests. voice_id: Speaker voice ID (e.g., "anushka", "meera"). Defaults to "anushka". - model: TTS model to use ("bulbul:v1" or "bulbul:v2"). Defaults to "bulbul:v2". + model: TTS model to use ("bulbul:v2" or "bulbul:v3-beta" or "bulbul:v3"). Defaults to "bulbul:v2". base_url: Sarvam AI API base URL. Defaults to "https://api.sarvam.ai". sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000). If None, uses default. params: Additional voice and preprocessing parameters. If None, uses defaults. @@ -138,16 +158,32 @@ class SarvamHttpTTSService(TTSService): self._base_url = base_url self._session = aiohttp_session + # Build base settings common to all models self._settings = { "language": ( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "pitch": params.pitch, - "pace": params.pace, - "loudness": params.loudness, "enable_preprocessing": params.enable_preprocessing, } + # Add model-specific parameters + if model in ("bulbul:v3-beta", "bulbul:v3"): + self._settings.update( + { + "temperature": getattr(params, "temperature", 0.6), + "model": model, + } + ) + else: + self._settings.update( + { + "pitch": params.pitch, + "pace": params.pace, + "loudness": params.loudness, + "model": model, + } + ) + self.set_model_name(model) self.set_voice(voice_id) @@ -275,6 +311,18 @@ class SarvamTTSService(InterruptibleTTSService): pace=1.2 ) ) + + # For bulbul v3 beta with any speaker and temperature: + # Note: pace and loudness are not supported for bulbul v3 and bulbul v3 beta + tts_v3 = SarvamTTSService( + api_key="your-api-key", + voice_id="speaker_name", + model="bulbul:v3", + params=SarvamTTSService.InputParams( + language=Language.HI, + temperature=0.8 + ) + ) """ class InputParams(BaseModel): @@ -310,6 +358,14 @@ class SarvamTTSService(InterruptibleTTSService): output_audio_codec: Optional[str] = "linear16" output_audio_bitrate: Optional[str] = "128k" language: Optional[Language] = Language.EN + temperature: Optional[float] = Field( + default=0.6, + ge=0.01, + le=1.0, + description="Controls the randomness of the output for bulbul v3 beta. " + "Lower values make the output more focused and deterministic, while " + "higher values make it more random. Range: 0.01 to 1.0. Default: 0.6.", + ) def __init__( self, @@ -318,7 +374,6 @@ class SarvamTTSService(InterruptibleTTSService): model: str = "bulbul:v2", voice_id: str = "anushka", url: str = "wss://api.sarvam.ai/text-to-speech/ws", - aiohttp_session: Optional[aiohttp.ClientSession] = None, aggregate_sentences: Optional[bool] = True, sample_rate: Optional[int] = None, params: Optional[InputParams] = None, @@ -329,13 +384,9 @@ class SarvamTTSService(InterruptibleTTSService): Args: api_key: Sarvam API key for authenticating TTS requests. model: Identifier of the Sarvam speech model (default "bulbul:v2"). + Supports "bulbul:v2", "bulbul:v3-beta" and "bulbul:v3". voice_id: Voice identifier for synthesis (default "anushka"). url: WebSocket URL for connecting to the TTS backend (default production URL). - aiohttp_session: Optional shared aiohttp session. To maintain backward compatibility. - - .. deprecated:: 0.0.81 - aiohttp_session is no longer used. This parameter will be removed in a future version. - aggregate_sentences: Whether to merge multiple sentences into one audio chunk (default True). sample_rate: Desired sample rate for the output audio in Hz (overrides default if set). params: Optional input parameters to override global configuration. @@ -356,30 +407,18 @@ class SarvamTTSService(InterruptibleTTSService): **kwargs, ) params = params or SarvamTTSService.InputParams() - if aiohttp_session is not None: - import warnings - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "The 'aiohttp_session' parameter is deprecated and will be removed in a future version. ", - DeprecationWarning, - stacklevel=2, - ) # WebSocket endpoint URL self._websocket_url = f"{url}?model={model}" self._api_key = api_key self.set_model_name(model) self.set_voice(voice_id) - # Configuration parameters + # Build base settings common to all models self._settings = { "target_language_code": ( self.language_to_service_language(params.language) if params.language else "en-IN" ), - "pitch": params.pitch, - "pace": params.pace, "speaker": voice_id, - "loudness": params.loudness, "speech_sample_rate": 0, "enable_preprocessing": params.enable_preprocessing, "min_buffer_size": params.min_buffer_size, @@ -387,6 +426,24 @@ class SarvamTTSService(InterruptibleTTSService): "output_audio_codec": params.output_audio_codec, "output_audio_bitrate": params.output_audio_bitrate, } + + # Add model-specific parameters + if model in ("bulbul:v3-beta", "bulbul:v3"): + self._settings.update( + { + "temperature": getattr(params, "temperature", 0.6), + "model": model, + } + ) + else: + self._settings.update( + { + "pitch": params.pitch, + "pace": params.pace, + "loudness": params.loudness, + "model": model, + } + ) self._started = False self._receive_task = None @@ -525,6 +582,7 @@ class SarvamTTSService(InterruptibleTTSService): logger.debug("Connected to Sarvam TTS Websocket") await self._send_config() + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -556,6 +614,10 @@ class SarvamTTSService(InterruptibleTTSService): await self._websocket.close() except Exception as e: logger.error(f"{self} error closing websocket: {e}") + finally: + self._started = False + self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index d48a744e0..383a8a3cb 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -7,9 +7,12 @@ """Simli video service for real-time avatar generation.""" import asyncio +import warnings +from typing import Optional import numpy as np from loguru import logger +from pydantic import BaseModel from pipecat.frames.frames import ( CancelFrame, @@ -41,30 +44,103 @@ class SimliVideoService(FrameProcessor): audio resampling, video frame processing, and connection management. """ + class InputParams(BaseModel): + """Input parameters for Simli video configuration. + + Parameters: + max_session_length: Absolute maximum session duration in seconds. + Avatar will disconnect after this time even if it's speaking. + max_idle_time: Maximum duration in seconds the avatar is not speaking + before the avatar disconnects. + """ + + max_session_length: Optional[int] = None + max_idle_time: Optional[int] = None + def __init__( self, - simli_config: SimliConfig, + *, + api_key: Optional[str] = None, + face_id: Optional[str] = None, + simli_config: Optional[SimliConfig] = None, use_turn_server: bool = False, latency_interval: int = 0, simli_url: str = "https://api.simli.ai", is_trinity_avatar: bool = False, + params: Optional[InputParams] = None, + **kwargs, ): """Initialize the Simli video service. Args: + api_key: Simli API key for authentication. + face_id: Simli Face ID. For Trinity avatars, specify "faceId/emotionId" + to use a different emotion than the default. simli_config: Configuration object for Simli client settings. - use_turn_server: Whether to use TURN server for connection. Defaults to False. - latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. - simli_url: URL of the simli servers. Can be changed for custom deployments of enterprise users. - is_trinity_avatar: boolean to tell simli client that this is a Trinity avatar which reduces latency when using Trinity. + Use api_key and face_id instead. + .. deprecated:: 0.0.92 + The 'simli_config' parameter is deprecated and will be removed in a future version. + Please use 'api_key' and 'face_id' parameters instead. + + use_turn_server: Whether to use TURN server for connection. Defaults to False. + latency_interval: Latency interval setting for sending health checks to check + the latency to Simli Servers. Defaults to 0. + simli_url: URL of the simli servers. Can be changed for custom deployments + of enterprise users. + is_trinity_avatar: Boolean to tell simli client that this is a Trinity avatar + which reduces latency when using Trinity. + params: Additional input parameters for session configuration. + **kwargs: Additional arguments passed to the parent FrameProcessor. """ - super().__init__() + super().__init__(**kwargs) + + params = params or SimliVideoService.InputParams() + + # Handle deprecated simli_config parameter + if simli_config is not None: + if api_key is not None or face_id is not None: + raise ValueError( + "Cannot specify both simli_config and api_key/face_id. " + "Please use api_key and face_id (simli_config is deprecated)." + ) + + warnings.warn( + "The 'simli_config' parameter is deprecated and will be removed in a future version. " + "Please use 'api_key' and 'face_id' parameters instead, with optional 'params' for " + "max_session_length and max_idle_time configuration.", + DeprecationWarning, + stacklevel=2, + ) + + # Use the provided simli_config + config = simli_config + else: + # Validate new parameters + if api_key is None: + raise ValueError("api_key is required") + if face_id is None: + raise ValueError("face_id is required") + + # Build SimliConfig from new parameters + # Only pass optional parameters if explicitly provided to use SimliConfig defaults + config_kwargs = { + "apiKey": api_key, + "faceId": face_id, + } + if params.max_session_length is not None: + config_kwargs["maxSessionLength"] = params.max_session_length + if params.max_idle_time is not None: + config_kwargs["maxIdleTime"] = params.max_idle_time + + config = SimliConfig(**config_kwargs) + self._initialized = False - simli_config.maxIdleTime += 5 - simli_config.maxSessionLength += 5 + # Add buffer time to session limits + config.maxIdleTime += 5 + config.maxSessionLength += 5 self._simli_client = SimliClient( - simli_config, + config, use_turn_server, latency_interval, simliURL=simli_url, diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 4028dd248..901edb0e8 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -577,6 +577,7 @@ class SpeechmaticsSTTService(STTService): ), ) logger.debug(f"{self} Connected to Speechmatics STT service") + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} Error connecting to Speechmatics: {e}") self._client = None @@ -595,6 +596,7 @@ class SpeechmaticsSTTService(STTService): logger.error(f"{self} Error closing Speechmatics client: {e}") finally: self._client = None + await self._call_event_handler("on_disconnected") def _process_config(self) -> None: """Create a formatted STT transcription config. @@ -618,7 +620,7 @@ class SpeechmaticsSTTService(STTService): transcription_config.additional_vocab = [ { "content": e.content, - "sounds_like": e.sounds_like, + **({"sounds_like": e.sounds_like} if e.sounds_like else {}), } for e in self._params.additional_vocab ] diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index a02619e44..6fb96c571 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -35,6 +35,25 @@ class STTService(AIService): Provides common functionality for STT services including audio passthrough, muting, settings management, and audio processing. Subclasses must implement the run_stt method to provide actual speech recognition. + + Event handlers: + on_connected: Called when connected to the STT service. + on_connected: Called when disconnected from the STT service. + on_connection_error: Called when a connection to the STT service error occurs. + + Example:: + + @stt.event_handler("on_connected") + async def on_connected(stt: STTService): + logger.debug(f"STT connected") + + @stt.event_handler("on_disconnected") + async def on_disconnected(stt: STTService): + logger.debug(f"STT disconnected") + + @stt.event_handler("on_connection_error") + async def on_connection_error(stt: STTService, error: str): + logger.error(f"STT connection error: {error}") """ def __init__( @@ -62,6 +81,10 @@ class STTService(AIService): self._muted: bool = False self._user_id: str = "" + self._register_event_handler("on_connected") + self._register_event_handler("on_disconnected") + self._register_event_handler("on_connection_error") + @property def is_muted(self) -> bool: """Check if the STT service is currently muted. @@ -292,15 +315,6 @@ class WebsocketSTTService(STTService, WebsocketService): Combines STT functionality with websocket connectivity, providing automatic error handling and reconnection capabilities. - - Event handlers: - on_connection_error: Called when a websocket connection error occurs. - - Example:: - - @stt.event_handler("on_connection_error") - async def on_connection_error(stt: STTService, error: str): - logger.error(f"STT connection error: {error}") """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): @@ -312,7 +326,6 @@ class WebsocketSTTService(STTService, WebsocketService): """ STTService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) - self._register_event_handler("on_connection_error") async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 02b80b609..b356c7244 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -8,7 +8,17 @@ import asyncio from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import ( + Any, + AsyncGenerator, + AsyncIterator, + Dict, + List, + Mapping, + Optional, + Sequence, + Tuple, +) from loguru import logger @@ -49,6 +59,25 @@ class TTSService(AIService): Provides common functionality for TTS services including text aggregation, filtering, audio generation, and frame management. Supports configurable sentence aggregation, silence insertion, and frame processing control. + + Event handlers: + on_connected: Called when connected to the STT service. + on_connected: Called when disconnected from the STT service. + on_connection_error: Called when a connection to the STT service error occurs. + + Example:: + + @tts.event_handler("on_connected") + async def on_connected(tts: TTSService): + logger.debug(f"TTS connected") + + @tts.event_handler("on_disconnected") + async def on_disconnected(tts: TTSService): + logger.debug(f"TTS disconnected") + + @tts.event_handler("on_connection_error") + async def on_connection_error(stt: TTSService, error: str): + logger.error(f"TTS connection error: {error}") """ def __init__( @@ -133,6 +162,10 @@ class TTSService(AIService): self._processing_text: bool = False + self._register_event_handler("on_connected") + self._register_event_handler("on_disconnected") + self._register_event_handler("on_connection_error") + @property def sample_rate(self) -> int: """Get the current sample rate for audio output. @@ -374,6 +407,36 @@ class TTSService(AIService): ): await self._stop_frame_queue.put(frame) + async def _stream_audio_frames_from_iterator( + self, iterator: AsyncIterator[bytes], *, strip_wav_header: bool + ) -> AsyncGenerator[Frame, None]: + buffer = bytearray() + need_to_strip_wav_header = strip_wav_header + async for chunk in iterator: + if need_to_strip_wav_header and chunk.startswith(b"RIFF"): + chunk = chunk[44:] + need_to_strip_wav_header = False + + # Append to current buffer. + buffer.extend(chunk) + + # Round to nearest even number. + aligned_length = len(buffer) & ~1 # 111111111...11110 + if aligned_length > 0: + aligned_chunk = buffer[:aligned_length] + buffer = buffer[aligned_length:] # keep any leftover byte + + if len(aligned_chunk) > 0: + frame = TTSAudioRawFrame(bytes(aligned_chunk), self.sample_rate, 1) + yield frame + + if len(buffer) > 0: + # Make sure we don't need an extra padding byte. + if len(buffer) % 2 == 1: + buffer.extend(b"\x00") + frame = TTSAudioRawFrame(bytes(buffer), self.sample_rate, 1) + yield frame + async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): self._processing_text = False await self._text_aggregator.handle_interruption() @@ -586,7 +649,6 @@ class WebsocketTTSService(TTSService, WebsocketService): """ TTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) - self._register_event_handler("on_connection_error") async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) @@ -638,15 +700,6 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): """Base class for websocket-based TTS services that support word timestamps. Combines word timestamp functionality with websocket connectivity. - - Event handlers: - on_connection_error: Called when a websocket connection error occurs. - - Example:: - - @tts.event_handler("on_connection_error") - async def on_connection_error(tts: TTSService, error: str): - logger.error(f"TTS connection error: {error}") """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): @@ -658,7 +711,6 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): """ WordTTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) - self._register_event_handler("on_connection_error") async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 25cc56f68..0f247c6dc 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -232,6 +232,9 @@ class BaseInputTransport(FrameProcessor): """ # Cancel and wait for the audio input task to finish. await self._cancel_audio_task() + # Stop audio filter. + if self._params.audio_in_filter: + await self._params.audio_in_filter.stop() async def set_transport_ready(self, frame: StartFrame): """Called when the transport is ready to stream. diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 9d4630100..3edf434a9 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -293,15 +293,15 @@ class BaseOutputTransport(FrameProcessor): """ await super().process_frame(frame, direction) - # - # System frames (like InterruptionFrame) are pushed immediately. Other - # frames require order so they are put in the sink queue. - # if isinstance(frame, StartFrame): # Push StartFrame before start(), because we want StartFrame to be # processed by every processor before any other frame is processed. await self.push_frame(frame, direction) await self.start(frame) + elif isinstance(frame, EndFrame): + await self.stop(frame) + # Keep pushing EndFrame down so all the pipeline stops nicely. + await self.push_frame(frame, direction) elif isinstance(frame, CancelFrame): await self.cancel(frame) await self.push_frame(frame, direction) @@ -314,21 +314,6 @@ class BaseOutputTransport(FrameProcessor): await self.write_dtmf(frame) elif isinstance(frame, SystemFrame): await self.push_frame(frame, direction) - # Control frames. - elif isinstance(frame, EndFrame): - await self.stop(frame) - # Keep pushing EndFrame down so all the pipeline stops nicely. - await self.push_frame(frame, direction) - elif isinstance(frame, MixerControlFrame): - await self._handle_frame(frame) - # Other frames. - elif isinstance(frame, OutputAudioRawFrame): - await self._handle_frame(frame) - elif isinstance(frame, (OutputImageRawFrame, SpriteFrame)): - await self._handle_frame(frame) - # TODO(aleix): Images and audio should support presentation timestamps. - elif frame.pts: - await self._handle_frame(frame) elif direction == FrameDirection.UPSTREAM: await self.push_frame(frame, direction) else: @@ -410,6 +395,13 @@ class BaseOutputTransport(FrameProcessor): # Indicates if the bot is currently speaking. self._bot_speaking = False + # Last time a BotSpeakingFrame was pushed. + self._bot_speaking_frame_time = 0 + # How often a BotSpeakingFrame should be pushed (value should be + # lower than the audio chunks). + self._bot_speaking_frame_period = 0.2 + # Last time the bot actually spoke. + self._bot_speech_last_time = 0 self._audio_task: Optional[asyncio.Task] = None self._video_task: Optional[asyncio.Task] = None @@ -601,39 +593,71 @@ class BaseOutputTransport(FrameProcessor): async def _bot_started_speaking(self): """Handle bot started speaking event.""" - if not self._bot_speaking: - logger.debug( - f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking" - ) + if self._bot_speaking: + return - downstream_frame = BotStartedSpeakingFrame() - downstream_frame.transport_destination = self._destination - upstream_frame = BotStartedSpeakingFrame() - upstream_frame.transport_destination = self._destination - await self._transport.push_frame(downstream_frame) - await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM) + logger.debug( + f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking" + ) - self._bot_speaking = True + downstream_frame = BotStartedSpeakingFrame() + downstream_frame.transport_destination = self._destination + upstream_frame = BotStartedSpeakingFrame() + upstream_frame.transport_destination = self._destination + await self._transport.push_frame(downstream_frame) + await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM) + + self._bot_speaking = True async def _bot_stopped_speaking(self): """Handle bot stopped speaking event.""" - if self._bot_speaking: - logger.debug( - f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking" - ) + if not self._bot_speaking: + return - downstream_frame = BotStoppedSpeakingFrame() - downstream_frame.transport_destination = self._destination - upstream_frame = BotStoppedSpeakingFrame() - upstream_frame.transport_destination = self._destination - await self._transport.push_frame(downstream_frame) - await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM) + logger.debug( + f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking" + ) - self._bot_speaking = False + downstream_frame = BotStoppedSpeakingFrame() + downstream_frame.transport_destination = self._destination + upstream_frame = BotStoppedSpeakingFrame() + upstream_frame.transport_destination = self._destination + await self._transport.push_frame(downstream_frame) + await self._transport.push_frame(upstream_frame, FrameDirection.UPSTREAM) - # Clean audio buffer (there could be tiny left overs if not multiple - # to our output chunk size). - self._audio_buffer = bytearray() + self._bot_speaking = False + + # Clean audio buffer (there could be tiny left overs if not multiple + # to our output chunk size). + self._audio_buffer = bytearray() + + async def _bot_currently_speaking(self): + """Handle bot speaking event.""" + await self._bot_started_speaking() + + diff_time = time.time() - self._bot_speaking_frame_time + if diff_time >= self._bot_speaking_frame_period: + await self._transport.push_frame(BotSpeakingFrame()) + await self._transport.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) + self._bot_speaking_frame_time = time.time() + + self._bot_speech_last_time = time.time() + + async def _maybe_bot_currently_speaking(self, frame: SpeechOutputAudioRawFrame): + if not is_silence(frame.audio): + await self._bot_currently_speaking() + else: + silence_duration = time.time() - self._bot_speech_last_time + if silence_duration > BOT_VAD_STOP_SECS: + await self._bot_stopped_speaking() + + async def _handle_bot_speech(self, frame: Frame): + # TTS case. + if isinstance(frame, TTSAudioRawFrame): + await self._bot_currently_speaking() + # Speech stream case. + elif isinstance(frame, SpeechOutputAudioRawFrame): + await self._maybe_bot_currently_speaking(frame) async def _handle_frame(self, frame: Frame): """Handle various frame types with appropriate processing. @@ -641,7 +665,9 @@ class BaseOutputTransport(FrameProcessor): Args: frame: The frame to handle. """ - if isinstance(frame, OutputImageRawFrame): + if isinstance(frame, OutputAudioRawFrame): + await self._handle_bot_speech(frame) + elif isinstance(frame, OutputImageRawFrame): await self._set_video_image(frame) elif isinstance(frame, SpriteFrame): await self._set_video_images(frame.images) @@ -705,39 +731,7 @@ class BaseOutputTransport(FrameProcessor): async def _audio_task_handler(self): """Main audio processing task handler.""" - # Push a BotSpeakingFrame every 200ms, we don't really need to push it - # at every audio chunk. If the audio chunk is bigger than 200ms, push at - # every audio chunk. - TOTAL_CHUNK_MS = self._params.audio_out_10ms_chunks * 10 - BOT_SPEAKING_CHUNK_PERIOD = max(int(200 / TOTAL_CHUNK_MS), 1) - bot_speaking_counter = 0 - speech_last_speaking_time = 0 - async for frame in self._next_frame(): - # Notify the bot started speaking upstream if necessary and that - # it's actually speaking. - is_speaking = False - if isinstance(frame, TTSAudioRawFrame): - is_speaking = True - elif isinstance(frame, SpeechOutputAudioRawFrame): - if not is_silence(frame.audio): - is_speaking = True - speech_last_speaking_time = time.time() - else: - silence_duration = time.time() - speech_last_speaking_time - if silence_duration > BOT_VAD_STOP_SECS: - await self._bot_stopped_speaking() - - if is_speaking: - await self._bot_started_speaking() - if bot_speaking_counter % BOT_SPEAKING_CHUNK_PERIOD == 0: - await self._transport.push_frame(BotSpeakingFrame()) - await self._transport.push_frame( - BotSpeakingFrame(), FrameDirection.UPSTREAM - ) - bot_speaking_counter = 0 - bot_speaking_counter += 1 - # No need to push EndFrame, it's pushed from process_frame(). if isinstance(frame, EndFrame): break diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 7f6b21ee2..2e48c4d2e 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -744,32 +744,27 @@ class DailyTransportClient(EventHandler): self._client.set_user_name(self._bot_name) - try: - (data, error) = await self._join() + (data, error) = await self._join() - if not error: - self._joined = True - self._joining = False - # Increment leave counter if we successfully joined. - self._leave_counter += 1 - - logger.info(f"Joined {self._room_url}") - - if self._params.transcription_enabled: - await self.start_transcription(self._params.transcription_settings) - - await self._callbacks.on_joined(data) - - self._joined_event.set() - else: - error_msg = f"Error joining {self._room_url}: {error}" - logger.error(error_msg) - await self._callbacks.on_error(error_msg) - except asyncio.TimeoutError: - error_msg = f"Time out joining {self._room_url}" - logger.error(error_msg) + if not error: + self._joined = True self._joining = False + # Increment leave counter if we successfully joined. + self._leave_counter += 1 + + logger.info(f"Joined {self._room_url}") + + if self._params.transcription_enabled: + await self.start_transcription(self._params.transcription_settings) + + await self._callbacks.on_joined(data) + + self._joined_event.set() + else: + error_msg = f"Error joining {self._room_url}: {error}" + logger.error(error_msg) await self._callbacks.on_error(error_msg) + self._joining = False async def _join(self): """Execute the actual room join operation.""" @@ -828,7 +823,7 @@ class DailyTransportClient(EventHandler): }, ) - return await asyncio.wait_for(future, timeout=10) + return await future async def leave(self): """Leave the Daily room and cleanup resources.""" @@ -854,17 +849,12 @@ class DailyTransportClient(EventHandler): for track_name, _ in self._custom_audio_tracks.items(): await self.remove_custom_audio_track(track_name) - try: - error = await self._leave() - if not error: - logger.info(f"Left {self._room_url}") - await self._callbacks.on_left() - else: - error_msg = f"Error leaving {self._room_url}: {error}" - logger.error(error_msg) - await self._callbacks.on_error(error_msg) - except asyncio.TimeoutError: - error_msg = f"Time out leaving {self._room_url}" + error = await self._leave() + if not error: + logger.info(f"Left {self._room_url}") + await self._callbacks.on_left() + else: + error_msg = f"Error leaving {self._room_url}: {error}" logger.error(error_msg) await self._callbacks.on_error(error_msg) @@ -875,7 +865,7 @@ class DailyTransportClient(EventHandler): future = self._get_event_loop().create_future() self._client.leave(completion=completion_callback(future)) - return await asyncio.wait_for(future, timeout=10) + return await future def _cleanup(self): """Cleanup the Daily client instance.""" diff --git a/src/pipecat/transports/smallwebrtc/connection.py b/src/pipecat/transports/smallwebrtc/connection.py index c77f4e77e..60dd7798c 100644 --- a/src/pipecat/transports/smallwebrtc/connection.py +++ b/src/pipecat/transports/smallwebrtc/connection.py @@ -689,3 +689,8 @@ class SmallWebRTCConnection(BaseObject): )() if track: track.set_enabled(signalling_message.enabled) + + async def add_ice_candidate(self, candidate): + """Handle incoming ICE candidates.""" + logger.debug(f"Adding remote candidate: {candidate}") + await self.pc.addIceCandidate(candidate) diff --git a/src/pipecat/transports/smallwebrtc/request_handler.py b/src/pipecat/transports/smallwebrtc/request_handler.py index 3a6c77843..b2c02a03e 100644 --- a/src/pipecat/transports/smallwebrtc/request_handler.py +++ b/src/pipecat/transports/smallwebrtc/request_handler.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from enum import Enum from typing import Any, Awaitable, Callable, Dict, List, Optional +from aiortc.sdp import candidate_from_sdp from fastapi import HTTPException from loguru import logger @@ -39,6 +40,34 @@ class SmallWebRTCRequest: request_data: Optional[Any] = None +@dataclass +class IceCandidate: + """The remote ice candidate object received from the peer connection. + + Parameters: + candidate: The ice candidate patch SDP string (Session Description Protocol). + sdp_mid: The SDP mid for the candidate patch. + sdp_mline_index: The SDP mline index for the candidate patch. + """ + + candidate: str + sdp_mid: str + sdp_mline_index: int + + +@dataclass +class SmallWebRTCPatchRequest: + """Small WebRTC transport session arguments for the runner. + + Parameters: + pc_id: Identifier for the peer connection. + candidates: A list of ICE candidate patches. + """ + + pc_id: str + candidates: List[IceCandidate] + + class ConnectionMode(Enum): """Enum defining the connection handling modes.""" @@ -116,6 +145,10 @@ class SmallWebRTCRequestHandler: detail="Cannot create new connection with existing connection active", ) + def update_ice_servers(self, ice_servers: Optional[List[IceServer]] = None): + """Update the list of ICE servers used for WebRTC connections.""" + self._ice_servers = ice_servers + async def handle_web_request( self, request: SmallWebRTCRequest, @@ -193,6 +226,19 @@ class SmallWebRTCRequestHandler: logger.debug(f"SmallWebRTC request details: {request}") raise + async def handle_patch_request(self, request: SmallWebRTCPatchRequest): + """Handle a SmallWebRTC patch candidate request.""" + peer_connection = self._pcs_map.get(request.pc_id) + + if not peer_connection: + raise HTTPException(status_code=404, detail="Peer connection not found") + + for c in request.candidates: + candidate = candidate_from_sdp(c.candidate) + candidate.sdpMid = c.sdp_mid + candidate.sdpMLineIndex = c.sdp_mline_index + await peer_connection.add_ice_candidate(candidate) + async def close(self): """Clear the connection map.""" coros = [pc.disconnect() for pc in self._pcs_map.values()] diff --git a/src/pipecat/transports/whatsapp/api.py b/src/pipecat/transports/whatsapp/api.py index b7e9388ee..b06ac5c6b 100644 --- a/src/pipecat/transports/whatsapp/api.py +++ b/src/pipecat/transports/whatsapp/api.py @@ -241,6 +241,14 @@ class WhatsAppApi: self._whatsapp_url = f"{self.BASE_URL}{phone_number_id}/calls" self._whatsapp_token = whatsapp_token + def update_whatsapp_token(self, whatsapp_token: str): + """Update the WhatsApp access token for authentication.""" + self._whatsapp_token = whatsapp_token + + def update_whatsapp_phone_number_id(self, phone_number_id: str): + """Update the WhatsApp phone number ID for authentication.""" + self._phone_number_id = phone_number_id + async def answer_call_to_whatsapp(self, call_id: str, action: str, sdp: str, from_: str): """Answer an incoming WhatsApp call. diff --git a/src/pipecat/transports/whatsapp/client.py b/src/pipecat/transports/whatsapp/client.py index 7c86ca2ce..f48174f90 100644 --- a/src/pipecat/transports/whatsapp/client.py +++ b/src/pipecat/transports/whatsapp/client.py @@ -12,6 +12,8 @@ WhatsApp call events. """ import asyncio +import hashlib +import hmac from typing import Awaitable, Callable, Dict, List, Optional import aiohttp @@ -47,6 +49,7 @@ class WhatsAppClient: phone_number_id: str, session: aiohttp.ClientSession, ice_servers: Optional[List[IceServer]] = None, + whatsapp_secret: Optional[str] = None, ) -> None: """Initialize the WhatsApp client. @@ -56,10 +59,12 @@ class WhatsAppClient: session: aiohttp session for making HTTP requests ice_servers: List of ICE servers for WebRTC connections. If None, defaults to Google's public STUN server + whatsapp_secret: WhatsApp APP secret for validating that the webhook request came from WhatsApp. """ self._whatsapp_api = WhatsAppApi( whatsapp_token=whatsapp_token, phone_number_id=phone_number_id, session=session ) + self._whatsapp_secret = whatsapp_secret self._ongoing_calls_map: Dict[str, SmallWebRTCConnection] = {} # Set default ICE servers if none provided @@ -68,6 +73,22 @@ class WhatsAppClient: else: self._ice_servers = ice_servers + def update_ice_servers(self, ice_servers: Optional[List[IceServer]] = None): + """Update the list of ICE servers used for WebRTC connections.""" + self._ice_servers = ice_servers + + def update_whatsapp_secret(self, whatsapp_secret: Optional[str] = None): + """Update the WhatsApp APP secret for validating that the webhook request came from WhatsApp.""" + self._whatsapp_secret = whatsapp_secret + + def update_whatsapp_token(self, whatsapp_token: str): + """Update the WhatsApp API access token.""" + self._whatsapp_api.update_whatsapp_token(whatsapp_token) + + def update_whatsapp_phone_number_id(self, phone_number_id: str): + """Update the WhatsApp phone number ID for authentication.""" + self._whatsapp_api.update_whatsapp_phone_number_id(phone_number_id) + async def terminate_all_calls(self) -> None: """Terminate all ongoing WhatsApp calls. @@ -133,10 +154,32 @@ class WhatsAppClient: return int(challenge) + async def _validate_whatsapp_webhook_request(self, raw_body: bytes, sha256_signature: str): + """Common handler for both /start and /connect endpoints.""" + # Compute HMAC SHA256 using your App Secret + expected_signature = hmac.new( + key=self._whatsapp_secret.encode("utf-8"), + msg=raw_body, + digestmod=hashlib.sha256, + ).hexdigest() + + # Extract signature from header (strip 'sha256=' prefix) + if not sha256_signature: + raise Exception("Missing X-Hub-Signature-256 header") + received_signature = sha256_signature.split("sha256=")[-1] + + # Compare signatures securely + if not hmac.compare_digest(expected_signature, received_signature): + raise Exception("Invalid webhook signature") + + logger.debug(f"Webhook signature verified!") + async def handle_webhook_request( self, request: WhatsAppWebhookRequest, connection_callback: Optional[Callable[[SmallWebRTCConnection], Awaitable[None]]] = None, + raw_body: Optional[bytes] = None, + sha256_signature: Optional[str] = None, ) -> bool: """Handle a webhook request from WhatsApp. @@ -150,6 +193,8 @@ class WhatsAppClient: connection_callback: Optional callback function to invoke when a new WebRTC connection is established. The callback receives the SmallWebRTCConnection instance. + raw_body: Optional bytes containing the raw request body. + sha256_signature: Optional X-Hub-Signature-256 header value from the request. Returns: bool: True if the webhook request was handled successfully, False otherwise @@ -159,6 +204,8 @@ class WhatsAppClient: Exception: If connection establishment or API calls fail """ try: + if self._whatsapp_secret: + await self._validate_whatsapp_webhook_request(raw_body, sha256_signature) for entry in request.entry: for change in entry.changes: # Handle connect events diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index c9cb05142..11964d23f 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -47,6 +47,7 @@ SENTENCE_ENDING_PUNCTUATION: FrozenSet[str] = frozenset( "!", "?", ";", + "…", # East Asian punctuation (Chinese (Traditional & Simplified), Japanese, Korean) "。", # Ideographic full stop "？", # Full-width question mark diff --git a/src/pipecat/utils/tracing/service_decorators.py b/src/pipecat/utils/tracing/service_decorators.py index 2edec9862..cf1ba912c 100644 --- a/src/pipecat/utils/tracing/service_decorators.py +++ b/src/pipecat/utils/tracing/service_decorators.py @@ -651,9 +651,9 @@ def traced_gemini_live(operation: str) -> Callable: elif operation == "llm_tool_call" and args: # Extract tool call information - evt = args[0] if args else None - if evt and hasattr(evt, "toolCall") and evt.toolCall.functionCalls: - function_calls = evt.toolCall.functionCalls + msg = args[0] if args else None + if msg and hasattr(msg, "tool_call") and msg.tool_call.function_calls: + function_calls = msg.tool_call.function_calls if function_calls: # Add information about the first function call call = function_calls[0] @@ -722,19 +722,19 @@ def traced_gemini_live(operation: str) -> Callable: elif operation == "llm_response" and args: # Extract usage and response metadata from turn complete event - evt = args[0] if args else None - if evt and hasattr(evt, "usageMetadata") and evt.usageMetadata: - usage = evt.usageMetadata + msg = args[0] if args else None + if msg and hasattr(msg, "usage_metadata") and msg.usage_metadata: + usage = msg.usage_metadata # Token usage - basic attributes for span visibility - if hasattr(usage, "promptTokenCount"): - operation_attrs["tokens.prompt"] = usage.promptTokenCount or 0 - if hasattr(usage, "responseTokenCount"): + if hasattr(usage, "prompt_token_count"): + operation_attrs["tokens.prompt"] = usage.prompt_token_count or 0 + if hasattr(usage, "response_token_count"): operation_attrs["tokens.completion"] = ( - usage.responseTokenCount or 0 + usage.response_token_count or 0 ) - if hasattr(usage, "totalTokenCount"): - operation_attrs["tokens.total"] = usage.totalTokenCount or 0 + if hasattr(usage, "total_token_count"): + operation_attrs["tokens.total"] = usage.total_token_count or 0 # Get output text and modality from service state text = getattr(self, "_bot_text_buffer", "") @@ -751,9 +751,9 @@ def traced_gemini_live(operation: str) -> Callable: # Add turn completion status if ( - evt - and hasattr(evt, "serverContent") - and evt.serverContent.turnComplete + msg + and hasattr(msg, "server_content") + and msg.server_content.turn_complete ): operation_attrs["turn_complete"] = True @@ -772,16 +772,16 @@ def traced_gemini_live(operation: str) -> Callable: # For llm_response operation, also handle token usage metrics if operation == "llm_response" and hasattr(self, "start_llm_usage_metrics"): - evt = args[0] if args else None - if evt and hasattr(evt, "usageMetadata") and evt.usageMetadata: - usage = evt.usageMetadata + msg = args[0] if args else None + if msg and hasattr(msg, "usage_metadata") and msg.usage_metadata: + usage = msg.usage_metadata # Create LLMTokenUsage object from pipecat.metrics.metrics import LLMTokenUsage tokens = LLMTokenUsage( - prompt_tokens=usage.promptTokenCount or 0, - completion_tokens=usage.responseTokenCount or 0, - total_tokens=usage.totalTokenCount or 0, + prompt_tokens=usage.prompt_token_count or 0, + completion_tokens=usage.response_token_count or 0, + total_tokens=usage.total_token_count or 0, ) _add_token_usage_to_span(current_span, tokens) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index fda6fa63e..7d9ebec6f 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -11,6 +11,7 @@ import unittest from pipecat.frames.frames import ( CancelFrame, EndFrame, + ErrorFrame, Frame, HeartbeatFrame, InputAudioRawFrame, @@ -253,7 +254,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): try: await asyncio.wait_for( - asyncio.shield(task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))), + task.run(PipelineTaskParams(loop=asyncio.get_event_loop())), timeout=1.0, ) except asyncio.TimeoutError: @@ -289,7 +290,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.queue_frame(TextFrame(text="Hello!")) try: await asyncio.wait_for( - asyncio.shield(task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))), + task.run(PipelineTaskParams(loop=asyncio.get_event_loop())), timeout=1.0, ) except asyncio.TimeoutError: @@ -300,11 +301,8 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): identity = IdentityFilter() pipeline = Pipeline([identity]) task = PipelineTask(pipeline, idle_timeout_secs=0.2) - try: - await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) - assert False - except asyncio.CancelledError: - assert True + # This shouldn't freeze, so nothing to check really. + await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) async def test_no_idle_task(self): identity = IdentityFilter() @@ -312,7 +310,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False) try: await asyncio.wait_for( - asyncio.shield(task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))), + task.run(PipelineTaskParams(loop=asyncio.get_event_loop())), timeout=0.3, ) except asyncio.TimeoutError: @@ -331,11 +329,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): ), idle_timeout_secs=0.3, ) - try: - await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) - assert False - except asyncio.CancelledError: - assert True + await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) async def test_idle_task_event_handler_no_frames(self): identity = IdentityFilter() @@ -350,11 +344,8 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): idle_timeout = True await task.cancel() - try: - await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) - assert False - except asyncio.CancelledError: - assert idle_timeout + await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) + assert idle_timeout async def test_idle_task_event_handler_quiet_user(self): identity = IdentityFilter() @@ -415,12 +406,15 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): asyncio.create_task(delayed_frames()), ] - await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + _, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) diff_time = time.time() - start_time self.assertGreater(diff_time, sleep_time_secs * 3) + # Wait for the pending tasks to complete. + await asyncio.gather(*pending) + async def test_task_cancel_timeout(self): class CancelFilter(FrameProcessor): def __init__(self, **kwargs): @@ -450,3 +444,34 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) except asyncio.CancelledError: assert cancelled + + async def test_task_error(self): + class ErrorProcessor(FrameProcessor): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TextFrame): + await self.push_error(ErrorFrame("Boo!")) + + await self.push_frame(frame, direction) + + error_received = False + + pipeline = Pipeline([ErrorProcessor()]) + task = PipelineTask(pipeline) + + @task.event_handler("on_pipeline_error") + async def on_pipeline_error(task: PipelineTask, frame: ErrorFrame): + nonlocal error_received + error_received = True + await task.cancel() + + await task.queue_frame(TextFrame(text="Hello from Pipecat!")) + + try: + await task.run(PipelineTaskParams(loop=asyncio.get_event_loop())) + except asyncio.CancelledError: + assert error_received diff --git a/tests/test_service_switcher.py b/tests/test_service_switcher.py index bf80d842e..83d2d226b 100644 --- a/tests/test_service_switcher.py +++ b/tests/test_service_switcher.py @@ -7,10 +7,12 @@ """Unit tests for ServiceSwitcher and related components.""" import unittest +from dataclasses import dataclass from pipecat.frames.frames import ( Frame, ManuallySwitchServiceFrame, + SystemFrame, TextFrame, ) from pipecat.pipeline.pipeline import Pipeline @@ -52,6 +54,13 @@ class MockFrameProcessor(FrameProcessor): self.frame_count = 0 +@dataclass +class DummySystemFrame(SystemFrame): + """A dummy system frame for testing purposes.""" + + text: str = "" + + class TestServiceSwitcherStrategyManual(unittest.IsolatedAsyncioTestCase): """Test cases for ServiceSwitcherStrategyManual.""" @@ -140,14 +149,22 @@ class TestServiceSwitcher(unittest.IsolatedAsyncioTestCase): # Send some test frames frames_to_send = [ TextFrame(text="Hello 1"), + DummySystemFrame(text="System Message 1"), TextFrame(text="Hello 2"), + DummySystemFrame(text="System Message 2"), TextFrame(text="Hello 3"), ] await run_test( switcher, frames_to_send=frames_to_send, - expected_down_frames=[TextFrame, TextFrame, TextFrame], + expected_down_frames=[ + DummySystemFrame, + DummySystemFrame, + TextFrame, + TextFrame, + TextFrame, + ], expected_up_frames=[], # Expect no error frames ) @@ -156,7 +173,13 @@ class TestServiceSwitcher(unittest.IsolatedAsyncioTestCase): text_frames = [f for f in self.service1.processed_frames if isinstance(f, TextFrame)] self.assertEqual(len(text_frames), 3) - # Check that other services don't receive text frames (they might get StartFrame/EndFrame) + # Only service1 should have processed the system frames + system_frames = [ + f for f in self.service1.processed_frames if isinstance(f, DummySystemFrame) + ] + self.assertEqual(len(system_frames), 2) + + # Check that other services don't receive text frames (they still get StartFrame/EndFrame) service2_text_frames = [ f for f in self.service2.processed_frames if isinstance(f, TextFrame) ] @@ -166,10 +189,24 @@ class TestServiceSwitcher(unittest.IsolatedAsyncioTestCase): self.assertEqual(len(service2_text_frames), 0) self.assertEqual(len(service3_text_frames), 0) + # Check that other services don't receive dummy system frames (they still get StartFrame/EndFrame) + service2_system_frames = [ + f for f in self.service2.processed_frames if isinstance(f, DummySystemFrame) + ] + service3_system_frames = [ + f for f in self.service3.processed_frames if isinstance(f, DummySystemFrame) + ] + self.assertEqual(len(service2_system_frames), 0) + self.assertEqual(len(service3_system_frames), 0) + # Verify the actual text frames processed for i, frame in enumerate(text_frames): self.assertEqual(frame.text, f"Hello {i + 1}") + # Verify the actual system frames processed + for i, frame in enumerate(system_frames): + self.assertEqual(frame.text, f"System Message {i + 1}") + async def test_service_switching(self): """Test that after service switching using ManuallySwitchServiceFrame, the new active service receives frames while others don't.""" switcher = ServiceSwitcher(self.services, ServiceSwitcherStrategyManual) diff --git a/uv.lock b/uv.lock index bef2f3a60..129491a61 100644 --- a/uv.lock +++ b/uv.lock @@ -410,18 +410,16 @@ wheels = [ [[package]] name = "aws-sdk-bedrock-runtime" -version = "0.0.2" +version = "0.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "smithy-aws-core", marker = "python_full_version >= '3.12'" }, - { name = "smithy-aws-event-stream", marker = "python_full_version >= '3.12'" }, + { name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, - { name = "smithy-json", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/8d/52ba543d5d2dbafbbb762ed7d87afd1b86d0e9abb6309d5956fcc92debf7/aws_sdk_bedrock_runtime-0.0.2.tar.gz", hash = "sha256:7a45752060713fccdc4ae560d34666c225c937e798f90fd1739566431e3c79dc", size = 76377, upload-time = "2025-04-09T20:37:21.192Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/78/48574454b3cac869df67665e4a403ebfc3abfcfba2c2ff01ccfd67d55f8f/aws_sdk_bedrock_runtime-0.1.1.tar.gz", hash = "sha256:c896f99e675c3a1ab600633a07b785f3dc9fe8ab94f640b1f992b63da2dfc784", size = 82446, upload-time = "2025-10-21T20:25:25.845Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/1e/cd601cf90c7344dbfaeb6cf4d23c001cfe001ea212e681be16bab134cf50/aws_sdk_bedrock_runtime-0.0.2-py3-none-any.whl", hash = "sha256:4d954d103e8e2d304e1c87ccb8c4c77fae27f4c31b70c0ca50817eb30423f9f7", size = 72205, upload-time = "2025-04-09T20:37:19.956Z" }, + { url = "https://files.pythonhosted.org/packages/83/07/62c0b70223d178c138f29124ac2f7973a6ba803abc7735b6a01a85217f3d/aws_sdk_bedrock_runtime-0.1.1-py3-none-any.whl", hash = "sha256:c0336b377b2112cf88197d3d44302fbeb3efb1101989fa49ae55e78f49cfe345", size = 74954, upload-time = "2025-10-21T20:25:24.973Z" }, ] [[package]] @@ -435,31 +433,31 @@ wheels = [ [[package]] name = "awscrt" -version = "0.28.1" +version = "0.28.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a0/1c/5c9e6a7375c2a1355aadeb2d06c96c95934ec37ff29ebaab2919f59c3ff1/awscrt-0.28.1.tar.gz", hash = "sha256:70a28fd6ff3e0abb7854ea8a9133bc9e5de681a0d9bdbd8a599a23d13a448685", size = 37956730, upload-time = "2025-09-19T00:58:31.564Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/1b/a885a699217967c3ff0e1c49ac5b1e2a050d1a8b87d1e85e958a56e3d3f5/awscrt-0.28.2.tar.gz", hash = "sha256:9715a888f2042e710dc8aeb355963a29b77e7a4cc25a14659cebd21a5fa476c1", size = 37894849, upload-time = "2025-10-14T19:06:16.867Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/75/dd62276f2907a9ffcf9f8f780c08ce9938bd0550a15c887db198b47f24d3/awscrt-0.28.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:47f885104065918d311102e2b08b943966717c0f3b0c5de5908d2fd08de32198", size = 3376838, upload-time = "2025-09-19T00:57:32.988Z" }, - { url = "https://files.pythonhosted.org/packages/a7/93/562709cdf13a7606548426ecc31326ba3f6839f91e98a1e9230208308afb/awscrt-0.28.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3df2316e77ad88c456b7eb2c9928007d379ed892154c1969d35b98653617e576", size = 3821522, upload-time = "2025-09-19T00:57:35.456Z" }, - { url = "https://files.pythonhosted.org/packages/43/f0/6c6ff81f5a4c6d085eb450854149087bf9240c37c467c747521f47901b32/awscrt-0.28.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3a060d930939f142345f46a344e19ffc0dada657b04d02216b8adffba550c0a0", size = 4087344, upload-time = "2025-09-19T00:57:36.62Z" }, - { url = "https://files.pythonhosted.org/packages/37/0a/71c097505add4ceea4ac05153311715acb7489cd82ec69db4570130f4698/awscrt-0.28.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43f81ca6bfe85c38ad9765605aaaa646a1ed6fd7210dbedf67c113dd245f425e", size = 3745148, upload-time = "2025-09-19T00:57:38Z" }, - { url = "https://files.pythonhosted.org/packages/79/1b/2b02b705a47b64e6c4d401087ddd30d4ad9af70172812ae8c62fb2b7a70c/awscrt-0.28.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fc8e2307d9dbe76842015a14701ff7e9cf2619d674621b2d55b769414e17b3fc", size = 3972439, upload-time = "2025-09-19T00:57:39.74Z" }, - { url = "https://files.pythonhosted.org/packages/f1/19/429c81c7a0d81a5edce9cc6d9a878c8b65d8b5b69fa5a2725a6e0b1380c1/awscrt-0.28.1-cp310-cp310-win32.whl", hash = "sha256:6e7b094587e5332d428300340dcc18794a1fcfa76d636f216fc0f5c8405ba604", size = 3915231, upload-time = "2025-09-19T00:57:41.096Z" }, - { url = "https://files.pythonhosted.org/packages/83/81/769ad51fc6dcfd8bf9e0aa59c252013da0eb9e32c050ecbd1fc25f71689a/awscrt-0.28.1-cp310-cp310-win_amd64.whl", hash = "sha256:ac02f10f7384fdb68187f8d5d94743a271b16fa94be81481ce7684942f6a4b35", size = 4051668, upload-time = "2025-09-19T00:57:42.696Z" }, - { url = "https://files.pythonhosted.org/packages/9e/55/0ee537d146f24d6e76eaf02d462a83c572788233603bb9bda969fbf23307/awscrt-0.28.1-cp311-abi3-macosx_10_15_universal2.whl", hash = "sha256:cb36052f9aa34e77687a8037559bbea331fc9d5d77cd71ab0cf4e6d72af73f72", size = 3376673, upload-time = "2025-09-19T00:57:43.875Z" }, - { url = "https://files.pythonhosted.org/packages/f0/54/12700a4b9545680baa3e2d4d0e543bb4775a639df56ee51cbb29b71e0947/awscrt-0.28.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc59829152a5806eb2708aca5c5084c11dd18ecbe765e03eb314d5a360eeaa62", size = 3782870, upload-time = "2025-09-19T00:57:45.737Z" }, - { url = "https://files.pythonhosted.org/packages/1d/e7/7b189ace9e187b9b55ed4a6ec9a451579b2f16bd01d402f79a19cc8e1603/awscrt-0.28.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2f20bc774599b9d85ce66689415da529ddd1d2215da818e005deedc4688fe61", size = 4048789, upload-time = "2025-09-19T00:57:47.327Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e0/2e5472019906dfcc5fadcdba4bad9e69dabb95bbc0c110cfe555ee8461dc/awscrt-0.28.1-cp311-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:491b8b9c73a288cfd5e0cbdac16aabb5313d5cfc33bbe461763a5ddc26624f70", size = 3687832, upload-time = "2025-09-19T00:57:48.563Z" }, - { url = "https://files.pythonhosted.org/packages/71/f2/7e05d371bb888ee9f15e83d189287838f7b6ea40dfc91eacb3acd24b8529/awscrt-0.28.1-cp311-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:4c6c7125b7e9fcc999eb685d1cace8d4f2ffc63f8f3d8ef7f77e1a97d9552863", size = 3913378, upload-time = "2025-09-19T00:57:50.185Z" }, - { url = "https://files.pythonhosted.org/packages/79/6b/a542a65a22edb85d64742970c21721e66e0f9f67911a11c7a5c3626a1b17/awscrt-0.28.1-cp311-abi3-win32.whl", hash = "sha256:1dcb33d7cf8f69881ac6ef75a5b9b40816be58678b1bb07ccbe0230281bdbc81", size = 3912809, upload-time = "2025-09-19T00:57:51.797Z" }, - { url = "https://files.pythonhosted.org/packages/df/64/16cc8a0011e3ca5dda13605befa7e6db29bfb3073c67f6e8dad90be0a8ae/awscrt-0.28.1-cp311-abi3-win_amd64.whl", hash = "sha256:670caaf556876913bcfb9d8183d43d67a6c7b52998f2f398abd1c21632a006f8", size = 4048979, upload-time = "2025-09-19T00:57:53.061Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/debbd3a2f03c5953b56b1c3b321bab16293f857ea3005e3f7e5dded5e0b2/awscrt-0.28.1-cp313-abi3-macosx_10_15_universal2.whl", hash = "sha256:22311d25135b937ee5617e35a6554961727527dcfa3e06efdefe187a6abe65c4", size = 3375565, upload-time = "2025-09-19T00:57:54.598Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4f/9388917ad45c043acd7c4ab2c28b9e2b5ddf29e21a82bfc01a7626c18c04/awscrt-0.28.1-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e58740cf0e41552fdf7909e10814b312ab090ebe54741354a61507e0c6d4ebfd", size = 3775366, upload-time = "2025-09-19T00:57:56.238Z" }, - { url = "https://files.pythonhosted.org/packages/8a/e3/3ef301cdef76b22ce14b041e04c6cf65ba4491d00e9f5b400c0699f6c63e/awscrt-0.28.1-cp313-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e69f163a207a8b172abbfea1f51045301ed1ac8bbaf76958a6b5e81d72e5b89", size = 4043403, upload-time = "2025-09-19T00:57:57.4Z" }, - { url = "https://files.pythonhosted.org/packages/60/9c/4f89922333724c4da851752549ca97dd147420734ef6c4ece56d5dd65e09/awscrt-0.28.1-cp313-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:592f4b234ecafa6cde86e55e42c4fe84c4e1ffe9fb11b0a8b8f0ffb8c62fa2cc", size = 3678742, upload-time = "2025-09-19T00:57:59.055Z" }, - { url = "https://files.pythonhosted.org/packages/0e/d4/adb97ba5f888ed201aa1f9e9f8d6cfc0dbaf80f0e937b3acb7411febdaa8/awscrt-0.28.1-cp313-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b16321f1d2bf5b4991a213059c1b5dc07954edfc424d154b093824465ec94ce2", size = 3908438, upload-time = "2025-09-19T00:58:00.71Z" }, - { url = "https://files.pythonhosted.org/packages/41/ac/600ea0a6f4ba6543c50417c8e78b09f2cd73dd0f0d4c3e9e52220a8badbe/awscrt-0.28.1-cp313-abi3-win32.whl", hash = "sha256:3e0a23635aa75b4af163ff9bf5a0873928369b1ac32c8b1351741a95472ccf71", size = 3907625, upload-time = "2025-09-19T00:58:03.235Z" }, - { url = "https://files.pythonhosted.org/packages/9e/24/d22c7197b1e53c76b5eb71d640a4728b9b7621075d8dbcc054e16b5b98f0/awscrt-0.28.1-cp313-abi3-win_amd64.whl", hash = "sha256:9849c88ca0830396724acf988e2759895118fe7dd2a23dab21978c8600d01a11", size = 4043878, upload-time = "2025-09-19T00:58:04.595Z" }, + { url = "https://files.pythonhosted.org/packages/73/b4/1a566e493bdfa6e918ba78bcd2e45dda99a25407a4fd974db2666228d154/awscrt-0.28.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:bec19c0dd780293a26c809aabb9f7675b28cb3a1bf05b4a5bc9f28d5ced75a81", size = 3380735, upload-time = "2025-10-14T19:05:16.58Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/6602a87aead1d413c7bd77d059b301745146635cda99ee2a61ec0d23691e/awscrt-0.28.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01f33076759ba6285f25ccc6016355607df2e715d0bab3a1ef2416b87a6c3ade", size = 3827084, upload-time = "2025-10-14T19:05:19.335Z" }, + { url = "https://files.pythonhosted.org/packages/d8/62/61fe39ae5950ad00e10dcbf6e4f4f344dc93957757160c0000390331a11b/awscrt-0.28.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b5c807b9972795ce54c05aea6918c60983c51d879ebbff7a67adb8b0d28a121", size = 4092678, upload-time = "2025-10-14T19:05:20.8Z" }, + { url = "https://files.pythonhosted.org/packages/25/7d/e38f18cfb203e8f09842c0e3f422992887ce285ecc3bf18816d559a13c80/awscrt-0.28.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf4ff9c8c6a233246320c2d41d939b6e25cdae97728d827186e4771a9edda688", size = 3749978, upload-time = "2025-10-14T19:05:22.16Z" }, + { url = "https://files.pythonhosted.org/packages/16/6f/e8a3c0daed8f7b60c76fc2721bd4e83580ddecace24e0cb0ebb99564f699/awscrt-0.28.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c738b83b66d1a8b43089556247fbe4adf2b73d610c7938d3bae1718a0fe8b1d", size = 3977237, upload-time = "2025-10-14T19:05:23.368Z" }, + { url = "https://files.pythonhosted.org/packages/92/3d/8400203f02dd924bcc8255703179b0c26efd03c84f838db6f026fcef9ba6/awscrt-0.28.2-cp310-cp310-win32.whl", hash = "sha256:23c30004c736a2f826a32c9720f1ccf71e8e4deb8535da5915d6073604853098", size = 3919413, upload-time = "2025-10-14T19:05:24.477Z" }, + { url = "https://files.pythonhosted.org/packages/c0/5e/b5ccf377880a70425b100f1e5f5ba516ff75e291585b3dc129239fbd1ec3/awscrt-0.28.2-cp310-cp310-win_amd64.whl", hash = "sha256:859ae8a195d51f15b631147d6792953a563bfe0a1cc7a75b6750977634de54b8", size = 4056024, upload-time = "2025-10-14T19:05:25.956Z" }, + { url = "https://files.pythonhosted.org/packages/ed/79/94e9f0ee7c60ec6233c7ad6293589c56d5145172e49eb5328eda37d3fdd1/awscrt-0.28.2-cp311-abi3-macosx_10_15_universal2.whl", hash = "sha256:025eab99b58586d8c95f8fafe1f4695ad477eda20d1207240ee4f8ee79742059", size = 3381061, upload-time = "2025-10-14T19:05:27.187Z" }, + { url = "https://files.pythonhosted.org/packages/2d/b8/0da80dd58682ddf3ec204e877d5891198654647c085e65b6b8eacd214edb/awscrt-0.28.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e5c18d035d6cd92228e1db2f043517c1bcf9e0f6430c0af60cc34257dcca092c", size = 3788011, upload-time = "2025-10-14T19:05:28.768Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d2/f51cf4364364399fe90d557e2fed14c1f114720191a5825898b1242bd607/awscrt-0.28.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c75f077e90d0220a49b75a9bca914e5aa1a3c8f28af6bce4d0332be0b98dd3cb", size = 4055226, upload-time = "2025-10-14T19:05:30.054Z" }, + { url = "https://files.pythonhosted.org/packages/41/47/0fde8738a8c76de278ce431d8468ef18aeaca424329decca9ad5092df812/awscrt-0.28.2-cp311-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1432c5c59a7e36b33eb2746cfbf30058f19ed43f2c117863897681f70bc246ba", size = 3692839, upload-time = "2025-10-14T19:05:31.471Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/cb3762f6b47fe503eea7f337eca7cfd044ab28bcc2452fbf298c6492ec8b/awscrt-0.28.2-cp311-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f96703c30b22ba1e43e1bb2fe996ac7af513bea411c54dbf09a3a1af329b9a76", size = 3918023, upload-time = "2025-10-14T19:05:33.162Z" }, + { url = "https://files.pythonhosted.org/packages/95/0a/0b609acd45dbb83c04c7ecb8c7c789f5c15bbdd422129360bde093bc4a99/awscrt-0.28.2-cp311-abi3-win32.whl", hash = "sha256:3e94f63497b454d30892d7a7ce917a451c6f33590964d3a475d93f93b20083b6", size = 3917048, upload-time = "2025-10-14T19:05:34.745Z" }, + { url = "https://files.pythonhosted.org/packages/d1/38/bf33abd6d09c8572f8e09488db2b0a60124767d7f5d6d9a33cf8b051b7af/awscrt-0.28.2-cp311-abi3-win_amd64.whl", hash = "sha256:3e094772b1f6fd0f8c5f7cf37655d0984739f99493f66f534979a2a7bb7fc9f6", size = 4052877, upload-time = "2025-10-14T19:05:36.01Z" }, + { url = "https://files.pythonhosted.org/packages/10/71/4be198e472d95702434cee1f9dd889c56e22bea8554b466fad754148fd24/awscrt-0.28.2-cp313-abi3-macosx_10_15_universal2.whl", hash = "sha256:5fda9e7d0eb800491fadebe2b6c2560ac2f5742b60f4106440dca4b49da7fb03", size = 3379585, upload-time = "2025-10-14T19:05:37.225Z" }, + { url = "https://files.pythonhosted.org/packages/43/09/77084249d07dca71352341ad3fbcfa75deaccf25bd65f9fdbb36ce1f978b/awscrt-0.28.2-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:994a795bdc83344922a15891abb30155ec292093e856eef3929dd63dd6cadaca", size = 3779843, upload-time = "2025-10-14T19:05:38.774Z" }, + { url = "https://files.pythonhosted.org/packages/a6/bb/fcee9365e58e5860582398317571a9a5517da258cd81c3d987b9882f61d4/awscrt-0.28.2-cp313-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28537c4517168927ef74aa007a2e0c9f436921227934d82da31e9a1cec7e0c4a", size = 4049154, upload-time = "2025-10-14T19:05:40.301Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8e/ac92b2707dbe05e56d0dd5af73cb4e07a3da4aee66936071123966523759/awscrt-0.28.2-cp313-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:b9fc6be63832da3ff244d56c7d9a43326d89d79e68162419c35f33e6ad033be0", size = 3683672, upload-time = "2025-10-14T19:05:41.536Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d0/15308ec37e762691f5d1871b0f1a6e462da8e421c6c38d6724e3cf0994b2/awscrt-0.28.2-cp313-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:efb57103a368de1d33148cb70a382c4f82ac376c744de9484e0f621cef8313f3", size = 3912823, upload-time = "2025-10-14T19:05:43.781Z" }, + { url = "https://files.pythonhosted.org/packages/bc/cd/7693b1d72069908b7a3ee30e4ef2b5fc8f54948a96397729277cb0b0c7b4/awscrt-0.28.2-cp313-abi3-win32.whl", hash = "sha256:594dc61f4f0c1c9fb7292364d25c21810b3608cd67c0de78a032ad48f7bfd88c", size = 3911514, upload-time = "2025-10-14T19:05:45.019Z" }, + { url = "https://files.pythonhosted.org/packages/93/d6/5d8545c967690f03d55d44ed56ceff26d88363cd7d0435fd80a1c843ac2a/awscrt-0.28.2-cp313-abi3-win_amd64.whl", hash = "sha256:a17f0ab9dc5e5301da0fb00ccc4511a136d13abbd4a9564827547333fcd7ba16", size = 4047912, upload-time = "2025-10-14T19:05:46.302Z" }, ] [[package]] @@ -571,6 +569,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/54/db7a801933dd2537f5376fb8a9e28caff488ef5c2d61f3a8fced55fe6336/blake3-1.0.7-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:d9046bb1e22a8607e1d0d7c3ff47e56e0a197c988502df4bf4d78563f3e9fe2c", size = 553411, upload-time = "2025-09-29T16:40:45.667Z" }, { url = "https://files.pythonhosted.org/packages/2c/08/949cf68d16d1f731d502968bb1486e1a4bf7ef032c38fbc2ef26a2353494/blake3-1.0.7-cp313-cp313t-win32.whl", hash = "sha256:bd2f638bcc00fc09ce985ea3c642d45940e1eda198ab1f4b90cfdecbebbc9315", size = 227049, upload-time = "2025-09-29T16:40:47.446Z" }, { url = "https://files.pythonhosted.org/packages/f2/ae/6783a5ca6235024e00a1e92ab6ca2cd855f4c61c763cf8d6d643846d110c/blake3-1.0.7-cp313-cp313t-win_amd64.whl", hash = "sha256:cb3aa1db14231c2ef0ec5acd805505ce128c39ffa510deb3384eed96fe4addcb", size = 214101, upload-time = "2025-09-29T16:40:48.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/aa/99b4b6c22972b9a854f77d97846a717448a77d079e4bd38e46a3f8ecea76/blake3-1.0.7-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:f7db997205aa420d59fb5639346e40beafb9c09252e2ec6efedca8f230f7520c", size = 346664, upload-time = "2025-10-11T18:02:54.609Z" }, + { url = "https://files.pythonhosted.org/packages/f9/44/e98bc5450be415a335a191b154e299e335046d11fe9514d93961902b7aed/blake3-1.0.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:19afec6e276f3bc154541248d92b1ecb198af2ee920025f7ce521028f9a69d8b", size = 324576, upload-time = "2025-10-11T18:02:57.062Z" }, + { url = "https://files.pythonhosted.org/packages/74/25/23a39913c8424ac3df705ed71a00efe34cc1cdbd4588ed6eaf458ea9d7ef/blake3-1.0.7-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:006a11bbba65a95e88ddc069cca751c8812fd144d582715eeea512452fdbe80d", size = 370545, upload-time = "2025-10-11T18:02:59.824Z" }, + { url = "https://files.pythonhosted.org/packages/db/83/9f53a86de9a5999b043febfd84765d240014da42055aeac06d1005b20b07/blake3-1.0.7-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7febeffdc8412fed105ca517cee641ac521fb9cfb750bf7e27a5cdf3ddf74a08", size = 374370, upload-time = "2025-10-11T18:03:01.412Z" }, + { url = "https://files.pythonhosted.org/packages/c4/4c/3290aa4fb7483975a7b3322a73692aa3cf491a77ce7ac61c216c71c6f834/blake3-1.0.7-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c032ce7c52b71015651c0abe9fe599aa2669e6be578aa17d5f993dc93373401", size = 447808, upload-time = "2025-10-11T18:03:02.893Z" }, + { url = "https://files.pythonhosted.org/packages/66/26/92b6e15552865416aae1aedad8b9b4d8b47ca9b73d25373622b1798c05a9/blake3-1.0.7-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b81455f7d24b58fe26be037cc3854c28ea6eb3671ceab3b1ec0b1239aeb6fef", size = 506118, upload-time = "2025-10-11T18:03:04.51Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ef/f158fc43a03fd366bc428a52a845bd0f884e518deda901c9216bd469867e/blake3-1.0.7-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:41b0127b0e7c8610054c421959dbe7140a81ac2c88fa9e099994fbaa529af3c1", size = 393239, upload-time = "2025-10-11T18:03:07.102Z" }, + { url = "https://files.pythonhosted.org/packages/10/49/2a56ce897ec7ed0e25953b3873da271ea60cc107ae02ecc6655252e554c7/blake3-1.0.7-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4755ca95b4114b629d8f3570bc661916d211d52d47f57ff70e9687377ab39cb9", size = 386267, upload-time = "2025-10-11T18:03:08.904Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c4/ee4c03ea419198b91c889ef173015b5d637a390d3f7d63cb70033a7201d6/blake3-1.0.7-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:8abe929cfd27b375e02e3dd7a690192fa4efecc52ef510df91ef01651ef08dc7", size = 549641, upload-time = "2025-10-11T18:03:10.64Z" }, + { url = "https://files.pythonhosted.org/packages/b2/cc/a918d6649b56fe705133e06d9958d90978aad30063d42cca4dfe23db16e9/blake3-1.0.7-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:dd607eb5ad5a9b44ff62243759aa0af4085f6f43c9b01f503561a70da63e3b94", size = 553691, upload-time = "2025-10-11T18:03:12.108Z" }, + { url = "https://files.pythonhosted.org/packages/fd/9f/568546f555fd1555d4867c497e9413f67bf769d076e773b9ca9e07a0b6f6/blake3-1.0.7-cp314-cp314-win32.whl", hash = "sha256:a51684d1f346e7680f7c244c25b0e279e3b297f1938126e4ea8e32425ea269f5", size = 227552, upload-time = "2025-10-11T18:03:13.468Z" }, + { url = "https://files.pythonhosted.org/packages/97/2b/d4ef7365d9f601c8a127b5993f2662d45d2cb6d430bf3dbbb7a6f0b33639/blake3-1.0.7-cp314-cp314-win_amd64.whl", hash = "sha256:a6a481719e28e2c61aafd4273d32663365d97613341b72fcdf2f6afbd426319b", size = 214719, upload-time = "2025-10-11T18:03:14.835Z" }, + { url = "https://files.pythonhosted.org/packages/2f/53/f697cc34e382a225d163ea0c6a35c7eb4cfd1011e85db6610adfac98e522/blake3-1.0.7-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:daa8933cd7db19143bd6b59f7ac4c7c7446767d7b2c3a748a4559aa483275fa2", size = 347071, upload-time = "2025-10-11T18:03:16.637Z" }, + { url = "https://files.pythonhosted.org/packages/4c/85/836dcb5c5709c2331f02ce065f7ebfaae710a6c1768cdc47ee3197645f98/blake3-1.0.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:24074adfffffe0fa7a7dd930cc608d6e965e70306e2c1e14d412e29ec94fa360", size = 324341, upload-time = "2025-10-11T18:03:18.073Z" }, + { url = "https://files.pythonhosted.org/packages/6d/48/36b2c25007933619ce60e24b9f360baaa77d08939284045476c8e157fe62/blake3-1.0.7-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dce6e6f03de2674f9860cf330d8a4fcdb63a60659435e5e31d72d174fc102d8e", size = 370140, upload-time = "2025-10-11T18:03:19.582Z" }, + { url = "https://files.pythonhosted.org/packages/70/82/8a8977e5d56b9fb719033940c8ce34afc733190d34ab868a647a9af7b584/blake3-1.0.7-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e783f33d53a2de8d2ab845235dd53393d521b5e4a76c23d03e77e472266359d3", size = 373022, upload-time = "2025-10-11T18:03:21.143Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/44017ba40804a528568b35a36c05187786830c4d891c5540d59a121a7cec/blake3-1.0.7-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:782784aef18eb61f4ce8bf2b9506b7d90f0d183176b453345b221837a18041b7", size = 447243, upload-time = "2025-10-11T18:03:22.707Z" }, + { url = "https://files.pythonhosted.org/packages/78/c1/4fa20e68624784082734d31b8c9c80ad226658c024e61b9f9b6751ba0a4a/blake3-1.0.7-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6062122e77f40e3733cac2ef3f25e0fc7f555e352fe6f513f8404ad11dc69974", size = 506149, upload-time = "2025-10-11T18:03:24.424Z" }, + { url = "https://files.pythonhosted.org/packages/8e/63/af65466e27e7b92800a068afaee11b2fa071e34a7f5900f8e13832f18185/blake3-1.0.7-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c2614bc9d69fd6067571f3bb37b3b07a6b86a56167553ad4784a3c508771f39", size = 393243, upload-time = "2025-10-11T18:03:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/54a4807a3243d0e094ada9d65687aeb40059587e374b3beb9c89f6552c9b/blake3-1.0.7-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6df2bd56c43bdeb6699d4af0a0dd0d77537d95cb4a5dde4b39ed6e54cc725d6", size = 386318, upload-time = "2025-10-11T18:03:27.338Z" }, + { url = "https://files.pythonhosted.org/packages/42/e8/32b56531b5d9da67e476735ceaec7c3bf89310629abeeafb03c724145c88/blake3-1.0.7-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:8b635cf4350caf459ecb335b32be622068423245bda457d5bc159106eb20f912", size = 548945, upload-time = "2025-10-11T18:03:28.779Z" }, + { url = "https://files.pythonhosted.org/packages/ad/50/33b1aca708be629e285a537f1adf34dfcabc4c30b28c436361323d11f593/blake3-1.0.7-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:f96a685775f87ddf75ff495dc9698703268c66c170caca977347427ef8d52324", size = 553564, upload-time = "2025-10-11T18:03:30.247Z" }, + { url = "https://files.pythonhosted.org/packages/fe/07/8b17cbf40ccd9afeed6ae9f55018181786b30ff4e079ac8bf4ca4799e47b/blake3-1.0.7-cp314-cp314t-win32.whl", hash = "sha256:0633b7d9bad87dc7fce545042353f2e056604d993f71d1dce666a9f5edc13e05", size = 227345, upload-time = "2025-10-11T18:03:31.933Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8a/ab9de8a73616350759356a483f440212bc2a22fc9aaa77cabbf06c3483db/blake3-1.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:5e356daa0089968dc1ff1d0d112e7cc1700533441d8f30ae99f835a94dc8b0f3", size = 213964, upload-time = "2025-10-11T18:03:33.919Z" }, ] [[package]] @@ -869,16 +891,16 @@ wheels = [ [[package]] name = "compressed-tensors" -version = "0.10.2" +version = "0.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "torch" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/86/d43d369abc81ec63ec7b8f6f27fc8b113ea0fd18a4116ae12063387b8b34/compressed_tensors-0.10.2.tar.gz", hash = "sha256:6de13ac535d7ffdd8890fad3d229444c33076170acaa8fab6bab8ecfa96c1d8f", size = 173459, upload-time = "2025-06-23T13:19:06.135Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/eb/2229523a539e8074b238c225d168f734f6f056ab4ea2278eefe752f4a6f3/compressed_tensors-0.10.1.tar.gz", hash = "sha256:f99ce620ddcf8a657eaa7995daf5faa8e988d4b4cadc595bf2c4ff9346c2c19a", size = 126778, upload-time = "2025-06-06T18:25:16.538Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/ac/56bb4b6b3150783119479e2f05e32ebfc39ca6ff8e6fcd45eb178743b39e/compressed_tensors-0.10.2-py3-none-any.whl", hash = "sha256:e1b4d9bc2006e3fd3a938e59085f318fdb280c5af64688a4792bf1bc263e579d", size = 169030, upload-time = "2025-06-23T13:19:03.487Z" }, + { url = "https://files.pythonhosted.org/packages/5b/07/e70a0b9efc24a32740396c404e7213c62b8aeb4a577ed5a3f191f8d7806b/compressed_tensors-0.10.1-py3-none-any.whl", hash = "sha256:b8890735522c119900e8d4192cced0b0f70a98440ae070448cb699165c404659", size = 116998, upload-time = "2025-06-06T18:25:14.54Z" }, ] [[package]] @@ -1260,13 +1282,13 @@ wheels = [ [[package]] name = "daily-python" -version = "0.19.9" +version = "0.21.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/85/6064c3225e5b190e522e8f3bc6a460efc5e3e6632f16fd5f9799c44ba57a/daily_python-0.19.9-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:cbc558ad7d49e79b550bf7567b9ceae75e2864d4fcaf41c90377b620e38a2461", size = 13365213, upload-time = "2025-09-06T00:31:00.224Z" }, - { url = "https://files.pythonhosted.org/packages/23/58/af986c6881180a46a7b60dd418ce58d6d7c0c4ffc48d261748067c679317/daily_python-0.19.9-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:446bb9ee848d88bc68ca29a2216793c9b5ebaf5991bf604daf76f7c5a53d5919", size = 11711673, upload-time = "2025-09-06T00:31:02.526Z" }, - { url = "https://files.pythonhosted.org/packages/9d/48/1cad4c3e92cdb5ef06467d972c76a510fe5e807513334b10ad7f8c21bf74/daily_python-0.19.9-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2facaf82b614404c642c70bbf0874fb045d8ad46400acb051470cd4df93cb4db", size = 13679393, upload-time = "2025-09-06T00:31:04.999Z" }, - { url = "https://files.pythonhosted.org/packages/3c/e9/354f4699619e83d13e266256b2352b21741ac527e3e5ab5f2264d5c482cd/daily_python-0.19.9-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ffc205efca7b47739efd358febab17577248c8db2ebc4d17d819307a83b9eefc", size = 14221932, upload-time = "2025-09-06T00:31:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/ff/11/99590f8b7aad077f3f9b5b59d39b010aee0bd01b14dece8ae1e93d8080e7/daily_python-0.21.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:bdec96417825181559769bb2258ae688d1215949a1878336194e36fb452274a8", size = 13277066, upload-time = "2025-10-29T00:20:49.523Z" }, + { url = "https://files.pythonhosted.org/packages/e5/db/8c57f1a1b713ba3393584ac2be32d8074d3022a2c2c17c28eb4cd2aa3629/daily_python-0.21.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:18677fa1415a0dc48b891cdf2fb8fe9dabc70e1b019d5aaa3d0699ccc8d187c9", size = 11644908, upload-time = "2025-10-29T00:20:52.106Z" }, + { url = "https://files.pythonhosted.org/packages/64/b6/b03f2f58a367d6ef4bb728715471542fdfa68afa8a177670139c3a2aadb7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:97eb97352fe74227061b678e330b8befcfa4c694feb6eb2b09fe6eacec00ad6d", size = 13652356, upload-time = "2025-10-29T00:20:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/bde65f6f8d4c1679dc6c185fa37dae9223f6ddb4b7ced728ef46504956f7/daily_python-0.21.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:68c3e36f609fc2fce79e4d17ecf1021eadd836506db6c5125f95c682bcf3612a", size = 14304643, upload-time = "2025-10-29T00:20:57.194Z" }, ] [[package]] @@ -1809,7 +1831,7 @@ wheels = [ [[package]] name = "google-cloud-speech" -version = "2.32.0" +version = "2.33.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-api-core", extra = ["grpc"] }, @@ -1817,14 +1839,14 @@ dependencies = [ { name = "proto-plus" }, { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dc/fc/7e47328069850f084ee17e26b5572de067e30fdab862e381702222d237b7/google_cloud_speech-2.32.0.tar.gz", hash = "sha256:89c2618b131d310c6c00e7c04d290ffa9a5d68c20191030766a7737850f04e77", size = 387621, upload-time = "2025-04-14T10:16:35.386Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/74/9c5a556f8af19cab461058aa15e1409e7afa453ca2383473a24a12801ef7/google_cloud_speech-2.33.0.tar.gz", hash = "sha256:fd08511b5124fdaa768d71a4054e84a5d8eb02531cb6f84f311c0387ea1314ed", size = 389072, upload-time = "2025-06-11T23:56:37.231Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/85/a4/f41f2737cd0597f2aa5855b0a12f353fad4506868887590671230df81c77/google_cloud_speech-2.32.0-py3-none-any.whl", hash = "sha256:537b279d8697fe5b5bc5f485f2d48a6b343fc76f73385b5776806c37bc5f8ea1", size = 334148, upload-time = "2025-04-14T10:16:33.89Z" }, + { url = "https://files.pythonhosted.org/packages/12/1d/880342b2541b4bad888ad8ab2ac77d4b5dad25b32a2a1c5f21140c14c8e3/google_cloud_speech-2.33.0-py3-none-any.whl", hash = "sha256:4ba16c8517c24a6abcde877289b0f40b719090504bf06b1adea248198ccd50a5", size = 335681, upload-time = "2025-06-11T23:56:36.026Z" }, ] [[package]] name = "google-cloud-texttospeech" -version = "2.26.0" +version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-api-core", extra = ["grpc"] }, @@ -1832,9 +1854,9 @@ dependencies = [ { name = "proto-plus" }, { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5b/3d/214506e1163138159a3ba172adc0945970843ce9a8c5332db06772806dff/google_cloud_texttospeech-2.26.0.tar.gz", hash = "sha256:43af1b88a6b9becde69a3bbf8aa80cdfa5f12f8999e56bcf9dec374354ed7f6a", size = 181084, upload-time = "2025-04-14T10:16:39.737Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/4b/7ccadbec28ee255a3176c3de0a14705c4b6469777f1c7ddbf4452fa893e3/google_cloud_texttospeech-2.31.0.tar.gz", hash = "sha256:1f0c0c6448f175e1e2f63d96fb13af5d9abee6970bbb22c1e4036f53136a5588", size = 184880, upload-time = "2025-09-25T14:03:22.786Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/eb/fb3a2c16f5612c4a131b2bfa242aaf7800ec0cee479759d9de2cc919ba70/google_cloud_texttospeech-2.26.0-py3-none-any.whl", hash = "sha256:837835aadeb261983d139ef1c5e60c99f80199e22330bf4f62e217360b9e07b8", size = 188122, upload-time = "2025-04-14T10:16:38.466Z" }, + { url = "https://files.pythonhosted.org/packages/18/3e/54ff1a5af26f90c5d76e7e80b9208f8484035b5bd8fb6a06c819fed6a8c9/google_cloud_texttospeech-2.31.0-py3-none-any.whl", hash = "sha256:9442134b4b8e7e3d179dfd3850a5a953a6a6a9cf000a3640caddb85cf97ab69b", size = 191280, upload-time = "2025-09-25T14:03:16.667Z" }, ] [[package]] @@ -1874,7 +1896,7 @@ wheels = [ [[package]] name = "google-genai" -version = "1.24.0" +version = "1.41.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1886,9 +1908,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8d/cf/37ac8cd4752e28e547b8a52765fe48a2ada2d0d286ea03f46e4d8c69ff4f/google_genai-1.24.0.tar.gz", hash = "sha256:bc896e30ad26d05a2af3d17c2ba10ea214a94f1c0cdb93d5c004dc038774e75a", size = 226740, upload-time = "2025-07-01T22:14:24.365Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/8b/ee20bcf707769b3b0e1106c3b5c811507736af7e8a60f29a70af1750ba19/google_genai-1.41.0.tar.gz", hash = "sha256:134f861bb0ace4e34af0501ecb75ceee15f7662fd8120698cd185e8cb39f2800", size = 245812, upload-time = "2025-10-02T22:30:29.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/28/a35f64fc02e599808101617a21d447d241dadeba2aac1f4dc2d1179b8218/google_genai-1.24.0-py3-none-any.whl", hash = "sha256:98be8c51632576289ecc33cd84bcdaf4356ef0bef04ac7578660c49175af22b9", size = 226065, upload-time = "2025-07-01T22:14:23.177Z" }, + { url = "https://files.pythonhosted.org/packages/15/14/e5e8fbca8863fee718208566c4e927b8e9f45fd46ec5cf89e24759da545b/google_genai-1.41.0-py3-none-any.whl", hash = "sha256:111a3ee64c1a0927d3879faddb368234594432479a40c311e5fe4db338ca8778", size = 245931, upload-time = "2025-10-02T22:30:27.885Z" }, ] [[package]] @@ -2728,7 +2750,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "0.3.77" +version = "0.3.79" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -2739,23 +2761,23 @@ dependencies = [ { name = "tenacity" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/cc/786184e5f6a921a2aa4d2ac51d3adf0cd037289f3becff39644bee9654ee/langchain_core-0.3.77.tar.gz", hash = "sha256:1d6f2ad6bb98dd806c6c66a822fa93808d821e9f0348b28af0814b3a149830e7", size = 580255, upload-time = "2025-10-01T14:34:37.368Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/99/f926495f467e0f43289f12e951655d267d1eddc1136c3cf4dd907794a9a7/langchain_core-0.3.79.tar.gz", hash = "sha256:024ba54a346dd9b13fb8b2342e0c83d0111e7f26fa01f545ada23ad772b55a60", size = 580895, upload-time = "2025-10-09T21:59:08.359Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/18/e7462ae0ce57caa9f6d5d975dca861e9a751e5ca253d60a809e0d833eac3/langchain_core-0.3.77-py3-none-any.whl", hash = "sha256:9966dfe3d8365847c5fb85f97dd20e3e21b1904ae87cfd9d362b7196fb516637", size = 449525, upload-time = "2025-10-01T14:34:35.672Z" }, + { url = "https://files.pythonhosted.org/packages/fc/71/46b0efaf3fc6ad2c2bd600aef500f1cb2b7038a4042f58905805630dd29d/langchain_core-0.3.79-py3-none-any.whl", hash = "sha256:92045bfda3e741f8018e1356f83be203ec601561c6a7becfefe85be5ddc58fdb", size = 449779, upload-time = "2025-10-09T21:59:06.493Z" }, ] [[package]] name = "langchain-openai" -version = "0.3.23" +version = "0.3.29" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "openai" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/f1/575120e829430f9bdcfc2c5c4121f04b1b5a143d96e572ff32399b787ef2/langchain_openai-0.3.23.tar.gz", hash = "sha256:73411c06e04bc145db7146a6fcf33dd0f1a85130499dcae988829a4441ddaa66", size = 647923, upload-time = "2025-06-13T14:24:31.388Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/56/2e2010d15118ac52760f92ebf6ce75b3508e7a1023107ea04233fd6263e0/langchain_openai-0.3.29.tar.gz", hash = "sha256:83a0455f8ce874aa1806131ca3b4db08e482be037b7457a9b3ca21a213d2ab47", size = 766499, upload-time = "2025-08-08T15:12:32.402Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/65/88060305d5d627841bc8da7e9fb31fb603e5b103b4e5ec5b4d1a7edfbc3b/langchain_openai-0.3.23-py3-none-any.whl", hash = "sha256:624794394482c0923823f0aac44979968d77fdcfa810e42d4b0abd8096199a40", size = 65392, upload-time = "2025-06-13T14:24:30.263Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f2/a6a73beec15e90605e6a24c4498a8592d79a72c8e81c18ed0f5e9b7308e9/langchain_openai-0.3.29-py3-none-any.whl", hash = "sha256:71ae6791b3e017ec892a8062f993edc882c6665fd8385aa66e9dc3bff8205996", size = 74316, upload-time = "2025-08-08T15:12:30.794Z" }, ] [[package]] @@ -3211,23 +3233,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/b4/b61eeb92c424947675492dec3a411bdbeae307dfd78162d65ab47e8c3b4f/mlx-0.29.2-cp313-cp313-manylinux_2_35_x86_64.whl", hash = "sha256:c3b9a9aee13f346d060966472954eebe99d9f1b295c9a237c9a000f1ef9adf2c", size = 648709, upload-time = "2025-09-26T22:26:03.452Z" }, ] -[[package]] -name = "mlx-lm" -version = "0.28.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "mlx" }, - { name = "numpy" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1c/d7/fdde445c7bd443a2ed23badda6064f1477c4051543922106f365e94082cd/mlx_lm-0.28.2.tar.gz", hash = "sha256:d28752635ed5c89ff2b41361916c928e6b16f765c07b2908044e1dcaf921ed9b", size = 209374, upload-time = "2025-10-02T14:23:57.497Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/1c/89e0f60d45e364de8507065f73aeb8d2fd810d6cb95a9a512880b09399d5/mlx_lm-0.28.2-py3-none-any.whl", hash = "sha256:1501529e625d0d648216f7bb543b8b449d5fd17bd598f635536dbc1fbde6d1d6", size = 284600, upload-time = "2025-10-02T14:23:56.395Z" }, -] - [[package]] name = "mlx-metal" version = "0.29.2" @@ -3853,7 +3858,7 @@ wheels = [ [[package]] name = "openai" -version = "1.74.0" +version = "1.97.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -3865,9 +3870,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/86/c605a6e84da0248f2cebfcd864b5a6076ecf78849245af5e11d2a5ec7977/openai-1.74.0.tar.gz", hash = "sha256:592c25b8747a7cad33a841958f5eb859a785caea9ee22b9e4f4a2ec062236526", size = 427571, upload-time = "2025-04-14T16:45:25.062Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/57/1c471f6b3efb879d26686d31582997615e969f3bb4458111c9705e56332e/openai-1.97.1.tar.gz", hash = "sha256:a744b27ae624e3d4135225da9b1c89c107a2a7e5bc4c93e5b7b5214772ce7a4e", size = 494267, upload-time = "2025-07-22T13:10:12.607Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/91/8c150f16a96367e14bd7d20e86e0bbbec3080e3eb593e63f21a7f013f8e4/openai-1.74.0-py3-none-any.whl", hash = "sha256:aff3e0f9fb209836382ec112778667027f4fd6ae38bdb2334bc9e173598b092a", size = 644790, upload-time = "2025-04-14T16:45:23.041Z" }, + { url = "https://files.pythonhosted.org/packages/ee/35/412a0e9c3f0d37c94ed764b8ac7adae2d834dbd20e69f6aca582118e0f55/openai-1.97.1-py3-none-any.whl", hash = "sha256:4e96bbdf672ec3d44968c9ea39d2c375891db1acc1794668d8149d5fa6000606", size = 764380, upload-time = "2025-07-22T13:10:10.689Z" }, ] [[package]] @@ -3906,7 +3911,7 @@ wheels = [ [[package]] name = "openpipe" -version = "4.50.0" +version = "5.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anthropic" }, @@ -3915,9 +3920,9 @@ dependencies = [ { name = "openai" }, { name = "python-dateutil" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/0b/5ac4afd2253e058463fe46b44ebdf9cf153af343b457f13e9e592943c16d/openpipe-4.50.0.tar.gz", hash = "sha256:a2b1bf7a30a8d4c2cf45b85c749839ea9811e36f9d03916df8ffa343d9193a0e", size = 98954, upload-time = "2025-04-15T18:13:36.935Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/34/b487bc0ff60d3ed634e6f7bc34b5138f04e6ae319cc6578001822df93901/openpipe-5.0.0.tar.gz", hash = "sha256:040acc526fece42ba505fcedd8cd584f42482c9bd01f16b2538c9ea9c82882f4", size = 98910, upload-time = "2025-07-31T01:36:29.482Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/39/04870a3157d4ad6e8b1671f584da3e064750ccd64aa08339c6fc6dbd3a1c/openpipe-4.50.0-py3-none-any.whl", hash = "sha256:2071c3edbba3e08ceb977ad8c12d407f4da86c0c3815447fa33674d918276e5e", size = 440892, upload-time = "2025-04-15T18:13:35.258Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/516010c25a32884a87e1f8303a292f3981fa382cc7570a9ed88fb28681d5/openpipe-5.0.0-py3-none-any.whl", hash = "sha256:c04af7afb4d9bcd52e1250757dd93d0e0ed19c9ff4b524f131dd94aadf4c1a9b", size = 439951, upload-time = "2025-07-31T01:36:28.003Z" }, ] [[package]] @@ -3933,6 +3938,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/48/28ed9e55dcf2f453128df738210a980e09f4e468a456fa3c763dbc8be70a/opentelemetry_api-1.37.0-py3-none-any.whl", hash = "sha256:accf2024d3e89faec14302213bc39550ec0f4095d1cf5ca688e1bfb1c8612f47", size = 65732, upload-time = "2025-09-11T10:28:41.826Z" }, ] +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/df/47fde1de15a3d5ad410e98710fac60cd3d509df5dc7ec1359b71d6bf7e70/opentelemetry_exporter_otlp-1.37.0.tar.gz", hash = "sha256:f85b1929dd0d750751cc9159376fb05aa88bb7a08b6cdbf84edb0054d93e9f26", size = 6145, upload-time = "2025-09-11T10:29:03.075Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/23/7e35e41111e3834d918e414eca41555d585e8860c9149507298bb3b9b061/opentelemetry_exporter_otlp-1.37.0-py3-none-any.whl", hash = "sha256:bd44592c6bc7fc3e5c0a9b60f2ee813c84c2800c449e59504ab93f356cc450fc", size = 7019, upload-time = "2025-09-11T10:28:44.094Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/6c/10018cbcc1e6fff23aac67d7fd977c3d692dbe5f9ef9bb4db5c1268726cc/opentelemetry_exporter_otlp_proto_common-1.37.0.tar.gz", hash = "sha256:c87a1bdd9f41fdc408d9cc9367bb53f8d2602829659f2b90be9f9d79d0bfe62c", size = 20430, upload-time = "2025-09-11T10:29:03.605Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/13/b4ef09837409a777f3c0af2a5b4ba9b7af34872bc43609dda0c209e4060d/opentelemetry_exporter_otlp_proto_common-1.37.0-py3-none-any.whl", hash = "sha256:53038428449c559b0c564b8d718df3314da387109c4d36bd1b94c9a641b0292e", size = 18359, upload-time = "2025-09-11T10:28:44.939Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/11/4ad0979d0bb13ae5a845214e97c8d42da43980034c30d6f72d8e0ebe580e/opentelemetry_exporter_otlp_proto_grpc-1.37.0.tar.gz", hash = "sha256:f55bcb9fc848ce05ad3dd954058bc7b126624d22c4d9e958da24d8537763bec5", size = 24465, upload-time = "2025-09-11T10:29:04.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/17/46630b74751031a658706bef23ac99cdc2953cd3b2d28ec90590a0766b3e/opentelemetry_exporter_otlp_proto_grpc-1.37.0-py3-none-any.whl", hash = "sha256:aee5104835bf7993b7ddaaf380b6467472abaedb1f1dbfcc54a52a7d781a3890", size = 19305, upload-time = "2025-09-11T10:28:45.776Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/e3/6e320aeb24f951449e73867e53c55542bebbaf24faeee7623ef677d66736/opentelemetry_exporter_otlp_proto_http-1.37.0.tar.gz", hash = "sha256:e52e8600f1720d6de298419a802108a8f5afa63c96809ff83becb03f874e44ac", size = 17281, upload-time = "2025-09-11T10:29:04.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/e9/70d74a664d83976556cec395d6bfedd9b85ec1498b778367d5f93e373397/opentelemetry_exporter_otlp_proto_http-1.37.0-py3-none-any.whl", hash = "sha256:54c42b39945a6cc9d9a2a33decb876eabb9547e0dcb49df090122773447f1aef", size = 19576, upload-time = "2025-09-11T10:28:46.726Z" }, +] + [[package]] name = "opentelemetry-instrumentation" version = "0.58b0" @@ -3962,6 +4028,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/54/add1076cb37980e617723a96e29c84006983e8ad6fc589dde7f69ddc57d4/opentelemetry_instrumentation_threading-0.58b0-py3-none-any.whl", hash = "sha256:eacc072881006aceb5b9b6831bcdce718c67ef6f31ac0b32bd6a23a94d979b4a", size = 9312, upload-time = "2025-09-11T11:41:58.603Z" }, ] +[[package]] +name = "opentelemetry-proto" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/ea/a75f36b463a36f3c5a10c0b5292c58b31dbdde74f6f905d3d0ab2313987b/opentelemetry_proto-1.37.0.tar.gz", hash = "sha256:30f5c494faf66f77faeaefa35ed4443c5edb3b0aa46dad073ed7210e1a789538", size = 46151, upload-time = "2025-09-11T10:29:11.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/25/f89ea66c59bd7687e218361826c969443c4fa15dfe89733f3bf1e2a9e971/opentelemetry_proto-1.37.0-py3-none-any.whl", hash = "sha256:8ed8c066ae8828bbf0c39229979bdf583a126981142378a9cbe9d6fd5701c6e2", size = 72534, upload-time = "2025-09-11T10:28:56.831Z" }, +] + [[package]] name = "opentelemetry-sdk" version = "1.37.0" @@ -3989,6 +4067,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/90/68152b7465f50285d3ce2481b3aec2f82822e3f52e5152eeeaf516bab841/opentelemetry_semantic_conventions-0.58b0-py3-none-any.whl", hash = "sha256:5564905ab1458b96684db1340232729fce3b5375a06e140e8904c78e4f815b28", size = 207954, upload-time = "2025-09-11T10:28:59.218Z" }, ] +[[package]] +name = "opentelemetry-semantic-conventions-ai" +version = "0.4.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/e6/40b59eda51ac47009fb47afcdf37c6938594a0bd7f3b9fadcbc6058248e3/opentelemetry_semantic_conventions_ai-0.4.13.tar.gz", hash = "sha256:94efa9fb4ffac18c45f54a3a338ffeb7eedb7e1bb4d147786e77202e159f0036", size = 5368, upload-time = "2025-08-22T10:14:17.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/b5/cf25da2218910f0d6cdf7f876a06bed118c4969eacaf60a887cbaef44f44/opentelemetry_semantic_conventions_ai-0.4.13-py3-none-any.whl", hash = "sha256:883a30a6bb5deaec0d646912b5f9f6dcbb9f6f72557b73d0f2560bf25d13e2d5", size = 6080, upload-time = "2025-08-22T10:14:16.477Z" }, +] + [[package]] name = "orjson" version = "3.11.3" @@ -4546,11 +4633,11 @@ requires-dist = [ { name = "aiortc", marker = "extra == 'webrtc'", specifier = ">=1.13.0,<2" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = "~=0.49.0" }, { name = "audioop-lts", marker = "python_full_version >= '3.13'", specifier = "~=0.2.1" }, - { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.0.2" }, + { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.1.1" }, { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.42.0" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, - { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.19.9" }, + { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.21.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = "~=4.7.0" }, { name = "docstring-parser", specifier = "~=0.16" }, { name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" }, @@ -4558,9 +4645,9 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'runner'", specifier = ">=0.115.6,<0.117.0" }, { name = "fastapi", marker = "extra == 'websocket'", specifier = ">=0.115.6,<0.117.0" }, { name = "faster-whisper", marker = "extra == 'whisper'", specifier = "~=1.1.1" }, - { name = "google-cloud-speech", marker = "extra == 'google'", specifier = "~=2.32.0" }, - { name = "google-cloud-texttospeech", marker = "extra == 'google'", specifier = "~=2.26.0" }, - { name = "google-genai", marker = "extra == 'google'", specifier = "~=1.24.0" }, + { name = "google-cloud-speech", marker = "extra == 'google'", specifier = ">=2.33.0,<3" }, + { name = "google-cloud-texttospeech", marker = "extra == 'google'", specifier = ">=2.31.0,<3" }, + { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.41.0,<2" }, { name = "groq", marker = "extra == 'groq'", specifier = "~=0.23.0" }, { name = "hume", marker = "extra == 'hume'", specifier = ">=0.11.2" }, { name = "langchain", marker = "extra == 'langchain'", specifier = "~=0.3.20" }, @@ -4581,9 +4668,9 @@ requires-dist = [ { name = "nvidia-riva-client", marker = "extra == 'riva'", specifier = "~=2.21.1" }, { name = "onnxruntime", marker = "extra == 'local-smart-turn-v3'", specifier = ">=1.20.1,<2" }, { name = "onnxruntime", marker = "extra == 'silero'", specifier = ">=1.20.1,<2" }, - { name = "openai", specifier = ">=1.74.0,<=1.99.1" }, + { name = "openai", specifier = ">=1.74.0,<3" }, { name = "opencv-python", marker = "extra == 'webrtc'", specifier = ">=4.11.0.86,<5" }, - { name = "openpipe", marker = "extra == 'openpipe'", specifier = "~=4.50.0" }, + { name = "openpipe", marker = "extra == 'openpipe'", specifier = ">=4.50.0,<6" }, { name = "opentelemetry-api", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "opentelemetry-instrumentation", marker = "extra == 'tracing'", specifier = ">=0.54b0" }, { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, @@ -4621,7 +4708,7 @@ requires-dist = [ { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=0.1.10" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.0" }, { name = "soxr", specifier = "~=0.5.0" }, - { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.4.0" }, + { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.5.0" }, { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, @@ -4963,172 +5050,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/8b/7f9a061c1cc2b230f9ac02a6003fcd14c85ce1828013aecbaf45aa988d20/PyAudio-0.2.14-cp313-cp313-win_amd64.whl", hash = "sha256:692d8c1446f52ed2662120bcd9ddcb5aa2b71f38bda31e58b19fb4672fffba69", size = 173655, upload-time = "2024-11-20T19:12:13.616Z" }, ] -[[package]] -name = "pybase64" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/14/43297a7b7f0c1bf0c00b596f754ee3ac946128c64d21047ccf9c9bbc5165/pybase64-1.4.2.tar.gz", hash = "sha256:46cdefd283ed9643315d952fe44de80dc9b9a811ce6e3ec97fd1827af97692d0", size = 137246, upload-time = "2025-07-27T13:08:57.808Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/6d/0a7159c24ed35c8b9190b148376ad9b96598354f94ede29df74861da9ec6/pybase64-1.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82b4593b480773b17698fef33c68bae0e1c474ba07663fad74249370c46b46c9", size = 38240, upload-time = "2025-07-27T13:02:17.876Z" }, - { url = "https://files.pythonhosted.org/packages/86/2e/dad4cd832a90a49d98867e824180585e7c928504987d37304bccae11a314/pybase64-1.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a126f29d29cb4a498db179135dbf955442a0de5b00f374523f5dcceb9074ff58", size = 31658, upload-time = "2025-07-27T13:02:20.823Z" }, - { url = "https://files.pythonhosted.org/packages/1d/d8/30ea35dc2c8c568be93e1379efcaa35092e37efa2ce7f1985ccc63babee7/pybase64-1.4.2-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:1eef93c29cc5567480d168f9cc1ebd3fc3107c65787aed2019a8ea68575a33e0", size = 65963, upload-time = "2025-07-27T13:02:22.376Z" }, - { url = "https://files.pythonhosted.org/packages/f6/da/1c22f2a21d6bb9ec2a214d15ae02d5b20a95335de218a0ecbf769c535a5c/pybase64-1.4.2-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:17b871a34aaeb0644145cb6bf28feb163f593abea11aec3dbcc34a006edfc828", size = 68887, upload-time = "2025-07-27T13:02:23.606Z" }, - { url = "https://files.pythonhosted.org/packages/ac/8d/e04d489ba99b444ce94b4d5b232365d00b0f0e8564275d7ba7434dcabe72/pybase64-1.4.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1f734e16293637a35d282ce594eb05a7a90ea3ae2bc84a3496a5df9e6b890725", size = 57503, upload-time = "2025-07-27T13:02:24.83Z" }, - { url = "https://files.pythonhosted.org/packages/7e/b8/5ec9c334f30cf898709a084d596bf4b47aec2e07870f07bac5cf39754eca/pybase64-1.4.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:22bd38db2d990d5545dde83511edeec366630d00679dbd945472315c09041dc6", size = 54517, upload-time = "2025-07-27T13:02:26.006Z" }, - { url = "https://files.pythonhosted.org/packages/b9/5a/6e4424ecca041e53aa7c14525f99edd43d0117c23c5d9cb14e931458a536/pybase64-1.4.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:dc65cee686dda72007b7541b2014f33ee282459c781b9b61305bd8b9cfadc8e1", size = 57167, upload-time = "2025-07-27T13:02:27.47Z" }, - { url = "https://files.pythonhosted.org/packages/5f/d0/13f1a9467cf565eecc21dce89fb0723458d8c563d2ccfb99b96e8318dfd5/pybase64-1.4.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1e79641c420a22e49c67c046895efad05bf5f8b1dbe0dd78b4af3ab3f2923fe2", size = 57718, upload-time = "2025-07-27T13:02:28.631Z" }, - { url = "https://files.pythonhosted.org/packages/3e/34/d80335c36ad9400b18b4f92e9f680cf7646102fe4919f7bce5786a2ccb7b/pybase64-1.4.2-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:12f5e7db522ef780a8b333dab5f7d750d270b23a1684bc2235ba50756c7ba428", size = 53021, upload-time = "2025-07-27T13:02:29.823Z" }, - { url = "https://files.pythonhosted.org/packages/68/57/504ff75f7c78df28be126fe6634083d28d7f84c17e04a74a7dcb50ab2377/pybase64-1.4.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a618b1e1a63e75dd40c2a397d875935ed0835464dc55cb1b91e8f880113d0444", size = 56306, upload-time = "2025-07-27T13:02:31.314Z" }, - { url = "https://files.pythonhosted.org/packages/bf/bc/2d21cda8b73c8c9f5cd3d7e6e26dd6dfc96491052112f282332a3d5bf1d9/pybase64-1.4.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:89b0a51702c7746fa914e75e680ad697b979cdead6b418603f56a6fc9de2f50f", size = 50101, upload-time = "2025-07-27T13:02:32.662Z" }, - { url = "https://files.pythonhosted.org/packages/88/6d/51942e7737bb0711ca3e55db53924fd7f07166d79da5508ab8f5fd5972a8/pybase64-1.4.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c5161b8b82f8ba5dbbc3f76e0270622a2c2fdb9ffaf092d8f774ad7ec468c027", size = 66555, upload-time = "2025-07-27T13:02:34.122Z" }, - { url = "https://files.pythonhosted.org/packages/b6/c8/c46024d196402e7be4d3fad85336863a34816c3436c51fcf9c7c0781bf11/pybase64-1.4.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:2168de920c9b1e57850e9ff681852923a953601f73cc96a0742a42236695c316", size = 55684, upload-time = "2025-07-27T13:02:35.427Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c5/953782c9d599ff5217ee87f19e317c494cd4840afcab4c48f99cb78ca201/pybase64-1.4.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7a1e3dc977562abe40ab43483223013be71b215a5d5f3c78a666e70a5076eeec", size = 52475, upload-time = "2025-07-27T13:02:36.634Z" }, - { url = "https://files.pythonhosted.org/packages/05/fb/57d36173631aab67ca4558cdbde1047fc67a09b77f9c53addd57c7e9fdd4/pybase64-1.4.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:4cf1e8a57449e48137ef4de00a005e24c3f1cffc0aafc488e36ceb5bb2cbb1da", size = 53943, upload-time = "2025-07-27T13:02:37.777Z" }, - { url = "https://files.pythonhosted.org/packages/75/73/23e5bb0bffac0cabe2d11d1c618f6ef73da9f430da03c5249931e3c49b63/pybase64-1.4.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d8e1a381ba124f26a93d5925efbf6e6c36287fc2c93d74958e8b677c30a53fc0", size = 68411, upload-time = "2025-07-27T13:02:39.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e7/0d5c99e5e61ff5e46949a0128b49fc2c47afc0d2b815333459b17aa9d467/pybase64-1.4.2-cp310-cp310-win32.whl", hash = "sha256:8fdd9c5b60ec9a1db854f5f96bba46b80a9520069282dc1d37ff433eb8248b1f", size = 33614, upload-time = "2025-07-27T13:02:40.478Z" }, - { url = "https://files.pythonhosted.org/packages/23/40/879b6de61d7c07a2cbf76b75e9739c4938c3a1f66ac03243f2ff7ec9fb6b/pybase64-1.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:37a6c73f14c6539c0ad1aebf0cce92138af25c99a6e7aee637d9f9fc634c8a40", size = 35790, upload-time = "2025-07-27T13:02:41.864Z" }, - { url = "https://files.pythonhosted.org/packages/d2/e2/75cec12880ce3f47a79a2b9a0cdc766dc0429a7ce967bb3ab3a4b55a7f6b/pybase64-1.4.2-cp310-cp310-win_arm64.whl", hash = "sha256:b3280d03b7b361622c469d005cc270d763d9e29d0a490c26addb4f82dfe71a79", size = 30900, upload-time = "2025-07-27T13:02:43.022Z" }, - { url = "https://files.pythonhosted.org/packages/da/fb/edaa56bbf04715efc3c36966cc0150e01d7a8336c3da182f850b7fd43d32/pybase64-1.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26284ef64f142067293347bcc9d501d2b5d44b92eab9d941cb10a085fb01c666", size = 38238, upload-time = "2025-07-27T13:02:44.224Z" }, - { url = "https://files.pythonhosted.org/packages/28/a4/ca1538e9adf08f5016b3543b0060c18aea9a6e805dd20712a197c509d90d/pybase64-1.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:52dd32fe5cbfd8af8f3f034a4a65ee61948c72e5c358bf69d59543fc0dbcf950", size = 31659, upload-time = "2025-07-27T13:02:45.445Z" }, - { url = "https://files.pythonhosted.org/packages/0b/8f/f9b49926a60848ba98350dd648227ec524fb78340b47a450c4dbaf24b1bb/pybase64-1.4.2-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:37f133e8c96427995480bb6d396d9d49e949a3e829591845bb6a5a7f215ca177", size = 68318, upload-time = "2025-07-27T13:02:46.644Z" }, - { url = "https://files.pythonhosted.org/packages/29/9b/6ed2dd2bc8007f33b8316d6366b0901acbdd5665b419c2893b3dd48708de/pybase64-1.4.2-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6ee3874b0abbdd4c903d3989682a3f016fd84188622879f6f95a5dc5718d7e5", size = 71357, upload-time = "2025-07-27T13:02:47.937Z" }, - { url = "https://files.pythonhosted.org/packages/fb/69/be9ac8127da8d8339db7129683bd2975cecb0bf40a82731e1a492577a177/pybase64-1.4.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c69f177b1e404b22b05802127d6979acf4cb57f953c7de9472410f9c3fdece7", size = 59817, upload-time = "2025-07-27T13:02:49.163Z" }, - { url = "https://files.pythonhosted.org/packages/f4/a2/e3e09e000b509609276ee28b71beb0b61462d4a43b3e0db0a44c8652880c/pybase64-1.4.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:80c817e88ef2ca3cc9a285fde267690a1cb821ce0da4848c921c16f0fec56fda", size = 56639, upload-time = "2025-07-27T13:02:50.384Z" }, - { url = "https://files.pythonhosted.org/packages/01/70/ad7eff88aa4f1be06db705812e1f01749606933bf8fe9df553bb04b703e6/pybase64-1.4.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7a4bb6e7e45bfdaea0f2aaf022fc9a013abe6e46ccea31914a77e10f44098688", size = 59368, upload-time = "2025-07-27T13:02:51.883Z" }, - { url = "https://files.pythonhosted.org/packages/9d/82/0cd1b4bcd2a4da7805cfa04587be783bf9583b34ac16cadc29cf119a4fa2/pybase64-1.4.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2710a80d41a2b41293cb0e5b84b5464f54aa3f28f7c43de88784d2d9702b8a1c", size = 59981, upload-time = "2025-07-27T13:02:53.16Z" }, - { url = "https://files.pythonhosted.org/packages/3c/4c/8029a03468307dfaf0f9694d31830487ee43af5f8a73407004907724e8ac/pybase64-1.4.2-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:aa6122c8a81f6597e1c1116511f03ed42cf377c2100fe7debaae7ca62521095a", size = 54908, upload-time = "2025-07-27T13:02:54.363Z" }, - { url = "https://files.pythonhosted.org/packages/a1/8b/70bd0fe659e242efd0f60895a8ce1fe88e3a4084fd1be368974c561138c9/pybase64-1.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7e22b02505d64db308e9feeb6cb52f1d554ede5983de0befa59ac2d2ffb6a5f", size = 58650, upload-time = "2025-07-27T13:02:55.905Z" }, - { url = "https://files.pythonhosted.org/packages/64/ca/9c1d23cbc4b9beac43386a32ad53903c816063cef3f14c10d7c3d6d49a23/pybase64-1.4.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:edfe4a3c8c4007f09591f49b46a89d287ef5e8cd6630339536fe98ff077263c2", size = 52323, upload-time = "2025-07-27T13:02:57.192Z" }, - { url = "https://files.pythonhosted.org/packages/aa/29/a6292e9047248c8616dc53131a49da6c97a61616f80e1e36c73d7ef895fe/pybase64-1.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b79b4a53dd117ffbd03e96953f2e6bd2827bfe11afeb717ea16d9b0893603077", size = 68979, upload-time = "2025-07-27T13:02:58.594Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e0/cfec7b948e170395d8e88066e01f50e71195db9837151db10c14965d6222/pybase64-1.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fd9afa7a61d89d170607faf22287290045757e782089f0357b8f801d228d52c3", size = 58037, upload-time = "2025-07-27T13:02:59.753Z" }, - { url = "https://files.pythonhosted.org/packages/74/7e/0ac1850198c9c35ef631174009cee576f4d8afff3bf493ce310582976ab4/pybase64-1.4.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5c17b092e4da677a595178d2db17a5d2fafe5c8e418d46c0c4e4cde5adb8cff3", size = 54416, upload-time = "2025-07-27T13:03:00.978Z" }, - { url = "https://files.pythonhosted.org/packages/1b/45/b0b037f27e86c50e62d927f0bc1bde8b798dd55ab39197b116702e508d05/pybase64-1.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:120799274cf55f3f5bb8489eaa85142f26170564baafa7cf3e85541c46b6ab13", size = 56257, upload-time = "2025-07-27T13:03:02.201Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0d/5034598aac56336d88fd5aaf6f34630330643b51d399336b8c788d798fc5/pybase64-1.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:522e4e712686acec2d25de9759dda0b0618cb9f6588523528bc74715c0245c7b", size = 70889, upload-time = "2025-07-27T13:03:03.437Z" }, - { url = "https://files.pythonhosted.org/packages/8a/3b/0645f21bb08ecf45635b624958b5f9e569069d31ecbf125dc7e0e5b83f60/pybase64-1.4.2-cp311-cp311-win32.whl", hash = "sha256:bfd828792982db8d787515535948c1e340f1819407c8832f94384c0ebeaf9d74", size = 33631, upload-time = "2025-07-27T13:03:05.194Z" }, - { url = "https://files.pythonhosted.org/packages/8f/08/24f8103c1f19e78761026cdd9f3b3be73239bc19cf5ab6fef0e8042d0bc6/pybase64-1.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:7a9e89d40dbf833af481d1d5f1a44d173c9c4b56a7c8dba98e39a78ee87cfc52", size = 35781, upload-time = "2025-07-27T13:03:06.779Z" }, - { url = "https://files.pythonhosted.org/packages/66/cd/832fb035a0ea7eb53d776a5cfa961849e22828f6dfdfcdb9eb43ba3c0166/pybase64-1.4.2-cp311-cp311-win_arm64.whl", hash = "sha256:ce5809fa90619b03eab1cd63fec142e6cf1d361731a9b9feacf27df76c833343", size = 30903, upload-time = "2025-07-27T13:03:07.903Z" }, - { url = "https://files.pythonhosted.org/packages/28/6d/11ede991e800797b9f5ebd528013b34eee5652df93de61ffb24503393fa5/pybase64-1.4.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2c75d1388855b5a1015b65096d7dbcc708e7de3245dcbedeb872ec05a09326", size = 38326, upload-time = "2025-07-27T13:03:09.065Z" }, - { url = "https://files.pythonhosted.org/packages/fe/84/87f1f565f42e2397e2aaa2477c86419f5173c3699881c42325c090982f0a/pybase64-1.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b621a972a01841368fdb9dedc55fd3c6e0c7217d0505ba3b1ebe95e7ef1b493", size = 31661, upload-time = "2025-07-27T13:03:10.295Z" }, - { url = "https://files.pythonhosted.org/packages/cb/2a/a24c810e7a61d2cc6f73fe9ee4872a03030887fa8654150901b15f376f65/pybase64-1.4.2-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f48c32ac6a16cbf57a5a96a073fef6ff7e3526f623cd49faa112b7f9980bafba", size = 68192, upload-time = "2025-07-27T13:03:11.467Z" }, - { url = "https://files.pythonhosted.org/packages/ee/87/d9baf98cbfc37b8657290ad4421f3a3c36aa0eafe4872c5859cfb52f3448/pybase64-1.4.2-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ace8b23093a6bb862477080d9059b784096ab2f97541e8bfc40d42f062875149", size = 71587, upload-time = "2025-07-27T13:03:12.719Z" }, - { url = "https://files.pythonhosted.org/packages/0b/89/3df043cc56ef3b91b7aa0c26ae822a2d7ec8da0b0fd7c309c879b0eb5988/pybase64-1.4.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1772c7532a7fb6301baea3dd3e010148dbf70cd1136a83c2f5f91bdc94822145", size = 59910, upload-time = "2025-07-27T13:03:14.266Z" }, - { url = "https://files.pythonhosted.org/packages/75/4f/6641e9edf37aeb4d4524dc7ba2168eff8d96c90e77f6283c2be3400ab380/pybase64-1.4.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:f86f7faddcba5cbfea475f8ab96567834c28bf09ca6c7c3d66ee445adac80d8f", size = 56701, upload-time = "2025-07-27T13:03:15.6Z" }, - { url = "https://files.pythonhosted.org/packages/2d/7f/20d8ac1046f12420a0954a45a13033e75f98aade36eecd00c64e3549b071/pybase64-1.4.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0b8c8e275b5294089f314814b4a50174ab90af79d6a4850f6ae11261ff6a7372", size = 59288, upload-time = "2025-07-27T13:03:16.823Z" }, - { url = "https://files.pythonhosted.org/packages/17/ea/9c0ca570e3e50b3c6c3442e280c83b321a0464c86a9db1f982a4ff531550/pybase64-1.4.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:864d85a0470c615807ae8b97d724d068b940a2d10ac13a5f1b9e75a3ce441758", size = 60267, upload-time = "2025-07-27T13:03:18.132Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ac/46894929d71ccedebbfb0284173b0fea96bc029cd262654ba8451a7035d6/pybase64-1.4.2-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:47254d97ed2d8351e30ecfdb9e2414547f66ba73f8a09f932c9378ff75cd10c5", size = 54801, upload-time = "2025-07-27T13:03:19.669Z" }, - { url = "https://files.pythonhosted.org/packages/6a/1e/02c95218ea964f0b2469717c2c69b48e63f4ca9f18af01a5b2a29e4c1216/pybase64-1.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:264b65ecc4f0ee73f3298ab83bbd8008f7f9578361b8df5b448f985d8c63e02a", size = 58599, upload-time = "2025-07-27T13:03:20.951Z" }, - { url = "https://files.pythonhosted.org/packages/15/45/ccc21004930789b8fb439d43e3212a6c260ccddb2bf450c39a20db093f33/pybase64-1.4.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fbcc2b30cd740c16c9699f596f22c7a9e643591311ae72b1e776f2d539e9dd9d", size = 52388, upload-time = "2025-07-27T13:03:23.064Z" }, - { url = "https://files.pythonhosted.org/packages/c4/45/22e46e549710c4c237d77785b6fb1bc4c44c288a5c44237ba9daf5c34b82/pybase64-1.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cda9f79c22d51ee4508f5a43b673565f1d26af4330c99f114e37e3186fdd3607", size = 68802, upload-time = "2025-07-27T13:03:24.673Z" }, - { url = "https://files.pythonhosted.org/packages/55/0c/232c6261b81296e5593549b36e6e7884a5da008776d12665923446322c36/pybase64-1.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0c91c6d2a7232e2a1cd10b3b75a8bb657defacd4295a1e5e80455df2dfc84d4f", size = 57841, upload-time = "2025-07-27T13:03:25.948Z" }, - { url = "https://files.pythonhosted.org/packages/20/8a/b35a615ae6f04550d696bb179c414538b3b477999435fdd4ad75b76139e4/pybase64-1.4.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a370dea7b1cee2a36a4d5445d4e09cc243816c5bc8def61f602db5a6f5438e52", size = 54320, upload-time = "2025-07-27T13:03:27.495Z" }, - { url = "https://files.pythonhosted.org/packages/d3/a9/8bd4f9bcc53689f1b457ecefed1eaa080e4949d65a62c31a38b7253d5226/pybase64-1.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9aa4de83f02e462a6f4e066811c71d6af31b52d7484de635582d0e3ec3d6cc3e", size = 56482, upload-time = "2025-07-27T13:03:28.942Z" }, - { url = "https://files.pythonhosted.org/packages/75/e5/4a7735b54a1191f61c3f5c2952212c85c2d6b06eb5fb3671c7603395f70c/pybase64-1.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83a1c2f9ed00fee8f064d548c8654a480741131f280e5750bb32475b7ec8ee38", size = 70959, upload-time = "2025-07-27T13:03:30.171Z" }, - { url = "https://files.pythonhosted.org/packages/d3/67/e2b6cb32c782e12304d467418e70da0212567f42bd4d3b5eb1fdf64920ad/pybase64-1.4.2-cp312-cp312-win32.whl", hash = "sha256:a6e5688b18d558e8c6b8701cc8560836c4bbeba61d33c836b4dba56b19423716", size = 33683, upload-time = "2025-07-27T13:03:31.775Z" }, - { url = "https://files.pythonhosted.org/packages/4f/bc/d5c277496063a09707486180f17abbdbdebbf2f5c4441b20b11d3cb7dc7c/pybase64-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:c995d21b8bd08aa179cd7dd4db0695c185486ecc72da1e8f6c37ec86cadb8182", size = 35817, upload-time = "2025-07-27T13:03:32.99Z" }, - { url = "https://files.pythonhosted.org/packages/e6/69/e4be18ae685acff0ae77f75d4586590f29d2cd187bf603290cf1d635cad4/pybase64-1.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:e254b9258c40509c2ea063a7784f6994988f3f26099d6e08704e3c15dfed9a55", size = 30900, upload-time = "2025-07-27T13:03:34.499Z" }, - { url = "https://files.pythonhosted.org/packages/f4/56/5337f27a8b8d2d6693f46f7b36bae47895e5820bfa259b0072574a4e1057/pybase64-1.4.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:0f331aa59549de21f690b6ccc79360ffed1155c3cfbc852eb5c097c0b8565a2b", size = 33888, upload-time = "2025-07-27T13:03:35.698Z" }, - { url = "https://files.pythonhosted.org/packages/4c/09/f3f4b11fc9beda7e8625e29fb0f549958fcbb34fea3914e1c1d95116e344/pybase64-1.4.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:9dad20bf1f3ed9e6fe566c4c9d07d9a6c04f5a280daebd2082ffb8620b0a880d", size = 40796, upload-time = "2025-07-27T13:03:36.927Z" }, - { url = "https://files.pythonhosted.org/packages/e3/ff/470768f0fe6de0aa302a8cb1bdf2f9f5cffc3f69e60466153be68bc953aa/pybase64-1.4.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:69d3f0445b0faeef7bb7f93bf8c18d850785e2a77f12835f49e524cc54af04e7", size = 30914, upload-time = "2025-07-27T13:03:38.475Z" }, - { url = "https://files.pythonhosted.org/packages/75/6b/d328736662665e0892409dc410353ebef175b1be5eb6bab1dad579efa6df/pybase64-1.4.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2372b257b1f4dd512f317fb27e77d313afd137334de64c87de8374027aacd88a", size = 31380, upload-time = "2025-07-27T13:03:39.7Z" }, - { url = "https://files.pythonhosted.org/packages/ca/96/7ff718f87c67f4147c181b73d0928897cefa17dc75d7abc6e37730d5908f/pybase64-1.4.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:fb794502b4b1ec91c4ca5d283ae71aef65e3de7721057bd9e2b3ec79f7a62d7d", size = 38230, upload-time = "2025-07-27T13:03:41.637Z" }, - { url = "https://files.pythonhosted.org/packages/4d/58/a3307b048d799ff596a3c7c574fcba66f9b6b8c899a3c00a698124ca7ad5/pybase64-1.4.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d5c532b03fd14a5040d6cf6571299a05616f925369c72ddf6fe2fb643eb36fed", size = 38319, upload-time = "2025-07-27T13:03:42.847Z" }, - { url = "https://files.pythonhosted.org/packages/08/a7/0bda06341b0a2c830d348c6e1c4d348caaae86c53dc9a046e943467a05e9/pybase64-1.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f699514dc1d5689ca9cf378139e0214051922732f9adec9404bc680a8bef7c0", size = 31655, upload-time = "2025-07-27T13:03:44.426Z" }, - { url = "https://files.pythonhosted.org/packages/87/df/e1d6e8479e0c5113c2c63c7b44886935ce839c2d99884c7304ca9e86547c/pybase64-1.4.2-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:cd3e8713cbd32c8c6aa935feaf15c7670e2b7e8bfe51c24dc556811ebd293a29", size = 68232, upload-time = "2025-07-27T13:03:45.729Z" }, - { url = "https://files.pythonhosted.org/packages/71/ab/db4dbdfccb9ca874d6ce34a0784761471885d96730de85cee3d300381529/pybase64-1.4.2-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d377d48acf53abf4b926c2a7a24a19deb092f366a04ffd856bf4b3aa330b025d", size = 71608, upload-time = "2025-07-27T13:03:47.01Z" }, - { url = "https://files.pythonhosted.org/packages/11/e9/508df958563951045d728bbfbd3be77465f9231cf805cb7ccaf6951fc9f1/pybase64-1.4.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d83c076e78d619b9e1dd674e2bf5fb9001aeb3e0b494b80a6c8f6d4120e38cd9", size = 59912, upload-time = "2025-07-27T13:03:48.277Z" }, - { url = "https://files.pythonhosted.org/packages/f2/58/7f2cef1ceccc682088958448d56727369de83fa6b29148478f4d2acd107a/pybase64-1.4.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:ab9cdb6a8176a5cb967f53e6ad60e40c83caaa1ae31c5e1b29e5c8f507f17538", size = 56413, upload-time = "2025-07-27T13:03:49.908Z" }, - { url = "https://files.pythonhosted.org/packages/08/7c/7e0af5c5728fa7e2eb082d88eca7c6bd17429be819d58518e74919d42e66/pybase64-1.4.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:adf0c103ad559dbfb9fe69edfd26a15c65d9c991a5ab0a25b04770f9eb0b9484", size = 59311, upload-time = "2025-07-27T13:03:51.238Z" }, - { url = "https://files.pythonhosted.org/packages/03/8b/09825d0f37e45b9a3f546e5f990b6cf2dd838e54ea74122c2464646e0c77/pybase64-1.4.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:0d03ef2f253d97ce0685d3624bf5e552d716b86cacb8a6c971333ba4b827e1fc", size = 60282, upload-time = "2025-07-27T13:03:52.56Z" }, - { url = "https://files.pythonhosted.org/packages/9c/3f/3711d2413f969bfd5b9cc19bc6b24abae361b7673ff37bcb90c43e199316/pybase64-1.4.2-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:e565abf906efee76ae4be1aef5df4aed0fda1639bc0d7732a3dafef76cb6fc35", size = 54845, upload-time = "2025-07-27T13:03:54.167Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3c/4c7ce1ae4d828c2bb56d144322f81bffbaaac8597d35407c3d7cbb0ff98f/pybase64-1.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3c6a5f15fd03f232fc6f295cce3684f7bb08da6c6d5b12cc771f81c9f125cc6", size = 58615, upload-time = "2025-07-27T13:03:55.494Z" }, - { url = "https://files.pythonhosted.org/packages/f5/8f/c2fc03bf4ed038358620065c75968a30184d5d3512d09d3ef9cc3bd48592/pybase64-1.4.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:bad9e3db16f448728138737bbd1af9dc2398efd593a8bdd73748cc02cd33f9c6", size = 52434, upload-time = "2025-07-27T13:03:56.808Z" }, - { url = "https://files.pythonhosted.org/packages/e2/0a/757d6df0a60327c893cfae903e15419914dd792092dc8cc5c9523d40bc9b/pybase64-1.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2683ef271328365c31afee0ed8fa29356fb8fb7c10606794656aa9ffb95e92be", size = 68824, upload-time = "2025-07-27T13:03:58.735Z" }, - { url = "https://files.pythonhosted.org/packages/a0/14/84abe2ed8c29014239be1cfab45dfebe5a5ca779b177b8b6f779bd8b69da/pybase64-1.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:265b20089cd470079114c09bb74b101b3bfc3c94ad6b4231706cf9eff877d570", size = 57898, upload-time = "2025-07-27T13:04:00.379Z" }, - { url = "https://files.pythonhosted.org/packages/7e/c6/d193031f90c864f7b59fa6d1d1b5af41f0f5db35439988a8b9f2d1b32a13/pybase64-1.4.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e53173badead10ef8b839aa5506eecf0067c7b75ad16d9bf39bc7144631f8e67", size = 54319, upload-time = "2025-07-27T13:04:01.742Z" }, - { url = "https://files.pythonhosted.org/packages/cb/37/ec0c7a610ff8f994ee6e0c5d5d66b6b6310388b96ebb347b03ae39870fdf/pybase64-1.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5823b8dcf74da7da0f761ed60c961e8928a6524e520411ad05fe7f9f47d55b40", size = 56472, upload-time = "2025-07-27T13:04:03.089Z" }, - { url = "https://files.pythonhosted.org/packages/c4/5a/e585b74f85cedd261d271e4c2ef333c5cfce7e80750771808f56fee66b98/pybase64-1.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1237f66c54357d325390da60aa5e21c6918fbcd1bf527acb9c1f4188c62cb7d5", size = 70966, upload-time = "2025-07-27T13:04:04.361Z" }, - { url = "https://files.pythonhosted.org/packages/ad/20/1b2fdd98b4ba36008419668c813025758214c543e362c66c49214ecd1127/pybase64-1.4.2-cp313-cp313-win32.whl", hash = "sha256:b0b851eb4f801d16040047f6889cca5e9dfa102b3e33f68934d12511245cef86", size = 33681, upload-time = "2025-07-27T13:04:06.126Z" }, - { url = "https://files.pythonhosted.org/packages/ff/64/3df4067d169c047054889f34b5a946cbe3785bca43404b93c962a5461a41/pybase64-1.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:19541c6e26d17d9522c02680fe242206ae05df659c82a657aabadf209cd4c6c7", size = 35822, upload-time = "2025-07-27T13:04:07.752Z" }, - { url = "https://files.pythonhosted.org/packages/d1/fd/db505188adf812e60ee923f196f9deddd8a1895b2b29b37f5db94afc3b1c/pybase64-1.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:77a191863d576c0a5dd81f8a568a5ca15597cc980ae809dce62c717c8d42d8aa", size = 30899, upload-time = "2025-07-27T13:04:09.062Z" }, - { url = "https://files.pythonhosted.org/packages/d9/27/5f5fecd206ec1e06e1608a380af18dcb76a6ab08ade6597a3251502dcdb2/pybase64-1.4.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2e194bbabe3fdf9e47ba9f3e157394efe0849eb226df76432126239b3f44992c", size = 38677, upload-time = "2025-07-27T13:04:10.334Z" }, - { url = "https://files.pythonhosted.org/packages/bf/0f/abe4b5a28529ef5f74e8348fa6a9ef27d7d75fbd98103d7664cf485b7d8f/pybase64-1.4.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:39aef1dadf4a004f11dd09e703abaf6528a87c8dbd39c448bb8aebdc0a08c1be", size = 32066, upload-time = "2025-07-27T13:04:11.641Z" }, - { url = "https://files.pythonhosted.org/packages/ac/7e/ea0ce6a7155cada5526017ec588b6d6185adea4bf9331565272f4ef583c2/pybase64-1.4.2-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:91cb920c7143e36ec8217031282c8651da3b2206d70343f068fac0e7f073b7f9", size = 72300, upload-time = "2025-07-27T13:04:12.969Z" }, - { url = "https://files.pythonhosted.org/packages/45/2d/e64c7a056c9ec48dfe130d1295e47a8c2b19c3984488fc08e5eaa1e86c88/pybase64-1.4.2-cp313-cp313t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6958631143fb9e71f9842000da042ec2f6686506b6706e2dfda29e97925f6aa0", size = 75520, upload-time = "2025-07-27T13:04:14.374Z" }, - { url = "https://files.pythonhosted.org/packages/43/e0/e5f93b2e1cb0751a22713c4baa6c6eaf5f307385e369180486c8316ed21e/pybase64-1.4.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dc35f14141ef3f1ac70d963950a278a2593af66fe5a1c7a208e185ca6278fa25", size = 65384, upload-time = "2025-07-27T13:04:16.204Z" }, - { url = "https://files.pythonhosted.org/packages/ff/23/8c645a1113ad88a1c6a3d0e825e93ef8b74ad3175148767853a0a4d7626e/pybase64-1.4.2-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5d949d2d677859c3a8507e1b21432a039d2b995e0bd3fe307052b6ded80f207a", size = 60471, upload-time = "2025-07-27T13:04:17.947Z" }, - { url = "https://files.pythonhosted.org/packages/8b/81/edd0f7d8b0526b91730a0dd4ce6b4c8be2136cd69d424afe36235d2d2a06/pybase64-1.4.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:09caacdd3e15fe7253a67781edd10a6a918befab0052a2a3c215fe5d1f150269", size = 63945, upload-time = "2025-07-27T13:04:19.383Z" }, - { url = "https://files.pythonhosted.org/packages/a5/a5/edc224cd821fd65100b7af7c7e16b8f699916f8c0226c9c97bbae5a75e71/pybase64-1.4.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:e44b0e793b23f28ea0f15a9754bd0c960102a2ac4bccb8fafdedbd4cc4d235c0", size = 64858, upload-time = "2025-07-27T13:04:20.807Z" }, - { url = "https://files.pythonhosted.org/packages/11/3b/92853f968f1af7e42b7e54d21bdd319097b367e7dffa2ca20787361df74c/pybase64-1.4.2-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:849f274d0bcb90fc6f642c39274082724d108e41b15f3a17864282bd41fc71d5", size = 58557, upload-time = "2025-07-27T13:04:22.229Z" }, - { url = "https://files.pythonhosted.org/packages/76/09/0ec6bd2b2303b0ea5c6da7535edc9a608092075ef8c0cdd96e3e726cd687/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:528dba7ef1357bd7ce1aea143084501f47f5dd0fff7937d3906a68565aa59cfe", size = 63624, upload-time = "2025-07-27T13:04:23.952Z" }, - { url = "https://files.pythonhosted.org/packages/73/6e/52cb1ced2a517a3118b2e739e9417432049013ac7afa15d790103059e8e4/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:1da54be743d9a68671700cfe56c3ab8c26e8f2f5cc34eface905c55bc3a9af94", size = 56174, upload-time = "2025-07-27T13:04:25.419Z" }, - { url = "https://files.pythonhosted.org/packages/5b/9d/820fe79347467e48af985fe46180e1dd28e698ade7317bebd66de8a143f5/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:9b07c0406c3eaa7014499b0aacafb21a6d1146cfaa85d56f0aa02e6d542ee8f3", size = 72640, upload-time = "2025-07-27T13:04:26.824Z" }, - { url = "https://files.pythonhosted.org/packages/53/58/e863e10d08361e694935c815b73faad7e1ab03f99ae154d86c4e2f331896/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:312f2aa4cf5d199a97fbcaee75d2e59ebbaafcd091993eb373b43683498cdacb", size = 62453, upload-time = "2025-07-27T13:04:28.562Z" }, - { url = "https://files.pythonhosted.org/packages/95/f0/c392c4ac8ccb7a34b28377c21faa2395313e3c676d76c382642e19a20703/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad59362fc267bf15498a318c9e076686e4beeb0dfe09b457fabbc2b32468b97a", size = 58103, upload-time = "2025-07-27T13:04:29.996Z" }, - { url = "https://files.pythonhosted.org/packages/32/30/00ab21316e7df8f526aa3e3dc06f74de6711d51c65b020575d0105a025b2/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:01593bd064e7dcd6c86d04e94e44acfe364049500c20ac68ca1e708fbb2ca970", size = 60779, upload-time = "2025-07-27T13:04:31.549Z" }, - { url = "https://files.pythonhosted.org/packages/a6/65/114ca81839b1805ce4a2b7d58bc16e95634734a2059991f6382fc71caf3e/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5b81547ad8ea271c79fdf10da89a1e9313cb15edcba2a17adf8871735e9c02a0", size = 74684, upload-time = "2025-07-27T13:04:32.976Z" }, - { url = "https://files.pythonhosted.org/packages/54/8f/aa9d445b9bb693b8f6bb1456bd6d8576d79b7a63bf6c69af3a539235b15f/pybase64-1.4.2-cp313-cp313t-win32.whl", hash = "sha256:7edbe70b5654545a37e6e6b02de738303b1bbdfcde67f6cfec374cfb5cc4099e", size = 33961, upload-time = "2025-07-27T13:04:34.806Z" }, - { url = "https://files.pythonhosted.org/packages/0e/e5/da37cfb173c646fd4fc7c6aae2bc41d40de2ee49529854af8f4e6f498b45/pybase64-1.4.2-cp313-cp313t-win_amd64.whl", hash = "sha256:385690addf87c25d6366fab5d8ff512eed8a7ecb18da9e8152af1c789162f208", size = 36199, upload-time = "2025-07-27T13:04:36.223Z" }, - { url = "https://files.pythonhosted.org/packages/66/3e/1eb68fb7d00f2cec8bd9838e2a30d183d6724ae06e745fd6e65216f170ff/pybase64-1.4.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c2070d0aa88580f57fe15ca88b09f162e604d19282915a95a3795b5d3c1c05b5", size = 31221, upload-time = "2025-07-27T13:04:37.704Z" }, - { url = "https://files.pythonhosted.org/packages/99/bf/00a87d951473ce96c8c08af22b6983e681bfabdb78dd2dcf7ee58eac0932/pybase64-1.4.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4157ad277a32cf4f02a975dffc62a3c67d73dfa4609b2c1978ef47e722b18b8e", size = 30924, upload-time = "2025-07-27T13:04:39.189Z" }, - { url = "https://files.pythonhosted.org/packages/ae/43/dee58c9d60e60e6fb32dc6da722d84592e22f13c277297eb4ce6baf99a99/pybase64-1.4.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:e113267dc349cf624eb4f4fbf53fd77835e1aa048ac6877399af426aab435757", size = 31390, upload-time = "2025-07-27T13:04:40.995Z" }, - { url = "https://files.pythonhosted.org/packages/e1/11/b28906fc2e330b8b1ab4bc845a7bef808b8506734e90ed79c6062b095112/pybase64-1.4.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:cea5aaf218fd9c5c23afacfe86fd4464dfedc1a0316dd3b5b4075b068cc67df0", size = 38212, upload-time = "2025-07-27T13:04:42.729Z" }, - { url = "https://files.pythonhosted.org/packages/24/9e/868d1e104413d14b19feaf934fc7fad4ef5b18946385f8bb79684af40f24/pybase64-1.4.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:41213497abbd770435c7a9c8123fb02b93709ac4cf60155cd5aefc5f3042b600", size = 38303, upload-time = "2025-07-27T13:04:44.095Z" }, - { url = "https://files.pythonhosted.org/packages/a3/73/f7eac96ca505df0600280d6bfc671a9e2e2f947c2b04b12a70e36412f7eb/pybase64-1.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8b522df7ee00f2ac1993ccd5e1f6608ae7482de3907668c2ff96a83ef213925", size = 31669, upload-time = "2025-07-27T13:04:45.845Z" }, - { url = "https://files.pythonhosted.org/packages/c6/43/8e18bea4fd455100112d6a73a83702843f067ef9b9272485b6bdfd9ed2f0/pybase64-1.4.2-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:06725022e540c5b098b978a0418ca979773e2cbdbb76f10bd97536f2ad1c5b49", size = 68452, upload-time = "2025-07-27T13:04:47.788Z" }, - { url = "https://files.pythonhosted.org/packages/e4/2e/851eb51284b97354ee5dfa1309624ab90920696e91a33cd85b13d20cc5c1/pybase64-1.4.2-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a3e54dcf0d0305ec88473c9d0009f698cabf86f88a8a10090efeff2879c421bb", size = 71674, upload-time = "2025-07-27T13:04:49.294Z" }, - { url = "https://files.pythonhosted.org/packages/57/0d/5cf1e5dc64aec8db43e8dee4e4046856d639a72bcb0fb3e716be42ced5f1/pybase64-1.4.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67675cee727a60dc91173d2790206f01aa3c7b3fbccfa84fd5c1e3d883fe6caa", size = 60027, upload-time = "2025-07-27T13:04:50.769Z" }, - { url = "https://files.pythonhosted.org/packages/a4/8e/3479266bc0e65f6cc48b3938d4a83bff045330649869d950a378f2ddece0/pybase64-1.4.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:753da25d4fd20be7bda2746f545935773beea12d5cb5ec56ec2d2960796477b1", size = 56461, upload-time = "2025-07-27T13:04:52.37Z" }, - { url = "https://files.pythonhosted.org/packages/20/b6/f2b6cf59106dd78bae8717302be5b814cec33293504ad409a2eb752ad60c/pybase64-1.4.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a78c768ce4ca550885246d14babdb8923e0f4a848dfaaeb63c38fc99e7ea4052", size = 59446, upload-time = "2025-07-27T13:04:53.967Z" }, - { url = "https://files.pythonhosted.org/packages/16/70/3417797dfccdfdd0a54e4ad17c15b0624f0fc2d6a362210f229f5c4e8fd0/pybase64-1.4.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:51b17f36d890c92f0618fb1c8db2ccc25e6ed07afa505bab616396fc9b0b0492", size = 60350, upload-time = "2025-07-27T13:04:55.881Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c6/6e4269dd98d150ae95d321b311a345eae0f7fd459d97901b4a586d7513bb/pybase64-1.4.2-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:f92218d667049ab4f65d54fa043a88ffdb2f07fff1f868789ef705a5221de7ec", size = 54989, upload-time = "2025-07-27T13:04:57.436Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e8/18c1b0c255f964fafd0412b0d5a163aad588aeccb8f84b9bf9c8611d80f6/pybase64-1.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3547b3d1499919a06491b3f879a19fbe206af2bd1a424ecbb4e601eb2bd11fea", size = 58724, upload-time = "2025-07-27T13:04:59.406Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ad/ddfbd2125fc20b94865fb232b2e9105376fa16eee492e4b7786d42a86cbf/pybase64-1.4.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:958af7b0e09ddeb13e8c2330767c47b556b1ade19c35370f6451d139cde9f2a9", size = 52285, upload-time = "2025-07-27T13:05:01.198Z" }, - { url = "https://files.pythonhosted.org/packages/b6/4c/b9d4ec9224add33c84b925a03d1a53cd4106efb449ea8e0ae7795fed7bf7/pybase64-1.4.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4facc57f6671e2229a385a97a618273e7be36a9ea0a9d1c1b9347f14d19ceba8", size = 69036, upload-time = "2025-07-27T13:05:03.109Z" }, - { url = "https://files.pythonhosted.org/packages/92/38/7b96794da77bed3d9b4fea40f14ae563648fba83a696e7602fabe60c0eb7/pybase64-1.4.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a32fc57d05d73a7c9b0ca95e9e265e21cf734195dc6873829a890058c35f5cfd", size = 57938, upload-time = "2025-07-27T13:05:04.744Z" }, - { url = "https://files.pythonhosted.org/packages/eb/c5/ae8bbce3c322d1b074e79f51f5df95961fe90cb8748df66c6bc97616e974/pybase64-1.4.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3dc853243c81ce89cc7318e6946f860df28ddb7cd2a0648b981652d9ad09ee5a", size = 54474, upload-time = "2025-07-27T13:05:06.662Z" }, - { url = "https://files.pythonhosted.org/packages/15/9a/c09887c4bb1b43c03fc352e2671ef20c6686c6942a99106a45270ee5b840/pybase64-1.4.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0e6d863a86b3e7bc6ac9bd659bebda4501b9da842521111b0b0e54eb51295df5", size = 56533, upload-time = "2025-07-27T13:05:08.368Z" }, - { url = "https://files.pythonhosted.org/packages/4f/0f/d5114d63d35d085639606a880cb06e2322841cd4b213adfc14d545c1186f/pybase64-1.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6579475140ff2067903725d8aca47f5747bcb211597a1edd60b58f6d90ada2bd", size = 71030, upload-time = "2025-07-27T13:05:10.3Z" }, - { url = "https://files.pythonhosted.org/packages/40/0e/fe6f1ed22ea52eb99f490a8441815ba21de288f4351aeef4968d71d20d2d/pybase64-1.4.2-cp314-cp314-win32.whl", hash = "sha256:373897f728d7b4f241a1f803ac732c27b6945d26d86b2741ad9b75c802e4e378", size = 34174, upload-time = "2025-07-27T13:05:12.254Z" }, - { url = "https://files.pythonhosted.org/packages/71/46/0e15bea52ffc63e8ae7935e945accbaf635e0aefa26d3e31fdf9bc9dcd01/pybase64-1.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:1afe3361344617d298c1d08bc657ef56d0f702d6b72cb65d968b2771017935aa", size = 36308, upload-time = "2025-07-27T13:05:13.898Z" }, - { url = "https://files.pythonhosted.org/packages/4f/dc/55849fee2577bda77c1e078da04cc9237e8e474a8c8308deb702a26f2511/pybase64-1.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:f131c9360babe522f3d90f34da3f827cba80318125cf18d66f2ee27e3730e8c4", size = 31341, upload-time = "2025-07-27T13:05:15.553Z" }, - { url = "https://files.pythonhosted.org/packages/39/44/c69d088e28b25e70ac742b6789cde038473815b2a69345c4bae82d5e244d/pybase64-1.4.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2583ac304131c1bd6e3120b0179333610f18816000db77c0a2dd6da1364722a8", size = 38678, upload-time = "2025-07-27T13:05:17.544Z" }, - { url = "https://files.pythonhosted.org/packages/00/93/2860ec067497b9cbb06242f96d44caebbd9eed32174e4eb8c1ffef760f94/pybase64-1.4.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:75a8116be4ea4cdd30a5c4f1a6f3b038e0d457eb03c8a2685d8ce2aa00ef8f92", size = 32066, upload-time = "2025-07-27T13:05:19.18Z" }, - { url = "https://files.pythonhosted.org/packages/d3/55/1e96249a38759332e8a01b31c370d88c60ceaf44692eb6ba4f0f451ee496/pybase64-1.4.2-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:217ea776a098d7c08668e5526b9764f5048bbfd28cac86834217ddfe76a4e3c4", size = 72465, upload-time = "2025-07-27T13:05:20.866Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ab/0f468605b899f3e35dbb7423fba3ff98aeed1ec16abb02428468494a58f4/pybase64-1.4.2-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4ec14683e343c95b14248cdfdfa78c052582be7a3865fd570aa7cffa5ab5cf37", size = 75693, upload-time = "2025-07-27T13:05:22.896Z" }, - { url = "https://files.pythonhosted.org/packages/91/d1/9980a0159b699e2489baba05b71b7c953b29249118ba06fdbb3e9ea1b9b5/pybase64-1.4.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:480ecf21e1e956c5a10d3cf7b3b7e75bce3f9328cf08c101e4aab1925d879f34", size = 65577, upload-time = "2025-07-27T13:05:25Z" }, - { url = "https://files.pythonhosted.org/packages/16/86/b27e7b95f9863d245c0179a7245582eda3d262669d8f822777364d8fd7d5/pybase64-1.4.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:1fe1ebdc55e9447142e2f6658944aadfb5a4fbf03dbd509be34182585515ecc1", size = 60662, upload-time = "2025-07-27T13:05:27.138Z" }, - { url = "https://files.pythonhosted.org/packages/28/87/a7f0dde0abc26bfbee761f1d3558eb4b139f33ddd9fe1f6825ffa7daa22d/pybase64-1.4.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c793a2b06753accdaf5e1a8bbe5d800aab2406919e5008174f989a1ca0081411", size = 64179, upload-time = "2025-07-27T13:05:28.996Z" }, - { url = "https://files.pythonhosted.org/packages/1e/88/5d6fa1c60e1363b4cac4c396978f39e9df4689e75225d7d9c0a5998e3a14/pybase64-1.4.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6acae6e1d1f7ebe40165f08076c7a73692b2bf9046fefe673f350536e007f556", size = 64968, upload-time = "2025-07-27T13:05:30.818Z" }, - { url = "https://files.pythonhosted.org/packages/20/6e/2ed585af5b2211040445d9849326dd2445320c9316268794f5453cfbaf30/pybase64-1.4.2-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:88b91cd0949358aadcea75f8de5afbcf3c8c5fb9ec82325bd24285b7119cf56e", size = 58738, upload-time = "2025-07-27T13:05:32.629Z" }, - { url = "https://files.pythonhosted.org/packages/ce/94/e2960b56322eabb3fbf303fc5a72e6444594c1b90035f3975c6fe666db5c/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:53316587e1b1f47a11a5ff068d3cbd4a3911c291f2aec14882734973684871b2", size = 63802, upload-time = "2025-07-27T13:05:34.687Z" }, - { url = "https://files.pythonhosted.org/packages/95/47/312139d764c223f534f751528ce3802887c279125eac64f71cd3b4e05abc/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:caa7f20f43d00602cf9043b5ba758d54f5c41707d3709b2a5fac17361579c53c", size = 56341, upload-time = "2025-07-27T13:05:36.554Z" }, - { url = "https://files.pythonhosted.org/packages/3f/d7/aec9a6ed53b128dac32f8768b646ca5730c88eef80934054d7fa7d02f3ef/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:2d93817e24fdd79c534ed97705df855af6f1d2535ceb8dfa80da9de75482a8d7", size = 72838, upload-time = "2025-07-27T13:05:38.459Z" }, - { url = "https://files.pythonhosted.org/packages/e3/a8/6ccc54c5f1f7c3450ad7c56da10c0f131d85ebe069ea6952b5b42f2e92d9/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:63cd769b51474d8d08f7f2ce73b30380d9b4078ec92ea6b348ea20ed1e1af88a", size = 62633, upload-time = "2025-07-27T13:05:40.624Z" }, - { url = "https://files.pythonhosted.org/packages/34/22/2b9d89f8ff6f2a01d6d6a88664b20a4817049cfc3f2c62caca040706660c/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cd07e6a9993c392ec8eb03912a43c6a6b21b2deb79ee0d606700fe276e9a576f", size = 58282, upload-time = "2025-07-27T13:05:42.565Z" }, - { url = "https://files.pythonhosted.org/packages/b2/14/dbf6266177532a6a11804ac080ebffcee272f491b92820c39886ee20f201/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6a8944e8194adff4668350504bc6b7dbde2dab9244c88d99c491657d145b5af5", size = 60948, upload-time = "2025-07-27T13:05:44.48Z" }, - { url = "https://files.pythonhosted.org/packages/fd/7a/b2ae9046a66dd5746cd72836a41386517b1680bea5ce02f2b4f1c9ebc688/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:04ab398ec4b6a212af57f6a21a6336d5a1d754ff4ccb215951366ab9080481b2", size = 74854, upload-time = "2025-07-27T13:05:46.416Z" }, - { url = "https://files.pythonhosted.org/packages/ef/7e/9856f6d6c38a7b730e001123d2d9fa816b8b1a45f0cdee1d509d5947b047/pybase64-1.4.2-cp314-cp314t-win32.whl", hash = "sha256:3b9201ecdcb1c3e23be4caebd6393a4e6615bd0722528f5413b58e22e3792dd3", size = 34490, upload-time = "2025-07-27T13:05:48.304Z" }, - { url = "https://files.pythonhosted.org/packages/c7/38/8523a9dc1ec8704dedbe5ccc95192ae9a7585f7eec85cc62946fe3cacd32/pybase64-1.4.2-cp314-cp314t-win_amd64.whl", hash = "sha256:36e9b0cad8197136d73904ef5a71d843381d063fd528c5ab203fc4990264f682", size = 36680, upload-time = "2025-07-27T13:05:50.264Z" }, - { url = "https://files.pythonhosted.org/packages/3c/52/5600104ef7b85f89fb8ec54f73504ead3f6f0294027e08d281f3cafb5c1a/pybase64-1.4.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f25140496b02db0e7401567cd869fb13b4c8118bf5c2428592ec339987146d8b", size = 31600, upload-time = "2025-07-27T13:05:52.24Z" }, - { url = "https://files.pythonhosted.org/packages/32/34/b67371f4fcedd5e2def29b1cf92a4311a72f590c04850f370c75297b48ce/pybase64-1.4.2-graalpy311-graalpy242_311_native-macosx_10_9_x86_64.whl", hash = "sha256:b4eed40a5f1627ee65613a6ac834a33f8ba24066656f569c852f98eb16f6ab5d", size = 38667, upload-time = "2025-07-27T13:07:25.315Z" }, - { url = "https://files.pythonhosted.org/packages/aa/3e/e57fe09ed1c7e740d21c37023c5f7c8963b4c36380f41d10261cc76f93b4/pybase64-1.4.2-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:57885fa521e9add235af4db13e9e048d3a2934cd27d7c5efac1925e1b4d6538d", size = 32094, upload-time = "2025-07-27T13:07:28.235Z" }, - { url = "https://files.pythonhosted.org/packages/51/34/f40d3262c3953814b9bcdcf858436bd5bc1133a698be4bcc7ed2a8c0730d/pybase64-1.4.2-graalpy311-graalpy242_311_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eef9255d926c64e2fca021d3aee98023bacb98e1518e5986d6aab04102411b04", size = 43212, upload-time = "2025-07-27T13:07:31.327Z" }, - { url = "https://files.pythonhosted.org/packages/8c/2a/5e05d25718cb8ffd68bd46553ddfd2b660893d937feda1716b8a3b21fb38/pybase64-1.4.2-graalpy311-graalpy242_311_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89614ea2d2329b6708746c540e0f14d692125df99fb1203ff0de948d9e68dfc9", size = 35789, upload-time = "2025-07-27T13:07:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/d5/9d/f56c3ee6e94faaae2896ecaf666428330cb24096abf7d2427371bb2b403a/pybase64-1.4.2-graalpy311-graalpy242_311_native-win_amd64.whl", hash = "sha256:e401cecd2d7ddcd558768b2140fd4430746be4d17fb14c99eec9e40789df136d", size = 35861, upload-time = "2025-07-27T13:07:37.099Z" }, - { url = "https://files.pythonhosted.org/packages/fb/04/bfe2bd0d76385750f3541724b4abfe4ea111b3cc01ff7e83f410054adc30/pybase64-1.4.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b29c93414ba965777643a9d98443f08f76ac04519ad717aa859113695372a07", size = 38226, upload-time = "2025-07-27T13:07:40.121Z" }, - { url = "https://files.pythonhosted.org/packages/22/13/c717855760b78ded1a9d308984c7e3e99fcf79c6cac5a231ed8c1238218f/pybase64-1.4.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5e0c3353c0bf099c5c3f8f750202c486abee8f23a566b49e9e7b1222fbf5f259", size = 31524, upload-time = "2025-07-27T13:07:43.946Z" }, - { url = "https://files.pythonhosted.org/packages/cf/da/2b7e69abfc62abe4d54b10d1e09ec78021a6b9b2d7e6e7b632243a19433e/pybase64-1.4.2-pp310-pypy310_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4f98c5c6152d3c01d933fcde04322cd9ddcf65b5346034aac69a04c1a7cbb012", size = 40667, upload-time = "2025-07-27T13:07:46.715Z" }, - { url = "https://files.pythonhosted.org/packages/f1/11/ba738655fb3ba85c7a0605eddd2709fef606e654840c72ee5c5ff7ab29bf/pybase64-1.4.2-pp310-pypy310_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9096a4977b7aff7ef250f759fb6a4b6b7b6199d99c84070c7fc862dd3b208b34", size = 41290, upload-time = "2025-07-27T13:07:49.534Z" }, - { url = "https://files.pythonhosted.org/packages/5d/38/2d5502fcaf712297b95c1b6ca924656dd7d17501fd7f9c9e0b3bbf8892ef/pybase64-1.4.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49d8597e2872966399410502310b1e2a5b7e8d8ba96766ee1fe242e00bd80775", size = 35438, upload-time = "2025-07-27T13:07:52.327Z" }, - { url = "https://files.pythonhosted.org/packages/b6/db/e03b8b6daa60a3fbef21741403e0cf18b2aff3beebdf6e3596bb9bab16c7/pybase64-1.4.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2ef16366565389a287df82659e055e88bdb6c36e46a3394950903e0a9cb2e5bf", size = 36121, upload-time = "2025-07-27T13:07:55.54Z" }, - { url = "https://files.pythonhosted.org/packages/0e/bf/5ebaa2d9ddb5fc506633bc8b820fc27e64da964937fb30929c0367c47d00/pybase64-1.4.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0a5393be20b0705870f5a8969749af84d734c077de80dd7e9f5424a247afa85e", size = 38162, upload-time = "2025-07-27T13:07:58.364Z" }, - { url = "https://files.pythonhosted.org/packages/25/41/795c5fd6e5571bb675bf9add8a048166dddf8951c2a903fea8557743886b/pybase64-1.4.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:448f0259a2f1a17eb086f70fe2ad9b556edba1fc5bc4e62ce6966179368ee9f8", size = 31452, upload-time = "2025-07-27T13:08:01.259Z" }, - { url = "https://files.pythonhosted.org/packages/aa/dd/c819003b59b2832256b72ad23cbeadbd95d083ef0318d07149a58b7a88af/pybase64-1.4.2-pp311-pypy311_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:1159e70cba8e76c3d8f334bd1f8fd52a1bb7384f4c3533831b23ab2df84a6ef3", size = 40668, upload-time = "2025-07-27T13:08:04.176Z" }, - { url = "https://files.pythonhosted.org/packages/0e/c5/38c6aba28678c4a4db49312a6b8171b93a0ffe9f21362cf4c0f325caa850/pybase64-1.4.2-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7d943bc5dad8388971494554b97f22ae06a46cc7779ad0de3d4bfdf7d0bbea30", size = 41281, upload-time = "2025-07-27T13:08:07.395Z" }, - { url = "https://files.pythonhosted.org/packages/e5/23/5927bd9e59714e4e8cefd1d21ccd7216048bb1c6c3e7104b1b200afdc63d/pybase64-1.4.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b99182c561d86422c5de4265fd1f8f172fb38efaed9d72c71fb31e279a7f94", size = 35433, upload-time = "2025-07-27T13:08:10.551Z" }, - { url = "https://files.pythonhosted.org/packages/01/0f/fab7ed5bf4926523c3b39f7621cea3e0da43f539fbc2270e042f1afccb79/pybase64-1.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bb082c1114f046e59fcbc4f2be13edc93b36d7b54b58605820605be948f8fdf6", size = 36131, upload-time = "2025-07-27T13:08:13.777Z" }, -] - [[package]] name = "pycairo" version = "1.28.0" @@ -6562,49 +6483,57 @@ wheels = [ [[package]] name = "smithy-aws-core" -version = "0.0.3" +version = "0.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aws-sdk-signers", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9b/20/262da16a1e41ffaa2865ad8bf5c1b98f92a5ecdc19022f2e9016c0acc21d/smithy_aws_core-0.0.3.tar.gz", hash = "sha256:ba891626798eec914a6b73c9cb5e1155f0767cc6b34190be582a49caba06c43d", size = 8370, upload-time = "2025-06-17T18:12:31.359Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/d3/f847e0fd36b95aa36ce3a4c9ce1a08e16b2aa9a56b71714045c9c924e282/smithy_aws_core-0.1.1.tar.gz", hash = "sha256:78dfd7040fc2bc72b6af293096642fc9a7bfd2db28ddbdf7c4110535eab9d662", size = 11196, upload-time = "2025-10-21T20:21:18.648Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/de/4d9652a3199dbed6f82d6f6a9f4f15f8f88e941a3f84b9479c0cb1009981/smithy_aws_core-0.0.3-py3-none-any.whl", hash = "sha256:cb17d9c0fc4e810c34ea67fcc60f9c72407c176dde9360fa3f2795e63677a75a", size = 15271, upload-time = "2025-06-17T18:12:30.599Z" }, + { url = "https://files.pythonhosted.org/packages/d0/04/87cb06f0f6d664b5cffdef6d4042dd52c11c138436084d30ffdaa3543031/smithy_aws_core-0.1.1-py3-none-any.whl", hash = "sha256:0d1634f276c2999dc2a04fafef63b9d28309de50d939d1d49df952773a7063c4", size = 18963, upload-time = "2025-10-21T20:21:17.692Z" }, +] + +[package.optional-dependencies] +eventstream = [ + { name = "smithy-aws-event-stream", marker = "python_full_version >= '3.12'" }, +] +json = [ + { name = "smithy-json", marker = "python_full_version >= '3.12'" }, ] [[package]] name = "smithy-aws-event-stream" -version = "0.0.1" +version = "0.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d8/97/0e4ad1cdf92e46502262b507e9d4c28fe3559ff7dcffb7358db1d57ef043/smithy_aws_event_stream-0.0.1.tar.gz", hash = "sha256:4c4369146a0194790d0169cf86ffaa9feefe4f5ffec393ddeae55e86b56c088e", size = 11399, upload-time = "2025-04-07T19:44:39.336Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/26/8ff24194efed60b2df18f610ea05fa2a4c6546858b80a0a51335a4943b9b/smithy_aws_event_stream-0.1.0.tar.gz", hash = "sha256:6634691a3bf5d4801a2c29f0761db2dc4771f3ae43cdee50c10d4b4bb2f86475", size = 12216, upload-time = "2025-09-29T19:37:14.659Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/b4/8237238261927aee8b583fe3bde95c51367725a59213f3f39e3e979207eb/smithy_aws_event_stream-0.0.1-py3-none-any.whl", hash = "sha256:d5c0dd9a117e9dd927bafa857072fd26ccfa208a2768e5600ea628aebc93696b", size = 14992, upload-time = "2025-04-07T19:44:37.877Z" }, + { url = "https://files.pythonhosted.org/packages/90/c4/2b63d31af58fc359577e5515bf730348a235f2f2fa10e17af8640495c81c/smithy_aws_event_stream-0.1.0-py3-none-any.whl", hash = "sha256:17a7300a85cb90df4c6c23f895ca6343361fa419203c3cf80019edd7d3b5f036", size = 15581, upload-time = "2025-09-29T19:37:13.589Z" }, ] [[package]] name = "smithy-core" -version = "0.0.2" +version = "0.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/30/cf766866ff5536cea5b4eec71efe97326a54f62a10d9e2615d303e9ff3fe/smithy_core-0.0.2.tar.gz", hash = "sha256:f81bc9a3008bc791dd025ce150802ccf564f2289afd093589b38f506e91327b5", size = 41327, upload-time = "2025-04-09T16:02:09.289Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/8d/16028d03456071d21de7591f1e1e6a1cc81b2389e53ef8663dbf59caf9cd/smithy_core-0.1.0.tar.gz", hash = "sha256:b159b8905264e1e4c613eab9f74cec0b2f5b8119c42fbadddb4da0a8ed8050e9", size = 48415, upload-time = "2025-09-29T19:37:16.873Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/2c/cc1c09352df57cc2f0a53e670042fc58902236c72b29ec1eed462695cddf/smithy_core-0.0.2-py3-none-any.whl", hash = "sha256:9f2a929e3c04d581448c5437cf69681c77bbc24656c3c8e30f9c2479963a4941", size = 53860, upload-time = "2025-04-09T16:02:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/ca/5b/563cb2beadcfa40597b0c3ff3f2d42e21f065b14782c4ba9cb41a44b745f/smithy_core-0.1.0-py3-none-any.whl", hash = "sha256:cb44e9355fb89e89f2c6ba6a1d59c5db4f2f7282c72d31d9307b6202d66cd0fa", size = 62895, upload-time = "2025-09-29T19:37:15.917Z" }, ] [[package]] name = "smithy-http" -version = "0.0.1" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/76/29/670680189584f6f282bf97c3d016660a04037119600fa1e1eb438e2a76b7/smithy_http-0.0.1.tar.gz", hash = "sha256:214d0f45a75078654c80ec13d518dcb690dcbec8b11a9a65b4cc2fe108c9bc33", size = 25050, upload-time = "2025-04-07T19:43:59.388Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/1c/44e99a7dfb8c39bf0c3d998accdf4573a7a3488863b90f21af260cec2d45/smithy_http-0.2.0.tar.gz", hash = "sha256:2382562fa9af326be455f14b18615f16ffe9db756e51b2a4ca0d23e1b881cff8", size = 26729, upload-time = "2025-10-21T20:21:06.146Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/37/7b/64efb9237630e6ec64f6c6ee6eef1dfbfd1aa5013788ea72edee378cdf39/smithy_http-0.0.1-py3-none-any.whl", hash = "sha256:b25ff39604de6998adc842455138c58411d2adac9b1130d58f115eea2f109f77", size = 35604, upload-time = "2025-04-07T19:43:58.024Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e2/d475fad81ac74ec0e145cb6d72afe5ecde4e2358bd632c2fd5d3f4bc87dc/smithy_http-0.2.0-py3-none-any.whl", hash = "sha256:49ee2402d7737798d70f99f491fbfb2a5767283ae562e21b6f86e3fd14f3e3e0", size = 37328, upload-time = "2025-10-21T20:21:05.362Z" }, ] [package.optional-dependencies] @@ -6614,15 +6543,15 @@ awscrt = [ [[package]] name = "smithy-json" -version = "0.0.1" +version = "0.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ijson", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/be/51/a0d795aac06233c93fbf97cfd54c6962e868f5611e5fb20e8a45e4bcc56f/smithy_json-0.0.1.tar.gz", hash = "sha256:97c559e559654892dbcf561a3e5fb73ebffc45ed6329cba08792f2a12e6487ff", size = 6095, upload-time = "2025-04-07T19:44:16.41Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e2/5b/0ecb10007475e1b8faca3bbff1be2fc6edb3ea12ffc5e939e2249be95325/smithy_json-0.1.0.tar.gz", hash = "sha256:84fb48e445b87d850c240d837702c16b259ea53bad76b655ac1bbd8094d48912", size = 7086, upload-time = "2025-09-29T19:37:20.432Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/b4/c5d207759ad967684a97d6ff2f24835d926a5fe4f96db8ea244e141a71b4/smithy_json-0.0.1-py3-none-any.whl", hash = "sha256:50d3b441c369bc16507f271699ad4f73e3961fec762c0d827a0e17709424948c", size = 8903, upload-time = "2025-04-07T19:44:15.138Z" }, + { url = "https://files.pythonhosted.org/packages/62/95/e11c04e56aae12b62e38c49000004a1dc598a64dc207018c08448efde322/smithy_json-0.1.0-py3-none-any.whl", hash = "sha256:80ff64734dccdabf1ba6a2908555b97e60f62c07c3a27df48e421ee058413cb9", size = 9914, upload-time = "2025-09-29T19:37:19.459Z" }, ] [[package]] @@ -6690,14 +6619,14 @@ wheels = [ [[package]] name = "speechmatics-rt" -version = "0.4.2" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/2e/d694390d58b9b6807280441d1275856f5a316c3e8a815c2037502636bbea/speechmatics_rt-0.4.2.tar.gz", hash = "sha256:c0f7ed34442b0f505a12d1b19c8cc8dc2cc0b1a423aeb5669ca0738fc5e59f0d", size = 26142, upload-time = "2025-09-30T10:50:36.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/57/26/10359e1f16c2aa6a198eb11a9056f4a86a8bb8d4e610bbbe4a118b227b59/speechmatics_rt-0.5.0.tar.gz", hash = "sha256:ca974a186a012f946fd997deeaf3bf1c4f203f6d6e05a866172d27709183afc8", size = 26832, upload-time = "2025-10-15T15:54:25.695Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/c7/2cd551c71e14256ca463f31feec17f466b57c2730d636e20803e7a541104/speechmatics_rt-0.4.2-py3-none-any.whl", hash = "sha256:70b91ff750e2f7516eaf1839d39f7a8ac65ff6665638b837cf67bab9cc9967bc", size = 32131, upload-time = "2025-09-30T10:50:35.656Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/9931ebe9360e9d385c68826b33137c2c9a4cfa361cd929d1ac6e72ebfe53/speechmatics_rt-0.5.0-py3-none-any.whl", hash = "sha256:58151488f891fa00cf7054f0cfab1b1eb94b55c3441be587f7941c726caef991", size = 32850, upload-time = "2025-10-15T15:54:24.5Z" }, ] [[package]] @@ -7522,7 +7451,7 @@ wheels = [ [[package]] name = "vllm" -version = "0.9.2" +version = "0.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -7546,6 +7475,10 @@ dependencies = [ { name = "numpy" }, { name = "openai" }, { name = "opencv-python-headless" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions-ai" }, { name = "outlines" }, { name = "partial-json-parser" }, { name = "pillow" }, @@ -7554,7 +7487,6 @@ dependencies = [ { name = "protobuf" }, { name = "psutil" }, { name = "py-cpuinfo" }, - { name = "pybase64" }, { name = "pydantic" }, { name = "python-json-logger" }, { name = "pyyaml" }, @@ -7577,11 +7509,11 @@ dependencies = [ { name = "typing-extensions" }, { name = "watchfiles" }, { name = "xformers", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/35/89/2fbf95d398b5751b44c7256bd80e57c589142f1bfcc15f5dc76438b8853a/vllm-0.9.2.tar.gz", hash = "sha256:6b0d855ea8ba18d76364c9b82ea94bfcaa9c9e724055438b5733e4716ed104e1", size = 8997087, upload-time = "2025-07-08T04:49:01.722Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c5/5b/5f42b41d045c01821be62162fc6b1cfb14db1674027c7b623adb3a66dccf/vllm-0.9.1.tar.gz", hash = "sha256:c5ad11603f49a1fad05c88debabb8b839780403ce1b51751ec4da4e8a838082c", size = 8670972, upload-time = "2025-06-10T21:46:12.114Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/72/c14ff1acac64294f45782769b9c8144a1c3e8d4f2228d4648197511b015a/vllm-0.9.2-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:f3c5da29a286f4933b480a5b4749fab226564f35c96928eeef547f88d385cd34", size = 383350132, upload-time = "2025-07-08T04:48:54.133Z" }, + { url = "https://files.pythonhosted.org/packages/b5/56/ffcf6215a571cf9aa58ded06a9640bff21b4918e27344677cd33290ab9da/vllm-0.9.1-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:28b99e8df39c7aaeda04f7e5353b18564a1a9d1c579691945523fc4777a1a8c8", size = 394637693, upload-time = "2025-06-10T21:46:01.784Z" }, ] [[package]] @@ -7891,7 +7823,6 @@ name = "xgrammar" version = "0.1.19" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, { name = "ninja" }, { name = "pydantic" }, { name = "sentencepiece" },