diff --git a/CHANGELOG.md b/CHANGELOG.md index 10cd8738f..3cd715776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,63 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added some new configuration options to `GeminiMultimodalLiveLLMService`: +- Added `on_connected` and `on_disconnected` events to TTS and STT + websocket-based services. + +- Added an `aggregate_sentences` arg in `ElevenLabsHttpTTSService`, where the + default value is True. + +- Added a `room_properties` arg to the Daily runner's `configure()` method, + allowing `DailyRoomProperties` to be provided. + +- The runner `--folder` argument now supports downloading files from + subdirectories. + +### Changed + +- `CartesiaSTTService` now inherits from `WebsocketSTTService`. + +- Package upgrades: + - `openai` upgraded to support up to 2.x.x. + - `openpipe` upgraded to support up to 5.x.x. + +### Fixed + +- Fixed an issue in `ElevenLabsTTSService` and `ElevenLabsHttpTTSService` where + the Flash models would split words, resulting in a space being inserted + between words. + +- Fixed an issue where audio filters' `stop()` would not be called when using + `CancelFrame`. + +- Fixed an issue in `ElevenLabsHttpTTSService`, where + `apply_text_normalization` was incorrectly set as a query parameter. It's now + being added as a request parameter. + +- Fixed an issue where `RimeHttpTTSService` and `PiperTTSService` could generate + incorrectly 16-bit aligned audio frames, potentially leading to internal + errors or static audio. + +### Other + +- Added foundational example `47-sentry-metrics.py`, demonstrating how to use the + `SentryMetrics` processor. + +- Added foundational example `14x-function-calling-openpipe.py`. + +## [0.0.90] - 2025-10-10 + +### Added + +- Added audio filter `KrispVivaFilter` using the Krisp VIVA SDK. + +- Added `--folder` argument to the runner, allowing files saved in that folder + to be downloaded from `http://HOST:PORT/file/FILE`. + +- Added `GeminiLiveVertexLLMService`, for accessing Gemini Live via Google + Vertex AI. + +- Added some new configuration options to `GeminiLiveLLMService`: - `thinking` - `enable_affective_dialog` @@ -34,16 +90,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Updated `GeminiMultimodalLiveLLMService` to use the `google-genai` library - rather than use WebSockets directly. +- Updated `GeminiLiveLLMService` to use the `google-genai` library rather than + use WebSockets directly. + +### Deprecated + +- `LivekitFrameSerializer` is now deprecated. Use `LiveKitTransport` instead. + +- `pipecat.service.openai_realtime` is now deprecated, use + `pipecat.services.openai.realtime` instead or + `pipecat.services.azure.realtime` for Azure Realtime. + +- `pipecat.service.aws_nova_sonic` is now deprecated, use + `pipecat.services.aws.nova_sonic` instead. + +- `GeminiMultimodalLiveLLMService` is now deprecated, use + `GeminiLiveLLMService`. ### Fixed -- `GeminiMultimodalLiveLLMService` will now end gracefully (i.e. after the bot - has finished) upon receiving an `EndFrame`. +- Fixed a `GoogleVertexLLMService` issue that would generate an error if no + token information was returned. -- `GeminiMultimodalLiveLLMService` will try to seamlessly reconnect when it - loses its connection. +- `GeminiLiveLLMService` will now end gracefully (i.e. after the bot has + finished) upon receiving an `EndFrame`. + +- `GeminiLiveLLMService` will try to seamlessly reconnect when it loses its + connection. ## [0.0.89] - 2025-10-07 @@ -1462,7 +1535,7 @@ quality and critical bugs impacting `ParallelPipelines` functionality.** - Added `session_token` parameter to `AWSNovaSonicLLMService`. - Added Gemini Multimodal Live File API for uploading, fetching, listing, and - deleting files. See `26f-gemini-multimodal-live-files-api.py` for example usage. + deleting files. See `26f-gemini-live-files-api.py` for example usage. ### Changed @@ -3468,7 +3541,7 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Added the new modalities option and helper function to set Gemini output modalities. -- Added `examples/foundational/26d-gemini-multimodal-live-text.py` which is +- Added `examples/foundational/26d-gemini-live-text.py` which is using Gemini as TEXT modality and using another TTS provider for TTS process. ### Changed @@ -3655,9 +3728,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Added new foundational examples for `GeminiMultimodalLiveLLMService`: - `26-gemini-multimodal-live.py` - - `26a-gemini-multimodal-live-transcription.py` - - `26b-gemini-multimodal-live-video.py` - - `26c-gemini-multimodal-live-video.py` + - `26a-gemini-live-transcription.py` + - `26b-gemini-live-video.py` + - `26c-gemini-live-video.py` - Added `SimliVideoService`. This is an integration for Simli AI avatars. (see https://www.simli.com) diff --git a/README.md b/README.md index 3eb9bf346..6b00832f3 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/pipecat-ai/pipecat) +[![](https://getmanta.ai/api/badges?text=Manta%20Graph&link=manta)](https://getmanta.ai/pipecat) # 🎙️ Pipecat: Real-Time Voice & Multimodal AI Agents @@ -62,24 +63,24 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout
  - +

## 🧩 Available services -| Category | Services | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | +| Category | Services | +| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | -| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | -| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) | -| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | -| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | -| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | -| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | +| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | +| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) | +| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | +| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) | +| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) diff --git a/docs/api/conf.py b/docs/api/conf.py index a3062b612..8946a34d3 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -50,6 +50,7 @@ autodoc_mock_imports = [ # Krisp - has build issues on some platforms "pipecat_ai_krisp", "krisp", + "krisp_audio", # System-specific GUI libraries "_tkinter", "tkinter", diff --git a/env.example b/env.example index 22768494b..f707f49c9 100644 --- a/env.example +++ b/env.example @@ -90,6 +90,9 @@ SIMLI_FACE_ID=... # Krisp KRISP_MODEL_PATH=... +# Krisp Viva +KRISP_VIVA_MODEL_PATH=... + # DeepSeek DEEPSEEK_API_KEY=... diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 81ba692c7..1e7bd5718 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -21,8 +21,8 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.stt import CartesiaSTTService from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -58,7 +58,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), diff --git a/examples/foundational/07p-interruptible-krisp-viva.py b/examples/foundational/07p-interruptible-krisp-viva.py new file mode 100644 index 000000000..c7ca15b40 --- /dev/null +++ b/examples/foundational/07p-interruptible-krisp-viva.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.deepgram.tts import DeepgramTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=KrispVivaFilter(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=KrispVivaFilter(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=KrispVivaFilter(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/13f-cartesia-transcription.py b/examples/foundational/13f-cartesia-transcription.py index 913dce797..d685d0ba8 100644 --- a/examples/foundational/13f-cartesia-transcription.py +++ b/examples/foundational/13f-cartesia-transcription.py @@ -48,10 +48,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = CartesiaSTTService( - api_key=os.getenv("CARTESIA_API_KEY"), - base_url=os.getenv("CARTESIA_BASE_URL"), - ) + stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY")) tl = TranscriptionLogger() diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index f9ad264eb..cba5eee60 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -76,9 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleVertexLLMService( credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), - params=GoogleVertexLLMService.InputParams( - project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), - ), + project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), + location=os.getenv("GOOGLE_CLOUD_LOCATION"), ) # You can aslo register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. diff --git a/examples/foundational/14x-function-calling-openpipe.py b/examples/foundational/14x-function-calling-openpipe.py new file mode 100644 index 000000000..3f2537bb7 --- /dev/null +++ b/examples/foundational/14x-function-calling-openpipe.py @@ -0,0 +1,182 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os +import time + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.services.openpipe.llm import OpenPipeLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +async def fetch_weather_from_api(params: FunctionCallParams): + await params.result_callback({"conditions": "nice", "temperature": "75"}) + + +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + timestamp = int(time.time()) + llm = OpenPipeLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + openpipe_api_key=os.getenv("OPENPIPE_API_KEY"), + tags={"conversation_id": f"pipecat-{timestamp}"}, + ) + + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 0f9309f3d..f182d7c8c 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -24,14 +24,15 @@ from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioNoiseReduction, InputAudioTranscription, - OpenAIRealtimeLLMService, SemanticTurnDetection, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index a39826b81..c4b0fc02a 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -21,13 +21,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport +from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( - AzureRealtimeLLMService, +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioTranscription, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py index b5d1d73e1..bb63a4814 100644 --- a/examples/foundational/19b-openai-realtime-text.py +++ b/examples/foundational/19b-openai-realtime-text.py @@ -22,16 +22,17 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioNoiseReduction, InputAudioTranscription, - OpenAIRealtimeLLMService, SemanticTurnDetection, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 52c7d8f49..629a17c67 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -25,13 +25,14 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioTranscription, - OpenAIRealtimeLLMService, SessionProperties, TurnDetection, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index bd3d9d545..2161aff8f 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.aws_nova_sonic.aws import AWSNovaSonicLLMService +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-live.py similarity index 97% rename from examples/foundational/26-gemini-multimodal-live.py rename to examples/foundational/26-gemini-live.py index b446a9b8c..dd5dfcffc 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-live.py @@ -17,7 +17,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): Respond to what the user said in a creative and helpful way. """ - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-live-transcription.py similarity index 97% rename from examples/foundational/26a-gemini-multimodal-live-transcription.py rename to examples/foundational/26a-gemini-live-transcription.py index ad3cd06ee..de277156b 100644 --- a/examples/foundational/26a-gemini-multimodal-live-transcription.py +++ b/examples/foundational/26a-gemini-live-transcription.py @@ -20,7 +20,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -65,7 +65,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-live-function-calling.py similarity index 95% rename from examples/foundational/26b-gemini-multimodal-live-function-calling.py rename to examples/foundational/26b-gemini-live-function-calling.py index f14713a5c..65d159bb0 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-live-function-calling.py @@ -22,7 +22,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -122,12 +122,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): required=["location"], ) search_tool = {"google_search": {}} + # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears + # you cannot use the "google_search" tool alongside other tools. + # See https://github.com/googleapis/python-genai/issues/941. tools = ToolsSchema( standard_tools=[weather_function, restaurant_function], custom_tools={AdapterType.GEMINI: [search_tool]}, ) - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, tools=tools, diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-live-video.py similarity index 96% rename from examples/foundational/26c-gemini-multimodal-live-video.py rename to examples/foundational/26c-gemini-live-video.py index a28eaaacf..7b765075e 100644 --- a/examples/foundational/26c-gemini-multimodal-live-video.py +++ b/examples/foundational/26c-gemini-live-video.py @@ -24,7 +24,7 @@ from pipecat.runner.utils import ( maybe_capture_participant_camera, maybe_capture_participant_screen, ) -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -58,7 +58,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-live-text.py similarity index 91% rename from examples/foundational/26d-gemini-multimodal-live-text.py rename to examples/foundational/26d-gemini-live-text.py index 667887b03..062c0231b 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-live-text.py @@ -20,9 +20,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.gemini_multimodal_live.gemini import ( - GeminiMultimodalLiveLLMService, - GeminiMultimodalModalities, +from pipecat.services.google.gemini_live.llm import ( + GeminiLiveLLMService, + GeminiModalities, InputParams, ) from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -80,11 +80,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - llm = GeminiMultimodalLiveLLMService( + # KNOWN ISSUE: If using GeminiLiveVertexLLMService, you cannot specify a + # modality other than AUDIO (at least not if using the service's default + # model, which is a native audio model: + # https://cloud.google.com/vertex-ai/generative-ai/docs/live-api/tools#native-audio). + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=SYSTEM_INSTRUCTION, tools=[{"google_search": {}}, {"code_execution": {}}], - params=InputParams(modalities=GeminiMultimodalModalities.TEXT), + params=InputParams(modalities=GeminiModalities.TEXT), ) # Optionally, you can set the response modalities via a function diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-live-google-search.py similarity index 97% rename from examples/foundational/26e-gemini-multimodal-google-search.py rename to examples/foundational/26e-gemini-live-google-search.py index 31fad54fe..178fdd282 100644 --- a/examples/foundational/26e-gemini-multimodal-google-search.py +++ b/examples/foundational/26e-gemini-live-google-search.py @@ -19,7 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") # Initialize the Gemini Multimodal Live model - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck system_instruction=system_instruction, diff --git a/examples/foundational/26f-gemini-multimodal-live-files-api.py b/examples/foundational/26f-gemini-live-files-api.py similarity index 98% rename from examples/foundational/26f-gemini-multimodal-live-files-api.py rename to examples/foundational/26f-gemini-live-files-api.py index b01c21803..eeda16f52 100644 --- a/examples/foundational/26f-gemini-multimodal-live-files-api.py +++ b/examples/foundational/26f-gemini-live-files-api.py @@ -19,9 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import ( - GeminiMultimodalLiveLLMService, -) +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -110,7 +108,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ # Initialize Gemini service with File API support - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py b/examples/foundational/26g-gemini-live-groundingMetadata.py similarity index 97% rename from examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py rename to examples/foundational/26g-gemini-live-groundingMetadata.py index 6cfeed51b..bea1756b2 100644 --- a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py +++ b/examples/foundational/26g-gemini-live-groundingMetadata.py @@ -9,13 +9,13 @@ from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import Frame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.services.google.frames import LLMSearchResponseFrame +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -105,7 +105,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): custom_tools={AdapterType.GEMINI: [{"google_search": {}}, {"code_execution": {}}]}, ) - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=SYSTEM_INSTRUCTION, voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck diff --git a/examples/foundational/26h-gemini-live-vertex-function-calling.py b/examples/foundational/26h-gemini-live-vertex-function-calling.py new file mode 100644 index 000000000..c0344a052 --- /dev/null +++ b/examples/foundational/26h-gemini-live-vertex-function-calling.py @@ -0,0 +1,191 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os +from datetime import datetime + +from dotenv import load_dotenv +from google.genai.types import HttpOptions +from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService +from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +async def fetch_weather_from_api(params: FunctionCallParams): + temperature = 75 if params.arguments["format"] == "fahrenheit" else 24 + await params.result_callback( + { + "conditions": "nice", + "temperature": temperature, + "format": params.arguments["format"], + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + } + ) + + +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + +system_instruction = """ +You are a helpful assistant who can answer questions and use tools. + +You have three tools available to you: +1. get_current_weather: Use this tool to get the current weather in a specific location. +2. get_restaurant_recommendation: Use this tool to get a restaurant recommendation in a specific location. +""" + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears + # you cannot use the "google_search" tool alongside other tools. + # See https://github.com/googleapis/python-genai/issues/941. + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) + + llm = GeminiLiveVertexLLMService( + credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"), + project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"), + location=os.getenv("GOOGLE_CLOUD_LOCATION"), + system_instruction=system_instruction, + voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck + tools=tools, + ) + + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + context = OpenAILLMContext( + [{"role": "user", "content": "Say hello."}], + ) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py b/examples/foundational/26i-gemini-live-graceful-end.py similarity index 98% rename from examples/foundational/26i-gemini-multimodal-live-graceful-end.py rename to examples/foundational/26i-gemini-live-graceful-end.py index e178997b8..e51bbb032 100644 --- a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py +++ b/examples/foundational/26i-gemini-live-graceful-end.py @@ -4,8 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # - -import asyncio import os from datetime import datetime @@ -24,7 +22,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -144,7 +142,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): custom_tools={AdapterType.GEMINI: [search_tool]}, ) - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, tools=tools, diff --git a/examples/foundational/40-aws-nova-sonic.py b/examples/foundational/40-aws-nova-sonic.py index de7bbf638..d9bd2257d 100644 --- a/examples/foundational/40-aws-nova-sonic.py +++ b/examples/foundational/40-aws-nova-sonic.py @@ -21,7 +21,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.aws_nova_sonic import AWSNovaSonicLLMService +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py index bfb718ff2..62ea5debe 100644 --- a/examples/foundational/46-video-processing.py +++ b/examples/foundational/46-video-processing.py @@ -20,7 +20,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.transports.base_transport import TransportParams from pipecat.transports.daily.transport import DailyParams, DailyTransport @@ -94,7 +94,7 @@ Respond to what the user said in a creative and helpful way. Keep your responses async def run_bot(pipecat_transport): - llm = GeminiMultimodalLiveLLMService( + llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck transcribe_user_audio=True, diff --git a/examples/foundational/47-sentry-metrics.py b/examples/foundational/47-sentry-metrics.py new file mode 100644 index 000000000..ae7b7a59d --- /dev/null +++ b/examples/foundational/47-sentry-metrics.py @@ -0,0 +1,142 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import sentry_sdk +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.metrics.sentry import SentryMetrics +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + # Initialize Sentry + sentry_sdk.init( + dsn=os.getenv("SENTRY_DSN"), + traces_sample_rate=1.0, + ) + + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + metrics=SentryMetrics(), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + metrics=SentryMetrics(), + ) + + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + metrics=SentryMetrics(), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/README.md b/examples/foundational/README.md index 9a6c26005..c1ead3ece 100644 --- a/examples/foundational/README.md +++ b/examples/foundational/README.md @@ -105,7 +105,7 @@ uv run 07-interruptible.py -t twilio -x NGROK_HOST_NAME ### Vision & Multimodal - **[12a-describe-video-gemini-flash.py](./12a-describe-video-gemini-flash.py)**: Bot describes user's video (Video input, Multimodal LLMs) -- **[26c-gemini-multimodal-live-video.py](./26c-gemini-multimodal-live-video.py)**: Gemini with video input (Streaming video, Function calls) +- **[26c-gemini-live-video.py](./26c-gemini-live-video.py)**: Gemini with video input (Streaming video, Function calls) ### Voice & Language diff --git a/examples/foundational/assets/moondream.png b/examples/foundational/assets/moondream.png new file mode 100644 index 000000000..17ba5401f Binary files /dev/null and b/examples/foundational/assets/moondream.png differ diff --git a/pyproject.toml b/pyproject.toml index f71358ff1..c211b8658 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "pyloudnorm~=0.1.1", "resampy~=0.4.3", "soxr~=0.5.0", - "openai>=1.74.0,<=1.99.1", + "openai>=1.74.0,<3", # Pinning numba to resolve package dependencies "numba==0.61.2", "wait_for2>=0.4.1; python_version<'3.12'", @@ -84,7 +84,7 @@ nim = [] neuphonic = [ "pipecat-ai[websockets-base]" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "pipecat-ai[websockets-base]" ] -openpipe = [ "openpipe~=4.50.0" ] +openpipe = [ "openpipe>=4.50.0,<6" ] openrouter = [] perplexity = [] playht = [ "pipecat-ai[websockets-base]" ] diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 206cf848e..14f9dee52 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -136,6 +136,7 @@ TESTS_14 = [ ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14x-function-calling-openpipe.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # Currently not working. # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), @@ -147,7 +148,10 @@ TESTS_15 = [ ] TESTS_19 = [ + ("19-openai-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # OpenAI Realtime not released on Azure yet + # ("19a-azure-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19b-openai-realtime-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("19b-openai-realtime-beta-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), @@ -160,18 +164,18 @@ TESTS_21 = [ TESTS_26 = [ ("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ( - "26a-gemini-multimodal-live-transcription.py", + "26a-gemini-live-transcription.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST, ), ( - "26b-gemini-multimodal-live-function-calling.py", + "26b-gemini-live-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST, ), - ("26c-gemini-multimodal-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("26c-gemini-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ( "26e-gemini-multimodal-google-search.py", PROMPT_ONLINE_SEARCH, @@ -179,7 +183,13 @@ TESTS_26 = [ BOT_SPEAKS_FIRST, ), # Currently not working. - # ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("26d-gemini-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ( + "26h-gemini-live-vertex-function-calling.py", + PROMPT_WEATHER, + EVAL_WEATHER, + BOT_SPEAKS_FIRST, + ), ] TESTS_27 = [ diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 63a86e6d2..abe33a8ec 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -87,9 +87,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Includes both converted standard tools and any custom Gemini-specific tools. """ functions_schema = tools_schema.standard_tools - formatted_standard_tools = [ - {"function_declarations": [func.to_default_dict() for func in functions_schema]} - ] + formatted_standard_tools = ( + [{"function_declarations": [func.to_default_dict() for func in functions_schema]}] + if functions_schema + else [] + ) custom_gemini_tools = [] if tools_schema.custom_tools: custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, []) diff --git a/src/pipecat/audio/filters/krisp_viva_filter.py b/src/pipecat/audio/filters/krisp_viva_filter.py new file mode 100644 index 000000000..ddb489168 --- /dev/null +++ b/src/pipecat/audio/filters/krisp_viva_filter.py @@ -0,0 +1,193 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Krisp noise reduction audio filter for Pipecat. + +This module provides an audio filter implementation using Krisp VIVA SDK. +""" + +import os + +import numpy as np +from loguru import logger + +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame + +try: + import krisp_audio +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the Krisp filter, you need to install krisp_audio.") + raise Exception(f"Missing module: {e}") + + +def _log_callback(log_message, log_level): + logger.info(f"[{log_level}] {log_message}") + + +class KrispVivaFilter(BaseAudioFilter): + """Audio filter using the Krisp VIVA SDK. + + Provides real-time noise reduction for audio streams using Krisp's + proprietary noise suppression algorithms. This filter requires a + valid Krisp model file to operate. + + Supported sample rates: + - 8000 Hz + - 16000 Hz + - 24000 Hz + - 32000 Hz + - 44100 Hz + - 48000 Hz + """ + + # Initialize Krisp Audio SDK globally + krisp_audio.globalInit("", _log_callback, krisp_audio.LogLevel.Off) + SDK_VERSION = krisp_audio.getVersion() + logger.debug( + f"Krisp Audio Python SDK Version: {SDK_VERSION.major}." + f"{SDK_VERSION.minor}.{SDK_VERSION.patch}" + ) + + SAMPLE_RATES = { + 8000: krisp_audio.SamplingRate.Sr8000Hz, + 16000: krisp_audio.SamplingRate.Sr16000Hz, + 24000: krisp_audio.SamplingRate.Sr24000Hz, + 32000: krisp_audio.SamplingRate.Sr32000Hz, + 44100: krisp_audio.SamplingRate.Sr44100Hz, + 48000: krisp_audio.SamplingRate.Sr48000Hz, + } + + FRAME_SIZE_MS = 10 # Krisp requires audio frames of 10ms duration for processing. + + def __init__(self, model_path: str = None, noise_suppression_level: int = 100) -> None: + """Initialize the Krisp noise reduction filter. + + Args: + model_path: Path to the Krisp model file (.kef extension). + If None, uses KRISP_VIVA_MODEL_PATH environment variable. + noise_suppression_level: Noise suppression level. + + Raises: + ValueError: If model_path is not provided and KRISP_VIVA_MODEL_PATH is not set. + Exception: If model file doesn't have .kef extension. + FileNotFoundError: If model file doesn't exist. + """ + super().__init__() + + # Set model path, checking environment if not specified + self._model_path = model_path or os.getenv("KRISP_VIVA_MODEL_PATH") + if not self._model_path: + logger.error("Model path is not provided and KRISP_VIVA_MODEL_PATH is not set.") + raise ValueError("Model path for KrispAudioProcessor must be provided.") + + if not self._model_path.endswith(".kef"): + raise Exception("Model is expected with .kef extension") + + if not os.path.isfile(self._model_path): + raise FileNotFoundError(f"Model file not found: {self._model_path}") + + self._filtering = True + self._session = None + self._samples_per_frame = None + self._noise_suppression_level = noise_suppression_level + + # Audio buffer to accumulate samples for complete frames + self._audio_buffer = bytearray() + + def _int_to_sample_rate(self, sample_rate): + """Convert integer sample rate to krisp_audio SamplingRate enum. + + Args: + sample_rate: Sample rate as integer + + Returns: + krisp_audio.SamplingRate enum value + + Raises: + ValueError: If sample rate is not supported + """ + if sample_rate not in self.SAMPLE_RATES: + raise ValueError("Unsupported sample rate") + return self.SAMPLE_RATES[sample_rate] + + async def start(self, sample_rate: int): + """Initialize the Krisp processor with the transport's sample rate. + + Args: + sample_rate: The sample rate of the input transport in Hz. + """ + model_info = krisp_audio.ModelInfo() + model_info.path = self._model_path + + nc_cfg = krisp_audio.NcSessionConfig() + nc_cfg.inputSampleRate = self._int_to_sample_rate(sample_rate) + nc_cfg.inputFrameDuration = krisp_audio.FrameDuration.Fd10ms + nc_cfg.outputSampleRate = nc_cfg.inputSampleRate + nc_cfg.modelInfo = model_info + + self._samples_per_frame = int((sample_rate * self.FRAME_SIZE_MS) / 1000) + self._session = krisp_audio.NcInt16.create(nc_cfg) + + async def stop(self): + """Clean up the Krisp processor when stopping.""" + self._session = None + + async def process_frame(self, frame: FilterControlFrame): + """Process control frames to enable/disable filtering. + + Args: + frame: The control frame containing filter commands. + """ + if isinstance(frame, FilterEnableFrame): + self._filtering = frame.enable + + async def filter(self, audio: bytes) -> bytes: + """Apply Krisp noise reduction to audio data. + + Args: + audio: Raw audio data as bytes to be filtered. + + Returns: + Noise-reduced audio data as bytes. + """ + if not self._filtering: + return audio + + # Add incoming audio to our buffer + self._audio_buffer.extend(audio) + + # Calculate how many complete frames we can process + total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample + num_complete_frames = total_samples // self._samples_per_frame + + if num_complete_frames == 0: + # Not enough samples for a complete frame yet, return empty + return b"" + + # Calculate how many bytes we need for complete frames + complete_samples_count = num_complete_frames * self._samples_per_frame + bytes_to_process = complete_samples_count * 2 # 2 bytes per sample + + # Extract the bytes we can process + audio_to_process = bytes(self._audio_buffer[:bytes_to_process]) + + # Remove processed bytes from buffer, keep the remainder + self._audio_buffer = self._audio_buffer[bytes_to_process:] + + # Process the complete frames + samples = np.frombuffer(audio_to_process, dtype=np.int16) + frames = samples.reshape(-1, self._samples_per_frame) + processed_samples = np.empty_like(samples) + + for i, frame in enumerate(frames): + cleaned_frame = self._session.process(frame, self._noise_suppression_level) + processed_samples[i * self._samples_per_frame : (i + 1) * self._samples_per_frame] = ( + cleaned_frame + ) + + return processed_samples.tobytes() diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 0b55082aa..1ca3333b5 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -877,6 +877,8 @@ class FrameProcessor(BaseObject): """ while True: + (frame, direction, callback) = await self.__input_queue.get() + if self.__should_block_system_frames and self.__input_event: logger.trace(f"{self}: system frame processing paused") await self.__input_event.wait() @@ -884,8 +886,6 @@ class FrameProcessor(BaseObject): self.__should_block_system_frames = False logger.trace(f"{self}: system frame processing resumed") - (frame, direction, callback) = await self.__input_queue.get() - if isinstance(frame, SystemFrame): await self.__process_frame(frame, direction, callback) elif self.__process_queue: @@ -900,6 +900,8 @@ class FrameProcessor(BaseObject): async def __process_frame_task_handler(self): """Handle non-system frames from the process queue.""" while True: + (frame, direction, callback) = await self.__process_queue.get() + if self.__should_block_frames and self.__process_event: logger.trace(f"{self}: frame processing paused") await self.__process_event.wait() @@ -907,8 +909,6 @@ class FrameProcessor(BaseObject): self.__should_block_frames = False logger.trace(f"{self}: frame processing resumed") - (frame, direction, callback) = await self.__process_queue.get() - await self.__process_frame(frame, direction, callback) self.__process_queue.task_done() diff --git a/src/pipecat/runner/daily.py b/src/pipecat/runner/daily.py index f52ccbc2b..5bff7d060 100644 --- a/src/pipecat/runner/daily.py +++ b/src/pipecat/runner/daily.py @@ -82,6 +82,7 @@ async def configure( sip_enable_video: Optional[bool] = False, sip_num_endpoints: Optional[int] = 1, sip_codecs: Optional[Dict[str, List[str]]] = None, + room_properties: Optional[DailyRoomProperties] = None, ) -> DailyRoomConfig: """Configure Daily room URL and token with optional SIP capabilities. @@ -99,6 +100,10 @@ async def configure( sip_num_endpoints: Number of allowed SIP endpoints. sip_codecs: Codecs to support for audio and video. If None, uses Daily defaults. Example: {"audio": ["OPUS"], "video": ["H264"]} + room_properties: Optional DailyRoomProperties to use instead of building from + individual parameters. When provided, this overrides room_exp_duration and + SIP-related parameters. If not provided, properties are built from the + individual parameters as before. Returns: DailyRoomConfig: Object with room_url, token, and optional sip_endpoint. @@ -115,6 +120,13 @@ async def configure( # SIP-enabled room sip_config = await configure(session, sip_caller_phone="+15551234567") print(f"SIP endpoint: {sip_config.sip_endpoint}") + + # Custom room properties with recording enabled + custom_props = DailyRoomProperties( + enable_recording="cloud", + max_participants=2, + ) + config = await configure(session, room_properties=custom_props) """ # Check for required API key api_key = os.getenv("DAILY_API_KEY") @@ -124,9 +136,32 @@ async def configure( "Get your API key from https://dashboard.daily.co/developers" ) + # Warn if both room_properties and individual parameters are provided + if room_properties is not None: + individual_params_provided = any( + [ + room_exp_duration != 2.0, + token_exp_duration != 2.0, + sip_caller_phone is not None, + sip_enable_video is not False, + sip_num_endpoints != 1, + sip_codecs is not None, + ] + ) + if individual_params_provided: + logger.warning( + "Both room_properties and individual parameters (room_exp_duration, token_exp_duration, " + "sip_*) were provided. The room_properties will be used and individual parameters " + "will be ignored." + ) + # Determine if SIP mode is enabled sip_enabled = sip_caller_phone is not None + # If room_properties is provided, check if it has SIP configuration + if room_properties and room_properties.sip: + sip_enabled = True + daily_rest_helper = DailyRESTHelper( daily_api_key=api_key, daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), @@ -150,27 +185,29 @@ async def configure( room_name = f"{room_prefix}-{uuid.uuid4().hex[:8]}" logger.info(f"Creating new Daily room: {room_name}") - # Calculate expiration time - expiration_time = time.time() + (room_exp_duration * 60 * 60) + # Use provided room_properties or build from parameters + if room_properties is None: + # Calculate expiration time + expiration_time = time.time() + (room_exp_duration * 60 * 60) - # Create room properties - room_properties = DailyRoomProperties( - exp=expiration_time, - eject_at_room_exp=True, - ) - - # Add SIP configuration if enabled - if sip_enabled: - sip_params = DailyRoomSipParams( - display_name=sip_caller_phone, - video=sip_enable_video, - sip_mode="dial-in", - num_endpoints=sip_num_endpoints, - codecs=sip_codecs, + # Create room properties + room_properties = DailyRoomProperties( + exp=expiration_time, + eject_at_room_exp=True, ) - room_properties.sip = sip_params - room_properties.enable_dialout = True # Enable outbound calls if needed - room_properties.start_video_off = not sip_enable_video # Voice-only by default + + # Add SIP configuration if enabled + if sip_enabled: + sip_params = DailyRoomSipParams( + display_name=sip_caller_phone, + video=sip_enable_video, + sip_mode="dial-in", + num_endpoints=sip_num_endpoints, + codecs=sip_codecs, + ) + room_properties.sip = sip_params + room_properties.enable_dialout = True # Enable outbound calls if needed + room_properties.start_video_off = not sip_enable_video # Voice-only by default # Create room parameters room_params = DailyRoomParams(name=room_name, properties=room_properties) diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index 1ab923314..a7af52e18 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -67,12 +67,15 @@ To run locally: import argparse import asyncio +import mimetypes import os import sys from contextlib import asynccontextmanager +from pathlib import Path from typing import Optional import aiohttp +from fastapi.responses import FileResponse from loguru import logger from pipecat.runner.types import ( @@ -98,6 +101,12 @@ except ImportError as e: load_dotenv(override=True) os.environ["ENV"] = "local" +TELEPHONY_TRANSPORTS = ["twilio", "telnyx", "plivo", "exotel"] + +RUNNER_DOWNLOADS_FOLDER: Optional[str] = None +RUNNER_HOST: str = "localhost" +RUNNER_PORT: int = 7860 + def _get_bot_module(): """Get the bot module from the calling script.""" @@ -152,7 +161,12 @@ async def _run_telephony_bot(websocket: WebSocket): def _create_server_app( - transport_type: str, host: str = "localhost", proxy: str = None, esp32_mode: bool = False + *, + transport_type: str, + host: str = "localhost", + proxy: str, + esp32_mode: bool = False, + folder: Optional[str] = None, ): """Create FastAPI app with transport-specific routes.""" app = FastAPI() @@ -167,19 +181,21 @@ def _create_server_app( # Set up transport-specific routes if transport_type == "webrtc": - _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host) + _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host, folder=folder) _setup_whatsapp_routes(app) elif transport_type == "daily": _setup_daily_routes(app) - elif transport_type in ["twilio", "telnyx", "plivo", "exotel"]: - _setup_telephony_routes(app, transport_type, proxy) + elif transport_type in TELEPHONY_TRANSPORTS: + _setup_telephony_routes(app, transport_type=transport_type, proxy=proxy) else: logger.warning(f"Unknown transport type: {transport_type}") return app -def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "localhost"): +def _setup_webrtc_routes( + app: FastAPI, *, esp32_mode: bool = False, host: str = "localhost", folder: Optional[str] = None +): """Set up WebRTC-specific routes.""" try: from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI @@ -201,6 +217,21 @@ def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "lo """Redirect root requests to client interface.""" return RedirectResponse(url="/client/") + @app.get("/files/{filename:path}") + async def download_file(filename: str): + """Handle file downloads.""" + if not folder: + logger.warning(f"Attempting to dowload {filename}, but downloads folder not setup.") + return + + file_path = Path(folder) / filename + if not os.path.exists(file_path): + raise HTTPException(404) + + media_type, _ = mimetypes.guess_type(file_path) + + return FileResponse(path=file_path, media_type=media_type, filename=filename) + # Initialize the SmallWebRTC request handler small_webrtc_handler: SmallWebRTCRequestHandler = SmallWebRTCRequestHandler( esp32_mode=esp32_mode, host=host @@ -284,7 +315,7 @@ def _setup_whatsapp_routes(app: FastAPI): WHATSAPP_WEBHOOK_VERIFICATION_TOKEN, ] ): - logger.debug( + logger.trace( "Missing required environment variables for WhatsApp transport. Keeping it disabled." ) return @@ -489,7 +520,7 @@ def _setup_daily_routes(app: FastAPI): return await _handle_rtvi_request(request) -def _setup_telephony_routes(app: FastAPI, transport_type: str, proxy: str): +def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str): """Set up telephony-specific routes.""" # XML response templates (Exotel doesn't use XML webhooks) XML_TEMPLATES = { @@ -592,6 +623,21 @@ def _validate_and_clean_proxy(proxy: str) -> str: return proxy +def runner_downloads_folder() -> Optional[str]: + """Returns the folder where files are stored for later download.""" + return RUNNER_DOWNLOADS_FOLDER + + +def runner_host() -> str: + """Returns the host name of this runner.""" + return RUNNER_HOST + + +def runner_port() -> int: + """Returns the port of this runner.""" + return RUNNER_PORT + + def main(): """Start the Pipecat development runner. @@ -612,14 +658,16 @@ def main(): The bot file must contain a `bot(runner_args)` function as the entry point. """ + global RUNNER_DOWNLOADS_FOLDER, RUNNER_HOST, RUNNER_PORT + parser = argparse.ArgumentParser(description="Pipecat Development Runner") - parser.add_argument("--host", type=str, default="localhost", help="Host address") - parser.add_argument("--port", type=int, default=7860, help="Port number") + parser.add_argument("--host", type=str, default=RUNNER_HOST, help="Host address") + parser.add_argument("--port", type=int, default=RUNNER_PORT, help="Port number") parser.add_argument( "-t", "--transport", type=str, - choices=["daily", "webrtc", "twilio", "telnyx", "plivo", "exotel"], + choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS], default="webrtc", help="Transport type", ) @@ -637,6 +685,7 @@ def main(): default=False, help="Connect directly to Daily room (automatically sets transport to daily)", ) + parser.add_argument("-f", "--folder", type=str, help="Path to downloads folder") parser.add_argument( "--verbose", "-v", action="count", default=0, help="Increase logging verbosity" ) @@ -659,6 +708,10 @@ def main(): logger.error("For ESP32, you need to specify `--host IP` so we can do SDP munging.") return + if args.transport in TELEPHONY_TRANSPORTS and not args.proxy: + logger.error(f"For telephony transports, you need to specify `--proxy PROXY`.") + return + # Log level logger.remove() logger.add(sys.stderr, level="TRACE" if args.verbose else "DEBUG") @@ -689,8 +742,18 @@ def main(): print(f" → Open http://{args.host}:{args.port} in your browser to start a session") print() + RUNNER_DOWNLOADS_FOLDER = args.folder + RUNNER_HOST = args.host + RUNNER_PORT = args.port + # Create the app with transport-specific setup - app = _create_server_app(args.transport, args.host, args.proxy, args.esp32) + app = _create_server_app( + transport_type=args.transport, + host=args.host, + proxy=args.proxy, + esp32_mode=args.esp32, + folder=args.folder, + ) # Run the server uvicorn.run(app, host=args.host, port=args.port) diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py index 3d4188960..f3a34c434 100644 --- a/src/pipecat/serializers/livekit.py +++ b/src/pipecat/serializers/livekit.py @@ -25,11 +25,31 @@ except ModuleNotFoundError as e: class LivekitFrameSerializer(FrameSerializer): """Serializer for converting between Pipecat frames and LiveKit audio frames. + .. deprecated:: 0.0.90 + + This class is deprecated and will be removed in a future version. + Please use LiveKitTransport instead, which handles audio streaming + and frame conversion natively. + This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects to LiveKit AudioFrame objects for transmission, and the reverse conversion for received audio data. """ + def __init__(self): + """Initialize the LiveKit frame serializer.""" + super().__init__() + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "LivekitFrameSerializer is deprecated and will be removed in a future version. " + "Please use LiveKitTransport instead, which handles audio streaming natively.", + DeprecationWarning, + stacklevel=2, + ) + @property def type(self) -> FrameSerializerType: """Get the serializer type. diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index eebdbfb0b..759efcf00 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -97,9 +97,7 @@ class AIService(FrameProcessor): pass async def _update_settings(self, settings: Mapping[str, Any]): - from pipecat.services.openai_realtime_beta.events import ( - SessionProperties, - ) + from pipecat.services.openai.realtime.events import SessionProperties for key, value in settings.items(): logger.debug("Update request for:", key, value) @@ -111,9 +109,7 @@ class AIService(FrameProcessor): logger.debug("Attempting to update", key, value) try: - from pipecat.services.openai_realtime_beta.events import ( - TurnDetection, - ) + from pipecat.services.openai.realtime.events import TurnDetection if isinstance(self._session_properties, SessionProperties): current_properties = self._session_properties diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index aa2fc36bc..381d60506 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -197,6 +197,8 @@ class AssemblyAISTTService(STTService): ) self._connected = True self._receive_task = self.create_task(self._receive_task_handler()) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"Failed to connect to AssemblyAI: {e}") self._connected = False @@ -238,6 +240,7 @@ class AssemblyAISTTService(STTService): self._websocket = None self._connected = False self._receive_task = None + await self._call_event_handler("on_disconnected") async def _receive_task_handler(self): """Handle incoming WebSocket messages.""" diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index b0ddf9275..3e4ff33cc 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -235,6 +235,8 @@ class AsyncAITTSService(InterruptibleTTSService): } await self._get_websocket().send(json.dumps(init_msg)) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -252,6 +254,7 @@ class AsyncAITTSService(InterruptibleTTSService): finally: self._websocket = None self._started = False + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: diff --git a/src/pipecat/services/aws/__init__.py b/src/pipecat/services/aws/__init__.py index b1f157bd3..3cdd4cc5a 100644 --- a/src/pipecat/services/aws/__init__.py +++ b/src/pipecat/services/aws/__init__.py @@ -9,6 +9,7 @@ import sys from pipecat.services import DeprecatedModuleProxy from .llm import * +from .nova_sonic import * from .stt import * from .tts import * diff --git a/src/pipecat/services/aws/nova_sonic/__init__.py b/src/pipecat/services/aws/nova_sonic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/aws/nova_sonic/context.py b/src/pipecat/services/aws/nova_sonic/context.py new file mode 100644 index 000000000..86aa0f0b5 --- /dev/null +++ b/src/pipecat/services/aws/nova_sonic/context.py @@ -0,0 +1,367 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Context management for AWS Nova Sonic LLM service. + +This module provides specialized context aggregators and message handling for AWS Nova Sonic, +including conversation history management and role-specific message processing. +""" + +import copy +from dataclasses import dataclass, field +from enum import Enum + +from loguru import logger + +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + DataFrame, + Frame, + FunctionCallResultFrame, + InterruptionFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolChoiceFrame, + LLMSetToolsFrame, + TextFrame, + UserImageRawFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) + + +class Role(Enum): + """Roles supported in AWS Nova Sonic conversations. + + Parameters: + SYSTEM: System-level messages (not used in conversation history). + USER: Messages sent by the user. + ASSISTANT: Messages sent by the assistant. + TOOL: Messages sent by tools (not used in conversation history). + """ + + SYSTEM = "SYSTEM" + USER = "USER" + ASSISTANT = "ASSISTANT" + TOOL = "TOOL" + + +@dataclass +class AWSNovaSonicConversationHistoryMessage: + """A single message in AWS Nova Sonic conversation history. + + Parameters: + role: The role of the message sender (USER or ASSISTANT only). + text: The text content of the message. + """ + + role: Role # only USER and ASSISTANT + text: str + + +@dataclass +class AWSNovaSonicConversationHistory: + """Complete conversation history for AWS Nova Sonic initialization. + + Parameters: + system_instruction: System-level instruction for the conversation. + messages: List of conversation messages between user and assistant. + """ + + system_instruction: str = None + messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) + + +class AWSNovaSonicLLMContext(OpenAILLMContext): + """Specialized LLM context for AWS Nova Sonic service. + + Extends OpenAI context with Nova Sonic-specific message handling, + conversation history management, and text buffering capabilities. + """ + + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize AWS Nova Sonic LLM context. + + Args: + messages: Initial messages for the context. + tools: Available tools for the context. + **kwargs: Additional arguments passed to parent class. + """ + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self, system_instruction: str = ""): + self._assistant_text = "" + self._user_text = "" + self._system_instruction = system_instruction + + @staticmethod + def upgrade_to_nova_sonic( + obj: OpenAILLMContext, system_instruction: str + ) -> "AWSNovaSonicLLMContext": + """Upgrade an OpenAI context to AWS Nova Sonic context. + + Args: + obj: The OpenAI context to upgrade. + system_instruction: System instruction for the context. + + Returns: + The upgraded AWS Nova Sonic context. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): + obj.__class__ = AWSNovaSonicLLMContext + obj.__setup_local(system_instruction) + return obj + + # NOTE: this method has the side-effect of updating _system_instruction from messages + def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: + """Get conversation history for initializing AWS Nova Sonic session. + + Processes stored messages and extracts system instruction and conversation + history in the format expected by AWS Nova Sonic. + + Returns: + Formatted conversation history with system instruction and messages. + """ + history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) + + # Bail if there are no messages + if not self.messages: + return history + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into "instruction" + if messages[0].get("role") == "system": + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + history.system_instruction = content + elif isinstance(content, list): + history.system_instruction = content[0].get("text") + if history.system_instruction: + self._system_instruction = history.system_instruction + + # Process remaining messages to fill out conversation history. + # Nova Sonic supports "user" and "assistant" messages in history. + for message in messages: + history_message = self.from_standard_message(message) + if history_message: + history.messages.append(history_message) + + return history + + def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Returns: + List of messages including system instruction if present. + """ + messages = super().get_messages_for_persistent_storage() + # If we have a system instruction and messages doesn't already contain it, add it + if self._system_instruction and not (messages and messages[0].get("role") == "system"): + messages.insert(0, {"role": "system", "content": self._system_instruction}) + return messages + + def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + """Convert standard message format to Nova Sonic format. + + Args: + message: Standard message dictionary to convert. + + Returns: + Nova Sonic conversation history message, or None if not convertible. + """ + role = message.get("role") + if message.get("role") == "user" or message.get("role") == "assistant": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + # There won't be content if this is an assistant tool call entry. + # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation + # history + if content: + return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova + # Sonic conversation history + + def buffer_user_text(self, text): + """Buffer user text for later flushing to context. + + Args: + text: User text to buffer. + """ + self._user_text += f" {text}" if self._user_text else text + # logger.debug(f"User text buffered: {self._user_text}") + + def flush_aggregated_user_text(self) -> str: + """Flush buffered user text to context as a complete message. + + Returns: + The flushed user text, or empty string if no text was buffered. + """ + if not self._user_text: + return "" + user_text = self._user_text + message = { + "role": "user", + "content": [{"type": "text", "text": user_text}], + } + self._user_text = "" + self.add_message(message) + # logger.debug(f"Context updated (user): {self.get_messages_for_logging()}") + return user_text + + def buffer_assistant_text(self, text): + """Buffer assistant text for later flushing to context. + + Args: + text: Assistant text to buffer. + """ + self._assistant_text += text + # logger.debug(f"Assistant text buffered: {self._assistant_text}") + + def flush_aggregated_assistant_text(self): + """Flush buffered assistant text to context as a complete message.""" + if not self._assistant_text: + return + message = { + "role": "assistant", + "content": [{"type": "text", "text": self._assistant_text}], + } + self._assistant_text = "" + self.add_message(message) + # logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}") + + +@dataclass +class AWSNovaSonicMessagesUpdateFrame(DataFrame): + """Frame containing updated AWS Nova Sonic context. + + Parameters: + context: The updated AWS Nova Sonic LLM context. + """ + + context: AWSNovaSonicLLMContext + + +class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): + """Context aggregator for user messages in AWS Nova Sonic conversations. + + Extends the OpenAI user context aggregator to emit Nova Sonic-specific + context update frames. + """ + + async def process_frame( + self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM + ): + """Process frames and emit Nova Sonic-specific context updates. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + await super().process_frame(frame, direction) + + # Parent does not push LLMMessagesUpdateFrame + if isinstance(frame, LLMMessagesUpdateFrame): + await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) + + +class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Context aggregator for assistant messages in AWS Nova Sonic conversations. + + Provides specialized handling for assistant responses and function calls + in AWS Nova Sonic context, with custom frame processing logic. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Nova Sonic-specific logic. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + # HACK: For now, disable the context aggregator by making it just pass through all frames + # that the parent handles (except the function call stuff, which we still need). + # For an explanation of this hack, see + # AWSNovaSonicLLMService._report_assistant_response_text_added. + if isinstance( + frame, + ( + InterruptionFrame, + LLMFullResponseStartFrame, + LLMFullResponseEndFrame, + TextFrame, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMSetToolChoiceFrame, + UserImageRawFrame, + BotStoppedSpeakingFrame, + ), + ): + await self.push_frame(frame, direction) + else: + await super().process_frame(frame, direction) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call results for AWS Nova Sonic. + + Args: + frame: The function call result frame to handle. + """ + await super().handle_function_call_result(frame) + + # The standard function callback code path pushes the FunctionCallResultFrame from the LLM + # itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side + # context. Let's push a special frame to do that. + await self.push_frame( + AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) + + +@dataclass +class AWSNovaSonicContextAggregatorPair: + """Pair of user and assistant context aggregators for AWS Nova Sonic. + + Parameters: + _user: The user context aggregator. + _assistant: The assistant context aggregator. + """ + + _user: AWSNovaSonicUserContextAggregator + _assistant: AWSNovaSonicAssistantContextAggregator + + def user(self) -> AWSNovaSonicUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ + return self._user + + def assistant(self) -> AWSNovaSonicAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ + return self._assistant diff --git a/src/pipecat/services/aws/nova_sonic/frames.py b/src/pipecat/services/aws/nova_sonic/frames.py new file mode 100644 index 000000000..7d4feb2ae --- /dev/null +++ b/src/pipecat/services/aws/nova_sonic/frames.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Custom frames for AWS Nova Sonic LLM service.""" + +from dataclasses import dataclass + +from pipecat.frames.frames import DataFrame, FunctionCallResultFrame + + +@dataclass +class AWSNovaSonicFunctionCallResultFrame(DataFrame): + """Frame containing function call result for AWS Nova Sonic processing. + + This frame wraps a standard function call result frame to enable + AWS Nova Sonic-specific handling and context updates. + + Parameters: + result_frame: The underlying function call result frame. + """ + + result_frame: FunctionCallResultFrame diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py new file mode 100644 index 000000000..2801de688 --- /dev/null +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -0,0 +1,1155 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AWS Nova Sonic LLM service implementation for Pipecat AI framework. + +This module provides a speech-to-speech LLM service using AWS Nova Sonic, which supports +bidirectional audio streaming, text generation, and function calling capabilities. +""" + +import asyncio +import base64 +import json +import time +import uuid +import wave +from dataclasses import dataclass +from enum import Enum +from importlib.resources import files +from typing import Any, List, Optional + +from loguru import logger +from pydantic import BaseModel, Field + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + Frame, + FunctionCallFromLLM, + InputAudioRawFrame, + InterimTranscriptionFrame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + StartFrame, + TranscriptionFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, + TTSTextFrame, +) +from pipecat.processors.aggregators.llm_response import ( + LLMAssistantAggregatorParams, + LLMUserAggregatorParams, +) +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws.nova_sonic.context import ( + AWSNovaSonicAssistantContextAggregator, + AWSNovaSonicContextAggregatorPair, + AWSNovaSonicLLMContext, + AWSNovaSonicUserContextAggregator, + Role, +) +from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame +from pipecat.services.llm_service import LLMService +from pipecat.utils.time import time_now_iso8601 + +try: + from aws_sdk_bedrock_runtime.client import ( + BedrockRuntimeClient, + InvokeModelWithBidirectionalStreamOperationInput, + ) + from aws_sdk_bedrock_runtime.config import Config + from aws_sdk_bedrock_runtime.models import ( + BidirectionalInputPayloadPart, + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamInputChunk, + InvokeModelWithBidirectionalStreamOperationOutput, + InvokeModelWithBidirectionalStreamOutput, + ) + from smithy_aws_core.auth.sigv4 import SigV4AuthScheme + from smithy_aws_core.identity.static import StaticCredentialsResolver + from smithy_core.aio.eventstream import DuplexEventStream +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use AWS services, you need to `pip install pipecat-ai[aws-nova-sonic]`." + ) + raise Exception(f"Missing module: {e}") + + +class AWSNovaSonicUnhandledFunctionException(Exception): + """Exception raised when the LLM attempts to call an unregistered function.""" + + pass + + +class ContentType(Enum): + """Content types supported by AWS Nova Sonic. + + Parameters: + AUDIO: Audio content type. + TEXT: Text content type. + TOOL: Tool content type. + """ + + AUDIO = "AUDIO" + TEXT = "TEXT" + TOOL = "TOOL" + + +class TextStage(Enum): + """Text generation stages in AWS Nova Sonic responses. + + Parameters: + FINAL: Final text that has been fully generated. + SPECULATIVE: Speculative text that is still being generated. + """ + + FINAL = "FINAL" # what has been said + SPECULATIVE = "SPECULATIVE" # what's planned to be said + + +@dataclass +class CurrentContent: + """Represents content currently being received from AWS Nova Sonic. + + Parameters: + type: The type of content (audio, text, or tool). + role: The role generating the content (user, assistant, etc.). + text_stage: The stage of text generation (final or speculative). + text_content: The actual text content if applicable. + """ + + type: ContentType + role: Role + text_stage: TextStage # None if not text + text_content: str # starts as None, then fills in if text + + def __str__(self): + """String representation of the current content.""" + return ( + f"CurrentContent(\n" + f" type={self.type.name},\n" + f" role={self.role.name},\n" + f" text_stage={self.text_stage.name if self.text_stage else 'None'}\n" + f")" + ) + + +class Params(BaseModel): + """Configuration parameters for AWS Nova Sonic. + + Parameters: + input_sample_rate: Audio input sample rate in Hz. + input_sample_size: Audio input sample size in bits. + input_channel_count: Number of input audio channels. + output_sample_rate: Audio output sample rate in Hz. + output_sample_size: Audio output sample size in bits. + output_channel_count: Number of output audio channels. + max_tokens: Maximum number of tokens to generate. + top_p: Nucleus sampling parameter. + temperature: Sampling temperature for text generation. + """ + + # Audio input + input_sample_rate: Optional[int] = Field(default=16000) + input_sample_size: Optional[int] = Field(default=16) + input_channel_count: Optional[int] = Field(default=1) + + # Audio output + output_sample_rate: Optional[int] = Field(default=24000) + output_sample_size: Optional[int] = Field(default=16) + output_channel_count: Optional[int] = Field(default=1) + + # Inference + max_tokens: Optional[int] = Field(default=1024) + top_p: Optional[float] = Field(default=0.9) + temperature: Optional[float] = Field(default=0.7) + + +class AWSNovaSonicLLMService(LLMService): + """AWS Nova Sonic speech-to-speech LLM service. + + Provides bidirectional audio streaming, real-time transcription, text generation, + and function calling capabilities using AWS Nova Sonic model. + """ + + # Override the default adapter to use the AWSNovaSonicLLMAdapter one + adapter_class = AWSNovaSonicLLMAdapter + + def __init__( + self, + *, + secret_access_key: str, + access_key_id: str, + session_token: Optional[str] = None, + region: str, + model: str = "amazon.nova-sonic-v1:0", + voice_id: str = "matthew", # matthew, tiffany, amy + params: Optional[Params] = None, + system_instruction: Optional[str] = None, + tools: Optional[ToolsSchema] = None, + send_transcription_frames: bool = True, + **kwargs, + ): + """Initializes the AWS Nova Sonic LLM service. + + Args: + secret_access_key: AWS secret access key for authentication. + access_key_id: AWS access key ID for authentication. + session_token: AWS session token for authentication. + region: AWS region where the service is hosted. + model: Model identifier. Defaults to "amazon.nova-sonic-v1:0". + voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy. + params: Model parameters for audio configuration and inference. + system_instruction: System-level instruction for the model. + tools: Available tools/functions for the model to use. + send_transcription_frames: Whether to emit transcription frames. + **kwargs: Additional arguments passed to the parent LLMService. + """ + super().__init__(**kwargs) + self._secret_access_key = secret_access_key + self._access_key_id = access_key_id + self._session_token = session_token + self._region = region + self._model = model + self._client: Optional[BedrockRuntimeClient] = None + self._voice_id = voice_id + self._params = params or Params() + self._system_instruction = system_instruction + self._tools = tools + self._send_transcription_frames = send_transcription_frames + self._context: Optional[AWSNovaSonicLLMContext] = None + self._stream: Optional[ + DuplexEventStream[ + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamOutput, + InvokeModelWithBidirectionalStreamOperationOutput, + ] + ] = None + self._receive_task: Optional[asyncio.Task] = None + self._prompt_name: Optional[str] = None + self._input_audio_content_name: Optional[str] = None + self._content_being_received: Optional[CurrentContent] = None + self._assistant_is_responding = False + self._ready_to_send_context = False + self._handling_bot_stopped_speaking = False + self._triggering_assistant_response = False + self._disconnecting = False + self._connected_time: Optional[float] = None + self._wants_connection = False + + file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav") + with wave.open(file_path.open("rb"), "rb") as wav_file: + self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) + + # + # standard AIService frame handling + # + + async def start(self, frame: StartFrame): + """Start the service and initiate connection to AWS Nova Sonic. + + Args: + frame: The start frame triggering service initialization. + """ + await super().start(frame) + self._wants_connection = True + await self._start_connecting() + + async def stop(self, frame: EndFrame): + """Stop the service and close connections. + + Args: + frame: The end frame triggering service shutdown. + """ + await super().stop(frame) + self._wants_connection = False + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the service and close connections. + + Args: + frame: The cancel frame triggering service cancellation. + """ + await super().cancel(frame) + self._wants_connection = False + await self._disconnect() + + # + # conversation resetting + # + + async def reset_conversation(self): + """Reset the conversation state while preserving context. + + Handles bot stopped speaking event, disconnects from the service, + and reconnects with the preserved context. + """ + logger.debug("Resetting conversation") + await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False) + + # Carry over previous context through disconnect + context = self._context + await self._disconnect() + self._context = context + + await self._start_connecting() + + # + # frame processing + # + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle service-specific logic. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, OpenAILLMContextFrame): + await self._handle_context(frame.context) + elif isinstance(frame, LLMContextFrame): + raise NotImplementedError( + "Universal LLMContext is not yet supported for AWS Nova Sonic." + ) + elif isinstance(frame, InputAudioRawFrame): + await self._handle_input_audio_frame(frame) + elif isinstance(frame, BotStoppedSpeakingFrame): + await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=True) + elif isinstance(frame, AWSNovaSonicFunctionCallResultFrame): + await self._handle_function_call_result(frame) + + await self.push_frame(frame, direction) + + async def _handle_context(self, context: OpenAILLMContext): + if not self._context: + # We got our initial context - try to finish connecting + self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic( + context, self._system_instruction + ) + await self._finish_connecting_if_context_available() + + async def _handle_input_audio_frame(self, frame: InputAudioRawFrame): + # Wait until we're done sending the assistant response trigger audio before sending audio + # from the user's mic + if self._triggering_assistant_response: + return + + await self._send_user_audio_event(frame.audio) + + async def _handle_bot_stopped_speaking(self, delay_to_catch_trailing_assistant_text: bool): + # Protect against back-to-back BotStoppedSpeaking calls, which I've observed + if self._handling_bot_stopped_speaking: + return + self._handling_bot_stopped_speaking = True + + async def finalize_assistant_response(): + if self._assistant_is_responding: + # Consider the assistant finished with their response (possibly after a short delay, + # to allow for any trailing FINAL assistant text block to come in that need to make + # it into context). + # + # TODO: ideally we could base this solely on the LLM output events, but I couldn't + # figure out a reliable way to determine when we've gotten our last FINAL text block + # after the LLM is done talking. + # + # First I looked at stopReason, but it doesn't seem like the last FINAL text block + # is reliably marked END_TURN (sometimes the *first* one is, but not the last... + # bug?) + # + # Then I considered schemes where we tally or match up SPECULATIVE text blocks with + # FINAL text blocks to know how many or which FINAL blocks to expect, but user + # interruptions throw a wrench in these schemes: depending on the exact timing of + # the interruption, we should or shouldn't expect some FINAL blocks. + if delay_to_catch_trailing_assistant_text: + # This delay length is a balancing act between "catching" trailing assistant + # text that is quite delayed but not waiting so long that user text comes in + # first and results in a bit of context message order scrambling. + await asyncio.sleep(1.25) + self._assistant_is_responding = False + await self._report_assistant_response_ended() + + self._handling_bot_stopped_speaking = False + + # Finalize the assistant response, either now or after a delay + if delay_to_catch_trailing_assistant_text: + self.create_task(finalize_assistant_response()) + else: + await finalize_assistant_response() + + async def _handle_function_call_result(self, frame: AWSNovaSonicFunctionCallResultFrame): + result = frame.result_frame + await self._send_tool_result(tool_call_id=result.tool_call_id, result=result.result) + + # + # LLM communication: lifecycle + # + + async def _start_connecting(self): + try: + logger.info("Connecting...") + + if self._client: + # Here we assume that if we have a client we are connected or connecting + return + + # Set IDs for the connection + self._prompt_name = str(uuid.uuid4()) + self._input_audio_content_name = str(uuid.uuid4()) + + # Create the client + self._client = self._create_client() + + # Start the bidirectional stream + self._stream = await self._client.invoke_model_with_bidirectional_stream( + InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) + ) + + # Send session start event + await self._send_session_start_event() + + # Finish connecting + self._ready_to_send_context = True + await self._finish_connecting_if_context_available() + except Exception as e: + logger.error(f"{self} initialization error: {e}") + await self._disconnect() + + async def _finish_connecting_if_context_available(self): + # We can only finish connecting once we've gotten our initial context and we're ready to + # send it + if not (self._context and self._ready_to_send_context): + return + + logger.info("Finishing connecting (setting up session)...") + + # Read context + history = self._context.get_messages_for_initializing_history() + + # Send prompt start event, specifying tools. + # Tools from context take priority over self._tools. + tools = ( + self._context.tools + if self._context.tools + else self.get_llm_adapter().from_standard_tools(self._tools) + ) + logger.debug(f"Using tools: {tools}") + await self._send_prompt_start_event(tools) + + # Send system instruction. + # Instruction from context takes priority over self._system_instruction. + # (NOTE: this prioritizing occurred automatically behind the scenes: the context was + # initialized with self._system_instruction and then updated itself from its messages when + # get_messages_for_initializing_history() was called). + logger.debug(f"Using system instruction: {history.system_instruction}") + if history.system_instruction: + await self._send_text_event(text=history.system_instruction, role=Role.SYSTEM) + + # Send conversation history + for message in history.messages: + await self._send_text_event(text=message.text, role=message.role) + + # Start audio input + await self._send_audio_input_start_event() + + # Start receiving events + self._receive_task = self.create_task(self._receive_task_handler()) + + # Record finished connecting time (must be done before sending assistant response trigger) + self._connected_time = time.time() + + logger.info("Finished connecting") + + # If we need to, send assistant response trigger (depends on self._connected_time) + if self._triggering_assistant_response: + await self._send_assistant_response_trigger() + + async def _disconnect(self): + try: + logger.info("Disconnecting...") + + # NOTE: see explanation of HACK, below + self._disconnecting = True + + # Clean up client + if self._client: + await self._send_session_end_events() + self._client = None + + # Clean up stream + if self._stream: + await self._stream.input_stream.close() + self._stream = None + + # NOTE: see explanation of HACK, below + await asyncio.sleep(1) + + # Clean up receive task + # HACK: we should ideally be able to cancel the receive task before stopping the input + # stream, above (meaning we wouldn't need self._disconnecting). But for some reason if + # we don't close the input stream and wait a second first, we're getting an error a lot + # like this one: https://github.com/awslabs/amazon-transcribe-streaming-sdk/issues/61. + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=1.0) + self._receive_task = None + + # Reset remaining connection-specific state + self._prompt_name = None + self._input_audio_content_name = None + self._content_being_received = None + self._assistant_is_responding = False + self._ready_to_send_context = False + self._handling_bot_stopped_speaking = False + self._triggering_assistant_response = False + self._disconnecting = False + self._connected_time = None + + logger.info("Finished disconnecting") + except Exception as e: + logger.error(f"{self} error disconnecting: {e}") + + def _create_client(self) -> BedrockRuntimeClient: + config = Config( + endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", + region=self._region, + aws_access_key_id=self._access_key_id, + aws_secret_access_key=self._secret_access_key, + aws_session_token=self._session_token, + aws_credentials_identity_resolver=StaticCredentialsResolver(), + auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="bedrock")}, + ) + return BedrockRuntimeClient(config=config) + + # + # LLM communication: input events (pipecat -> LLM) + # + + async def _send_session_start_event(self): + session_start = f""" + {{ + "event": {{ + "sessionStart": {{ + "inferenceConfiguration": {{ + "maxTokens": {self._params.max_tokens}, + "topP": {self._params.top_p}, + "temperature": {self._params.temperature} + }} + }} + }} + }} + """ + await self._send_client_event(session_start) + + async def _send_prompt_start_event(self, tools: List[Any]): + if not self._prompt_name: + return + + tools_config = ( + f""", + "toolUseOutputConfiguration": {{ + "mediaType": "application/json" + }}, + "toolConfiguration": {{ + "tools": {json.dumps(tools)} + }} + """ + if tools + else "" + ) + + prompt_start = f''' + {{ + "event": {{ + "promptStart": {{ + "promptName": "{self._prompt_name}", + "textOutputConfiguration": {{ + "mediaType": "text/plain" + }}, + "audioOutputConfiguration": {{ + "mediaType": "audio/lpcm", + "sampleRateHertz": {self._params.output_sample_rate}, + "sampleSizeBits": {self._params.output_sample_size}, + "channelCount": {self._params.output_channel_count}, + "voiceId": "{self._voice_id}", + "encoding": "base64", + "audioType": "SPEECH" + }}{tools_config} + }} + }} + }} + ''' + await self._send_client_event(prompt_start) + + async def _send_audio_input_start_event(self): + if not self._prompt_name: + return + + audio_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{self._input_audio_content_name}", + "type": "AUDIO", + "interactive": true, + "role": "USER", + "audioInputConfiguration": {{ + "mediaType": "audio/lpcm", + "sampleRateHertz": {self._params.input_sample_rate}, + "sampleSizeBits": {self._params.input_sample_size}, + "channelCount": {self._params.input_channel_count}, + "audioType": "SPEECH", + "encoding": "base64" + }} + }} + }} + }} + ''' + await self._send_client_event(audio_content_start) + + async def _send_text_event(self, text: str, role: Role): + if not self._stream or not self._prompt_name or not text: + return + + content_name = str(uuid.uuid4()) + + text_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "type": "TEXT", + "interactive": true, + "role": "{role.value}", + "textInputConfiguration": {{ + "mediaType": "text/plain" + }} + }} + }} + }} + ''' + await self._send_client_event(text_content_start) + + escaped_text = json.dumps(text) # includes quotes + text_input = f''' + {{ + "event": {{ + "textInput": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "content": {escaped_text} + }} + }} + }} + ''' + await self._send_client_event(text_input) + + text_content_end = f''' + {{ + "event": {{ + "contentEnd": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}" + }} + }} + }} + ''' + await self._send_client_event(text_content_end) + + async def _send_user_audio_event(self, audio: bytes): + if not self._stream: + return + + blob = base64.b64encode(audio) + audio_event = f''' + {{ + "event": {{ + "audioInput": {{ + "promptName": "{self._prompt_name}", + "contentName": "{self._input_audio_content_name}", + "content": "{blob.decode("utf-8")}" + }} + }} + }} + ''' + await self._send_client_event(audio_event) + + async def _send_session_end_events(self): + if not self._stream or not self._prompt_name: + return + + prompt_end = f''' + {{ + "event": {{ + "promptEnd": {{ + "promptName": "{self._prompt_name}" + }} + }} + }} + ''' + await self._send_client_event(prompt_end) + + session_end = """ + { + "event": { + "sessionEnd": {} + } + } + """ + await self._send_client_event(session_end) + + async def _send_tool_result(self, tool_call_id, result): + if not self._stream or not self._prompt_name: + return + + content_name = str(uuid.uuid4()) + + result_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "interactive": false, + "type": "TOOL", + "role": "TOOL", + "toolResultInputConfiguration": {{ + "toolUseId": "{tool_call_id}", + "type": "TEXT", + "textInputConfiguration": {{ + "mediaType": "text/plain" + }} + }} + }} + }} + }} + ''' + await self._send_client_event(result_content_start) + + result_content = json.dumps( + { + "event": { + "toolResult": { + "promptName": self._prompt_name, + "contentName": content_name, + "content": json.dumps(result) if isinstance(result, dict) else result, + } + } + } + ) + await self._send_client_event(result_content) + + result_content_end = f""" + {{ + "event": {{ + "contentEnd": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}" + }} + }} + }} + """ + await self._send_client_event(result_content_end) + + async def _send_client_event(self, event_json: str): + if not self._stream: # should never happen + return + + event = InvokeModelWithBidirectionalStreamInputChunk( + value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) + ) + await self._stream.input_stream.send(event) + + # + # LLM communication: output events (LLM -> pipecat) + # + + # Receive events for the session. + # A few different kinds of content can be delivered: + # - Transcription of user audio + # - Tool use + # - Text preview of planned response speech before audio delivered + # - User interruption notification + # - Text of response speech that whose audio was actually delivered + # - Audio of response speech + # Each piece of content is wrapped by "contentStart" and "contentEnd" events. The content is + # delivered sequentially: one piece of content will end before another starts. + # The overall completion is wrapped by "completionStart" and "completionEnd" events. + async def _receive_task_handler(self): + try: + while self._stream and not self._disconnecting: + output = await self._stream.await_output() + result = await output[1].receive() + + if result.value and result.value.bytes_: + response_data = result.value.bytes_.decode("utf-8") + json_data = json.loads(response_data) + + if "event" in json_data: + event_json = json_data["event"] + if "completionStart" in event_json: + # Handle the LLM completion starting + await self._handle_completion_start_event(event_json) + elif "contentStart" in event_json: + # Handle a piece of content starting + await self._handle_content_start_event(event_json) + elif "textOutput" in event_json: + # Handle text output content + await self._handle_text_output_event(event_json) + elif "audioOutput" in event_json: + # Handle audio output content + await self._handle_audio_output_event(event_json) + elif "toolUse" in event_json: + # Handle tool use + await self._handle_tool_use_event(event_json) + elif "contentEnd" in event_json: + # Handle a piece of content ending + await self._handle_content_end_event(event_json) + elif "completionEnd" in event_json: + # Handle the LLM completion ending + await self._handle_completion_end_event(event_json) + except Exception as e: + logger.error(f"{self} error processing responses: {e}") + if self._wants_connection: + await self.reset_conversation() + + async def _handle_completion_start_event(self, event_json): + pass + + async def _handle_content_start_event(self, event_json): + content_start = event_json["contentStart"] + type = content_start["type"] + role = content_start["role"] + generation_stage = None + if "additionalModelFields" in content_start: + additional_model_fields = json.loads(content_start["additionalModelFields"]) + generation_stage = additional_model_fields.get("generationStage") + + # Bookkeeping: track current content being received + content = CurrentContent( + type=ContentType(type), + role=Role(role), + text_stage=TextStage(generation_stage) if generation_stage else None, + text_content=None, + ) + self._content_being_received = content + + if content.role == Role.ASSISTANT: + if content.type == ContentType.AUDIO: + # Note that an assistant response can comprise of multiple audio blocks + if not self._assistant_is_responding: + # The assistant has started responding. + self._assistant_is_responding = True + await self._report_user_transcription_ended() # Consider user turn over + await self._report_assistant_response_started() + + async def _handle_text_output_event(self, event_json): + if not self._content_being_received: # should never happen + return + content = self._content_being_received + + text_content = event_json["textOutput"]["content"] + + # Bookkeeping: augment the current content being received with text + # Assumption: only one text content per content block + content.text_content = text_content + + async def _handle_audio_output_event(self, event_json): + if not self._content_being_received: # should never happen + return + + # Get audio + audio_content = event_json["audioOutput"]["content"] + + # Push audio frame + audio = base64.b64decode(audio_content) + frame = TTSAudioRawFrame( + audio=audio, + sample_rate=self._params.output_sample_rate, + num_channels=self._params.output_channel_count, + ) + await self.push_frame(frame) + + async def _handle_tool_use_event(self, event_json): + if not self._content_being_received or not self._context: # should never happen + return + + # Consider user turn over + await self._report_user_transcription_ended() + + # Get tool use details + tool_use = event_json["toolUse"] + function_name = tool_use["toolName"] + tool_call_id = tool_use["toolUseId"] + arguments = json.loads(tool_use["content"]) + + # Call tool function + if self.has_function(function_name): + if function_name in self._functions.keys() or None in self._functions.keys(): + function_calls_llm = [ + FunctionCallFromLLM( + context=self._context, + tool_call_id=tool_call_id, + function_name=function_name, + arguments=arguments, + ) + ] + + await self.run_function_calls(function_calls_llm) + else: + raise AWSNovaSonicUnhandledFunctionException( + f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function." + ) + + async def _handle_content_end_event(self, event_json): + if not self._content_being_received: # should never happen + return + content = self._content_being_received + + content_end = event_json["contentEnd"] + stop_reason = content_end["stopReason"] + + # Bookkeeping: clear current content being received + self._content_being_received = None + + if content.role == Role.ASSISTANT: + if content.type == ContentType.TEXT: + # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) + if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": + if self._assistant_is_responding: + # Text added to the ongoing assistant response + await self._report_assistant_response_text_added(content.text_content) + elif content.role == Role.USER: + if content.type == ContentType.TEXT: + if content.text_stage == TextStage.FINAL: + # User transcription text added + await self._report_user_transcription_text_added(content.text_content) + + async def _handle_completion_end_event(self, event_json): + pass + + # + # assistant response reporting + # + # 1. Started + # 2. Text added + # 3. Ended + # + + async def _report_assistant_response_started(self): + logger.debug("Assistant response started") + + # Report that the assistant has started their response. + await self.push_frame(LLMFullResponseStartFrame()) + + # Report that equivalent of TTS (this is a speech-to-speech model) started + await self.push_frame(TTSStartedFrame()) + + async def _report_assistant_response_text_added(self, text): + if not self._context: # should never happen + return + + logger.debug(f"Assistant response text added: {text}") + + # Report some text added to the ongoing assistant response + await self.push_frame(LLMTextFrame(text)) + + # Report some text added to the *equivalent* of TTS (this is a speech-to-speech model) + await self.push_frame(TTSTextFrame(text)) + + # TODO: this is a (hopefully temporary) HACK. Here we directly manipulate the context rather + # than relying on the frames pushed to the assistant context aggregator. The pattern of + # receiving full-sentence text after the assistant has spoken does not easily fit with the + # Pipecat expectation of chunks of text streaming in while the assistant is speaking. + # Interruption handling was especially challenging. Rather than spend days trying to fit a + # square peg in a round hole, I decided on this hack for the time being. We can most cleanly + # abandon this hack if/when AWS Nova Sonic implements streaming smaller text chunks + # interspersed with audio. Note that when we move away from this hack, we need to make sure + # that on an interruption we avoid sending LLMFullResponseEndFrame, which gets the + # LLMAssistantContextAggregator into a bad state. + self._context.buffer_assistant_text(text) + + async def _report_assistant_response_ended(self): + if not self._context: # should never happen + return + + logger.debug("Assistant response ended") + + # Report that the assistant has finished their response. + await self.push_frame(LLMFullResponseEndFrame()) + + # Report that equivalent of TTS (this is a speech-to-speech model) stopped. + await self.push_frame(TTSStoppedFrame()) + + # For an explanation of this hack, see _report_assistant_response_text_added. + self._context.flush_aggregated_assistant_text() + + # + # user transcription reporting + # + # 1. Text added + # 2. Ended + # + # Note: "started" does not need to be reported + # + + async def _report_user_transcription_text_added(self, text): + if not self._context: # should never happen + return + + logger.debug(f"User transcription text added: {text}") + + # Manually add new user transcription text to context. + # We can't rely on the user context aggregator to do this since it's upstream from the LLM. + self._context.buffer_user_text(text) + + # Report that some new user transcription text is available. + if self._send_transcription_frames: + await self.push_frame( + InterimTranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) + ) + + async def _report_user_transcription_ended(self): + if not self._context: # should never happen + return + + # Manually add user transcription to context (if any has been buffered). + # We can't rely on the user context aggregator to do this since it's upstream from the LLM. + transcription = self._context.flush_aggregated_user_text() + + if not transcription: + return + + logger.debug(f"User transcription ended") + + if self._send_transcription_frames: + await self.push_frame( + TranscriptionFrame(text=transcription, user_id="", timestamp=time_now_iso8601()) + ) + + # + # context + # + + def create_context_aggregator( + self, + context: OpenAILLMContext, + *, + user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), + assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), + ) -> AWSNovaSonicContextAggregatorPair: + """Create context aggregator pair for managing conversation context. + + Args: + context: The OpenAI LLM context to upgrade. + user_params: Parameters for the user context aggregator. + assistant_params: Parameters for the assistant context aggregator. + + Returns: + A pair of user and assistant context aggregators. + """ + context.set_llm_adapter(self.get_llm_adapter()) + + user = AWSNovaSonicUserContextAggregator(context=context, params=user_params) + assistant = AWSNovaSonicAssistantContextAggregator(context=context, params=assistant_params) + + return AWSNovaSonicContextAggregatorPair(user, assistant) + + # + # assistant response trigger (HACK) + # + + # Class variable + AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION = ( + "Start speaking when you hear the user say 'ready', but don't consider that 'ready' to be " + "a meaningful part of the conversation other than as a trigger for you to start speaking." + ) + + async def trigger_assistant_response(self): + """Trigger an assistant response by sending audio cue. + + Sends a pre-recorded "ready" audio trigger to prompt the assistant + to start speaking. This is useful for controlling conversation flow. + + Returns: + False if already triggering a response, True otherwise. + """ + if self._triggering_assistant_response: + return False + + self._triggering_assistant_response = True + + # Send the trigger audio, if we're fully connected and set up + if self._connected_time: + await self._send_assistant_response_trigger() + + async def _send_assistant_response_trigger(self): + if not self._connected_time: + # should never happen + return + + try: + logger.debug("Sending assistant response trigger...") + + chunk_duration = 0.02 # what we might get from InputAudioRawFrame + chunk_size = int( + chunk_duration + * self._params.input_sample_rate + * self._params.input_channel_count + * (self._params.input_sample_size / 8) + ) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes + + # Lead with a bit of blank audio, if needed. + # It seems like the LLM can't quite "hear" the first little bit of audio sent on a + # connection. + current_time = time.time() + max_blank_audio_duration = 0.5 + blank_audio_duration = ( + max_blank_audio_duration - (current_time - self._connected_time) + if self._connected_time is not None + and (current_time - self._connected_time) < max_blank_audio_duration + else None + ) + if blank_audio_duration: + logger.debug( + f"Leading assistant response trigger with {blank_audio_duration}s of blank audio" + ) + blank_audio_chunk = b"\x00" * chunk_size + num_chunks = int(blank_audio_duration / chunk_duration) + for _ in range(num_chunks): + await self._send_user_audio_event(blank_audio_chunk) + await asyncio.sleep(chunk_duration) + + # Send trigger audio + # NOTE: this audio *will* be transcribed and eventually make it into the context. That's OK: + # if we ever need to seed this service again with context it would make sense to include it + # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the + # context as well. + audio_chunks = [ + self._assistant_response_trigger_audio[i : i + chunk_size] + for i in range(0, len(self._assistant_response_trigger_audio), chunk_size) + ] + for chunk in audio_chunks: + await self._send_user_audio_event(chunk) + await asyncio.sleep(chunk_duration) + finally: + # We need to clean up in case sending the trigger was cancelled, e.g. in the case of a user interruption. + # (An asyncio.CancelledError would be raised in that case.) + self._triggering_assistant_response = False diff --git a/src/pipecat/services/aws_nova_sonic/ready.wav b/src/pipecat/services/aws/nova_sonic/ready.wav similarity index 100% rename from src/pipecat/services/aws_nova_sonic/ready.wav rename to src/pipecat/services/aws/nova_sonic/ready.wav diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index 59cda5865..b019fc058 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -286,6 +286,7 @@ class AWSTranscribeSTTService(STTService): logger.info(f"{self} Successfully connected to AWS Transcribe") + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} Failed to connect to AWS Transcribe: {e}") await self._disconnect() @@ -310,6 +311,7 @@ class AWSTranscribeSTTService(STTService): logger.warning(f"{self} Error closing WebSocket connection: {e}") finally: self._ws_client = None + await self._call_event_handler("on_disconnected") def language_to_service_language(self, language: Language) -> str | None: """Convert internal language enum to AWS Transcribe language code. diff --git a/src/pipecat/services/aws_nova_sonic/__init__.py b/src/pipecat/services/aws_nova_sonic/__init__.py index 4da394cf6..e1cb912b6 100644 --- a/src/pipecat/services/aws_nova_sonic/__init__.py +++ b/src/pipecat/services/aws_nova_sonic/__init__.py @@ -1 +1,19 @@ -from .aws import AWSNovaSonicLLMService, Params +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import warnings + +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService, Params + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic are deprecated. " + "Please use the equivalent types from " + "pipecat.services.aws.nova_sonic.llm instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 66b020055..0524829df 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -10,1146 +10,16 @@ This module provides a speech-to-speech LLM service using AWS Nova Sonic, which bidirectional audio streaming, text generation, and function calling capabilities. """ -import asyncio -import base64 -import json -import time -import uuid -import wave -from dataclasses import dataclass -from enum import Enum -from importlib.resources import files -from typing import Any, List, Optional +import warnings -from loguru import logger -from pydantic import BaseModel, Field +from pipecat.services.aws.nova_sonic.llm import * -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter -from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, - CancelFrame, - EndFrame, - Frame, - FunctionCallFromLLM, - InputAudioRawFrame, - InterimTranscriptionFrame, - LLMContextFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMTextFrame, - StartFrame, - TranscriptionFrame, - TTSAudioRawFrame, - TTSStartedFrame, - TTSStoppedFrame, - TTSTextFrame, -) -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantAggregatorParams, - LLMUserAggregatorParams, -) -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.aws_nova_sonic.context import ( - AWSNovaSonicAssistantContextAggregator, - AWSNovaSonicContextAggregatorPair, - AWSNovaSonicLLMContext, - AWSNovaSonicUserContextAggregator, - Role, -) -from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame -from pipecat.services.llm_service import LLMService -from pipecat.utils.time import time_now_iso8601 - -try: - from aws_sdk_bedrock_runtime.client import ( - BedrockRuntimeClient, - InvokeModelWithBidirectionalStreamOperationInput, +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic.aws are deprecated. " + "Please use the equivalent types from " + "pipecat.services.aws.nova_sonic.llm instead.", + DeprecationWarning, + stacklevel=2, ) - from aws_sdk_bedrock_runtime.config import Config - from aws_sdk_bedrock_runtime.models import ( - BidirectionalInputPayloadPart, - InvokeModelWithBidirectionalStreamInput, - InvokeModelWithBidirectionalStreamInputChunk, - InvokeModelWithBidirectionalStreamOperationOutput, - InvokeModelWithBidirectionalStreamOutput, - ) - from smithy_aws_core.auth.sigv4 import SigV4AuthScheme - from smithy_aws_core.identity.static import StaticCredentialsResolver - from smithy_core.aio.eventstream import DuplexEventStream -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use AWS services, you need to `pip install pipecat-ai[aws-nova-sonic]`." - ) - raise Exception(f"Missing module: {e}") - - -class AWSNovaSonicUnhandledFunctionException(Exception): - """Exception raised when the LLM attempts to call an unregistered function.""" - - pass - - -class ContentType(Enum): - """Content types supported by AWS Nova Sonic. - - Parameters: - AUDIO: Audio content type. - TEXT: Text content type. - TOOL: Tool content type. - """ - - AUDIO = "AUDIO" - TEXT = "TEXT" - TOOL = "TOOL" - - -class TextStage(Enum): - """Text generation stages in AWS Nova Sonic responses. - - Parameters: - FINAL: Final text that has been fully generated. - SPECULATIVE: Speculative text that is still being generated. - """ - - FINAL = "FINAL" # what has been said - SPECULATIVE = "SPECULATIVE" # what's planned to be said - - -@dataclass -class CurrentContent: - """Represents content currently being received from AWS Nova Sonic. - - Parameters: - type: The type of content (audio, text, or tool). - role: The role generating the content (user, assistant, etc.). - text_stage: The stage of text generation (final or speculative). - text_content: The actual text content if applicable. - """ - - type: ContentType - role: Role - text_stage: TextStage # None if not text - text_content: str # starts as None, then fills in if text - - def __str__(self): - """String representation of the current content.""" - return ( - f"CurrentContent(\n" - f" type={self.type.name},\n" - f" role={self.role.name},\n" - f" text_stage={self.text_stage.name if self.text_stage else 'None'}\n" - f")" - ) - - -class Params(BaseModel): - """Configuration parameters for AWS Nova Sonic. - - Parameters: - input_sample_rate: Audio input sample rate in Hz. - input_sample_size: Audio input sample size in bits. - input_channel_count: Number of input audio channels. - output_sample_rate: Audio output sample rate in Hz. - output_sample_size: Audio output sample size in bits. - output_channel_count: Number of output audio channels. - max_tokens: Maximum number of tokens to generate. - top_p: Nucleus sampling parameter. - temperature: Sampling temperature for text generation. - """ - - # Audio input - input_sample_rate: Optional[int] = Field(default=16000) - input_sample_size: Optional[int] = Field(default=16) - input_channel_count: Optional[int] = Field(default=1) - - # Audio output - output_sample_rate: Optional[int] = Field(default=24000) - output_sample_size: Optional[int] = Field(default=16) - output_channel_count: Optional[int] = Field(default=1) - - # Inference - max_tokens: Optional[int] = Field(default=1024) - top_p: Optional[float] = Field(default=0.9) - temperature: Optional[float] = Field(default=0.7) - - -class AWSNovaSonicLLMService(LLMService): - """AWS Nova Sonic speech-to-speech LLM service. - - Provides bidirectional audio streaming, real-time transcription, text generation, - and function calling capabilities using AWS Nova Sonic model. - """ - - # Override the default adapter to use the AWSNovaSonicLLMAdapter one - adapter_class = AWSNovaSonicLLMAdapter - - def __init__( - self, - *, - secret_access_key: str, - access_key_id: str, - session_token: Optional[str] = None, - region: str, - model: str = "amazon.nova-sonic-v1:0", - voice_id: str = "matthew", # matthew, tiffany, amy - params: Optional[Params] = None, - system_instruction: Optional[str] = None, - tools: Optional[ToolsSchema] = None, - send_transcription_frames: bool = True, - **kwargs, - ): - """Initializes the AWS Nova Sonic LLM service. - - Args: - secret_access_key: AWS secret access key for authentication. - access_key_id: AWS access key ID for authentication. - session_token: AWS session token for authentication. - region: AWS region where the service is hosted. - model: Model identifier. Defaults to "amazon.nova-sonic-v1:0". - voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy. - params: Model parameters for audio configuration and inference. - system_instruction: System-level instruction for the model. - tools: Available tools/functions for the model to use. - send_transcription_frames: Whether to emit transcription frames. - **kwargs: Additional arguments passed to the parent LLMService. - """ - super().__init__(**kwargs) - self._secret_access_key = secret_access_key - self._access_key_id = access_key_id - self._session_token = session_token - self._region = region - self._model = model - self._client: Optional[BedrockRuntimeClient] = None - self._voice_id = voice_id - self._params = params or Params() - self._system_instruction = system_instruction - self._tools = tools - self._send_transcription_frames = send_transcription_frames - self._context: Optional[AWSNovaSonicLLMContext] = None - self._stream: Optional[ - DuplexEventStream[ - InvokeModelWithBidirectionalStreamInput, - InvokeModelWithBidirectionalStreamOutput, - InvokeModelWithBidirectionalStreamOperationOutput, - ] - ] = None - self._receive_task: Optional[asyncio.Task] = None - self._prompt_name: Optional[str] = None - self._input_audio_content_name: Optional[str] = None - self._content_being_received: Optional[CurrentContent] = None - self._assistant_is_responding = False - self._ready_to_send_context = False - self._handling_bot_stopped_speaking = False - self._triggering_assistant_response = False - self._disconnecting = False - self._connected_time: Optional[float] = None - self._wants_connection = False - - file_path = files("pipecat.services.aws_nova_sonic").joinpath("ready.wav") - with wave.open(file_path.open("rb"), "rb") as wav_file: - self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) - - # - # standard AIService frame handling - # - - async def start(self, frame: StartFrame): - """Start the service and initiate connection to AWS Nova Sonic. - - Args: - frame: The start frame triggering service initialization. - """ - await super().start(frame) - self._wants_connection = True - await self._start_connecting() - - async def stop(self, frame: EndFrame): - """Stop the service and close connections. - - Args: - frame: The end frame triggering service shutdown. - """ - await super().stop(frame) - self._wants_connection = False - await self._disconnect() - - async def cancel(self, frame: CancelFrame): - """Cancel the service and close connections. - - Args: - frame: The cancel frame triggering service cancellation. - """ - await super().cancel(frame) - self._wants_connection = False - await self._disconnect() - - # - # conversation resetting - # - - async def reset_conversation(self): - """Reset the conversation state while preserving context. - - Handles bot stopped speaking event, disconnects from the service, - and reconnects with the preserved context. - """ - logger.debug("Resetting conversation") - await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False) - - # Carry over previous context through disconnect - context = self._context - await self._disconnect() - self._context = context - - await self._start_connecting() - - # - # frame processing - # - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames and handle service-specific logic. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - await super().process_frame(frame, direction) - - if isinstance(frame, OpenAILLMContextFrame): - await self._handle_context(frame.context) - elif isinstance(frame, LLMContextFrame): - raise NotImplementedError( - "Universal LLMContext is not yet supported for AWS Nova Sonic." - ) - elif isinstance(frame, InputAudioRawFrame): - await self._handle_input_audio_frame(frame) - elif isinstance(frame, BotStoppedSpeakingFrame): - await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=True) - elif isinstance(frame, AWSNovaSonicFunctionCallResultFrame): - await self._handle_function_call_result(frame) - - await self.push_frame(frame, direction) - - async def _handle_context(self, context: OpenAILLMContext): - if not self._context: - # We got our initial context - try to finish connecting - self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic( - context, self._system_instruction - ) - await self._finish_connecting_if_context_available() - - async def _handle_input_audio_frame(self, frame: InputAudioRawFrame): - # Wait until we're done sending the assistant response trigger audio before sending audio - # from the user's mic - if self._triggering_assistant_response: - return - - await self._send_user_audio_event(frame.audio) - - async def _handle_bot_stopped_speaking(self, delay_to_catch_trailing_assistant_text: bool): - # Protect against back-to-back BotStoppedSpeaking calls, which I've observed - if self._handling_bot_stopped_speaking: - return - self._handling_bot_stopped_speaking = True - - async def finalize_assistant_response(): - if self._assistant_is_responding: - # Consider the assistant finished with their response (possibly after a short delay, - # to allow for any trailing FINAL assistant text block to come in that need to make - # it into context). - # - # TODO: ideally we could base this solely on the LLM output events, but I couldn't - # figure out a reliable way to determine when we've gotten our last FINAL text block - # after the LLM is done talking. - # - # First I looked at stopReason, but it doesn't seem like the last FINAL text block - # is reliably marked END_TURN (sometimes the *first* one is, but not the last... - # bug?) - # - # Then I considered schemes where we tally or match up SPECULATIVE text blocks with - # FINAL text blocks to know how many or which FINAL blocks to expect, but user - # interruptions throw a wrench in these schemes: depending on the exact timing of - # the interruption, we should or shouldn't expect some FINAL blocks. - if delay_to_catch_trailing_assistant_text: - # This delay length is a balancing act between "catching" trailing assistant - # text that is quite delayed but not waiting so long that user text comes in - # first and results in a bit of context message order scrambling. - await asyncio.sleep(1.25) - self._assistant_is_responding = False - await self._report_assistant_response_ended() - - self._handling_bot_stopped_speaking = False - - # Finalize the assistant response, either now or after a delay - if delay_to_catch_trailing_assistant_text: - self.create_task(finalize_assistant_response()) - else: - await finalize_assistant_response() - - async def _handle_function_call_result(self, frame: AWSNovaSonicFunctionCallResultFrame): - result = frame.result_frame - await self._send_tool_result(tool_call_id=result.tool_call_id, result=result.result) - - # - # LLM communication: lifecycle - # - - async def _start_connecting(self): - try: - logger.info("Connecting...") - - if self._client: - # Here we assume that if we have a client we are connected or connecting - return - - # Set IDs for the connection - self._prompt_name = str(uuid.uuid4()) - self._input_audio_content_name = str(uuid.uuid4()) - - # Create the client - self._client = self._create_client() - - # Start the bidirectional stream - self._stream = await self._client.invoke_model_with_bidirectional_stream( - InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) - ) - - # Send session start event - await self._send_session_start_event() - - # Finish connecting - self._ready_to_send_context = True - await self._finish_connecting_if_context_available() - except Exception as e: - logger.error(f"{self} initialization error: {e}") - await self._disconnect() - - async def _finish_connecting_if_context_available(self): - # We can only finish connecting once we've gotten our initial context and we're ready to - # send it - if not (self._context and self._ready_to_send_context): - return - - logger.info("Finishing connecting (setting up session)...") - - # Read context - history = self._context.get_messages_for_initializing_history() - - # Send prompt start event, specifying tools. - # Tools from context take priority over self._tools. - tools = ( - self._context.tools - if self._context.tools - else self.get_llm_adapter().from_standard_tools(self._tools) - ) - logger.debug(f"Using tools: {tools}") - await self._send_prompt_start_event(tools) - - # Send system instruction. - # Instruction from context takes priority over self._system_instruction. - # (NOTE: this prioritizing occurred automatically behind the scenes: the context was - # initialized with self._system_instruction and then updated itself from its messages when - # get_messages_for_initializing_history() was called). - logger.debug(f"Using system instruction: {history.system_instruction}") - if history.system_instruction: - await self._send_text_event(text=history.system_instruction, role=Role.SYSTEM) - - # Send conversation history - for message in history.messages: - await self._send_text_event(text=message.text, role=message.role) - - # Start audio input - await self._send_audio_input_start_event() - - # Start receiving events - self._receive_task = self.create_task(self._receive_task_handler()) - - # Record finished connecting time (must be done before sending assistant response trigger) - self._connected_time = time.time() - - logger.info("Finished connecting") - - # If we need to, send assistant response trigger (depends on self._connected_time) - if self._triggering_assistant_response: - await self._send_assistant_response_trigger() - - async def _disconnect(self): - try: - logger.info("Disconnecting...") - - # NOTE: see explanation of HACK, below - self._disconnecting = True - - # Clean up client - if self._client: - await self._send_session_end_events() - self._client = None - - # Clean up stream - if self._stream: - await self._stream.input_stream.close() - self._stream = None - - # NOTE: see explanation of HACK, below - await asyncio.sleep(1) - - # Clean up receive task - # HACK: we should ideally be able to cancel the receive task before stopping the input - # stream, above (meaning we wouldn't need self._disconnecting). But for some reason if - # we don't close the input stream and wait a second first, we're getting an error a lot - # like this one: https://github.com/awslabs/amazon-transcribe-streaming-sdk/issues/61. - if self._receive_task: - await self.cancel_task(self._receive_task, timeout=1.0) - self._receive_task = None - - # Reset remaining connection-specific state - self._prompt_name = None - self._input_audio_content_name = None - self._content_being_received = None - self._assistant_is_responding = False - self._ready_to_send_context = False - self._handling_bot_stopped_speaking = False - self._triggering_assistant_response = False - self._disconnecting = False - self._connected_time = None - - logger.info("Finished disconnecting") - except Exception as e: - logger.error(f"{self} error disconnecting: {e}") - - def _create_client(self) -> BedrockRuntimeClient: - config = Config( - endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", - region=self._region, - aws_access_key_id=self._access_key_id, - aws_secret_access_key=self._secret_access_key, - aws_session_token=self._session_token, - aws_credentials_identity_resolver=StaticCredentialsResolver(), - auth_schemes={"aws.auth#sigv4": SigV4AuthScheme(service="bedrock")}, - ) - return BedrockRuntimeClient(config=config) - - # - # LLM communication: input events (pipecat -> LLM) - # - - async def _send_session_start_event(self): - session_start = f""" - {{ - "event": {{ - "sessionStart": {{ - "inferenceConfiguration": {{ - "maxTokens": {self._params.max_tokens}, - "topP": {self._params.top_p}, - "temperature": {self._params.temperature} - }} - }} - }} - }} - """ - await self._send_client_event(session_start) - - async def _send_prompt_start_event(self, tools: List[Any]): - if not self._prompt_name: - return - - tools_config = ( - f""", - "toolUseOutputConfiguration": {{ - "mediaType": "application/json" - }}, - "toolConfiguration": {{ - "tools": {json.dumps(tools)} - }} - """ - if tools - else "" - ) - - prompt_start = f''' - {{ - "event": {{ - "promptStart": {{ - "promptName": "{self._prompt_name}", - "textOutputConfiguration": {{ - "mediaType": "text/plain" - }}, - "audioOutputConfiguration": {{ - "mediaType": "audio/lpcm", - "sampleRateHertz": {self._params.output_sample_rate}, - "sampleSizeBits": {self._params.output_sample_size}, - "channelCount": {self._params.output_channel_count}, - "voiceId": "{self._voice_id}", - "encoding": "base64", - "audioType": "SPEECH" - }}{tools_config} - }} - }} - }} - ''' - await self._send_client_event(prompt_start) - - async def _send_audio_input_start_event(self): - if not self._prompt_name: - return - - audio_content_start = f''' - {{ - "event": {{ - "contentStart": {{ - "promptName": "{self._prompt_name}", - "contentName": "{self._input_audio_content_name}", - "type": "AUDIO", - "interactive": true, - "role": "USER", - "audioInputConfiguration": {{ - "mediaType": "audio/lpcm", - "sampleRateHertz": {self._params.input_sample_rate}, - "sampleSizeBits": {self._params.input_sample_size}, - "channelCount": {self._params.input_channel_count}, - "audioType": "SPEECH", - "encoding": "base64" - }} - }} - }} - }} - ''' - await self._send_client_event(audio_content_start) - - async def _send_text_event(self, text: str, role: Role): - if not self._stream or not self._prompt_name or not text: - return - - content_name = str(uuid.uuid4()) - - text_content_start = f''' - {{ - "event": {{ - "contentStart": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}", - "type": "TEXT", - "interactive": true, - "role": "{role.value}", - "textInputConfiguration": {{ - "mediaType": "text/plain" - }} - }} - }} - }} - ''' - await self._send_client_event(text_content_start) - - escaped_text = json.dumps(text) # includes quotes - text_input = f''' - {{ - "event": {{ - "textInput": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}", - "content": {escaped_text} - }} - }} - }} - ''' - await self._send_client_event(text_input) - - text_content_end = f''' - {{ - "event": {{ - "contentEnd": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}" - }} - }} - }} - ''' - await self._send_client_event(text_content_end) - - async def _send_user_audio_event(self, audio: bytes): - if not self._stream: - return - - blob = base64.b64encode(audio) - audio_event = f''' - {{ - "event": {{ - "audioInput": {{ - "promptName": "{self._prompt_name}", - "contentName": "{self._input_audio_content_name}", - "content": "{blob.decode("utf-8")}" - }} - }} - }} - ''' - await self._send_client_event(audio_event) - - async def _send_session_end_events(self): - if not self._stream or not self._prompt_name: - return - - prompt_end = f''' - {{ - "event": {{ - "promptEnd": {{ - "promptName": "{self._prompt_name}" - }} - }} - }} - ''' - await self._send_client_event(prompt_end) - - session_end = """ - { - "event": { - "sessionEnd": {} - } - } - """ - await self._send_client_event(session_end) - - async def _send_tool_result(self, tool_call_id, result): - if not self._stream or not self._prompt_name: - return - - content_name = str(uuid.uuid4()) - - result_content_start = f''' - {{ - "event": {{ - "contentStart": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}", - "interactive": false, - "type": "TOOL", - "role": "TOOL", - "toolResultInputConfiguration": {{ - "toolUseId": "{tool_call_id}", - "type": "TEXT", - "textInputConfiguration": {{ - "mediaType": "text/plain" - }} - }} - }} - }} - }} - ''' - await self._send_client_event(result_content_start) - - result_content = json.dumps( - { - "event": { - "toolResult": { - "promptName": self._prompt_name, - "contentName": content_name, - "content": json.dumps(result) if isinstance(result, dict) else result, - } - } - } - ) - await self._send_client_event(result_content) - - result_content_end = f""" - {{ - "event": {{ - "contentEnd": {{ - "promptName": "{self._prompt_name}", - "contentName": "{content_name}" - }} - }} - }} - """ - await self._send_client_event(result_content_end) - - async def _send_client_event(self, event_json: str): - if not self._stream: # should never happen - return - - event = InvokeModelWithBidirectionalStreamInputChunk( - value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) - ) - await self._stream.input_stream.send(event) - - # - # LLM communication: output events (LLM -> pipecat) - # - - # Receive events for the session. - # A few different kinds of content can be delivered: - # - Transcription of user audio - # - Tool use - # - Text preview of planned response speech before audio delivered - # - User interruption notification - # - Text of response speech that whose audio was actually delivered - # - Audio of response speech - # Each piece of content is wrapped by "contentStart" and "contentEnd" events. The content is - # delivered sequentially: one piece of content will end before another starts. - # The overall completion is wrapped by "completionStart" and "completionEnd" events. - async def _receive_task_handler(self): - try: - while self._stream and not self._disconnecting: - output = await self._stream.await_output() - result = await output[1].receive() - - if result.value and result.value.bytes_: - response_data = result.value.bytes_.decode("utf-8") - json_data = json.loads(response_data) - - if "event" in json_data: - event_json = json_data["event"] - if "completionStart" in event_json: - # Handle the LLM completion starting - await self._handle_completion_start_event(event_json) - elif "contentStart" in event_json: - # Handle a piece of content starting - await self._handle_content_start_event(event_json) - elif "textOutput" in event_json: - # Handle text output content - await self._handle_text_output_event(event_json) - elif "audioOutput" in event_json: - # Handle audio output content - await self._handle_audio_output_event(event_json) - elif "toolUse" in event_json: - # Handle tool use - await self._handle_tool_use_event(event_json) - elif "contentEnd" in event_json: - # Handle a piece of content ending - await self._handle_content_end_event(event_json) - elif "completionEnd" in event_json: - # Handle the LLM completion ending - await self._handle_completion_end_event(event_json) - except Exception as e: - logger.error(f"{self} error processing responses: {e}") - if self._wants_connection: - await self.reset_conversation() - - async def _handle_completion_start_event(self, event_json): - pass - - async def _handle_content_start_event(self, event_json): - content_start = event_json["contentStart"] - type = content_start["type"] - role = content_start["role"] - generation_stage = None - if "additionalModelFields" in content_start: - additional_model_fields = json.loads(content_start["additionalModelFields"]) - generation_stage = additional_model_fields.get("generationStage") - - # Bookkeeping: track current content being received - content = CurrentContent( - type=ContentType(type), - role=Role(role), - text_stage=TextStage(generation_stage) if generation_stage else None, - text_content=None, - ) - self._content_being_received = content - - if content.role == Role.ASSISTANT: - if content.type == ContentType.AUDIO: - # Note that an assistant response can comprise of multiple audio blocks - if not self._assistant_is_responding: - # The assistant has started responding. - self._assistant_is_responding = True - await self._report_user_transcription_ended() # Consider user turn over - await self._report_assistant_response_started() - - async def _handle_text_output_event(self, event_json): - if not self._content_being_received: # should never happen - return - content = self._content_being_received - - text_content = event_json["textOutput"]["content"] - - # Bookkeeping: augment the current content being received with text - # Assumption: only one text content per content block - content.text_content = text_content - - async def _handle_audio_output_event(self, event_json): - if not self._content_being_received: # should never happen - return - - # Get audio - audio_content = event_json["audioOutput"]["content"] - - # Push audio frame - audio = base64.b64decode(audio_content) - frame = TTSAudioRawFrame( - audio=audio, - sample_rate=self._params.output_sample_rate, - num_channels=self._params.output_channel_count, - ) - await self.push_frame(frame) - - async def _handle_tool_use_event(self, event_json): - if not self._content_being_received or not self._context: # should never happen - return - - # Consider user turn over - await self._report_user_transcription_ended() - - # Get tool use details - tool_use = event_json["toolUse"] - function_name = tool_use["toolName"] - tool_call_id = tool_use["toolUseId"] - arguments = json.loads(tool_use["content"]) - - # Call tool function - if self.has_function(function_name): - if function_name in self._functions.keys() or None in self._functions.keys(): - function_calls_llm = [ - FunctionCallFromLLM( - context=self._context, - tool_call_id=tool_call_id, - function_name=function_name, - arguments=arguments, - ) - ] - - await self.run_function_calls(function_calls_llm) - else: - raise AWSNovaSonicUnhandledFunctionException( - f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function." - ) - - async def _handle_content_end_event(self, event_json): - if not self._content_being_received: # should never happen - return - content = self._content_being_received - - content_end = event_json["contentEnd"] - stop_reason = content_end["stopReason"] - - # Bookkeeping: clear current content being received - self._content_being_received = None - - if content.role == Role.ASSISTANT: - if content.type == ContentType.TEXT: - # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) - if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": - if self._assistant_is_responding: - # Text added to the ongoing assistant response - await self._report_assistant_response_text_added(content.text_content) - elif content.role == Role.USER: - if content.type == ContentType.TEXT: - if content.text_stage == TextStage.FINAL: - # User transcription text added - await self._report_user_transcription_text_added(content.text_content) - - async def _handle_completion_end_event(self, event_json): - pass - - # - # assistant response reporting - # - # 1. Started - # 2. Text added - # 3. Ended - # - - async def _report_assistant_response_started(self): - logger.debug("Assistant response started") - - # Report that the assistant has started their response. - await self.push_frame(LLMFullResponseStartFrame()) - - # Report that equivalent of TTS (this is a speech-to-speech model) started - await self.push_frame(TTSStartedFrame()) - - async def _report_assistant_response_text_added(self, text): - if not self._context: # should never happen - return - - logger.debug(f"Assistant response text added: {text}") - - # Report some text added to the ongoing assistant response - await self.push_frame(LLMTextFrame(text)) - - # Report some text added to the *equivalent* of TTS (this is a speech-to-speech model) - await self.push_frame(TTSTextFrame(text)) - - # TODO: this is a (hopefully temporary) HACK. Here we directly manipulate the context rather - # than relying on the frames pushed to the assistant context aggregator. The pattern of - # receiving full-sentence text after the assistant has spoken does not easily fit with the - # Pipecat expectation of chunks of text streaming in while the assistant is speaking. - # Interruption handling was especially challenging. Rather than spend days trying to fit a - # square peg in a round hole, I decided on this hack for the time being. We can most cleanly - # abandon this hack if/when AWS Nova Sonic implements streaming smaller text chunks - # interspersed with audio. Note that when we move away from this hack, we need to make sure - # that on an interruption we avoid sending LLMFullResponseEndFrame, which gets the - # LLMAssistantContextAggregator into a bad state. - self._context.buffer_assistant_text(text) - - async def _report_assistant_response_ended(self): - if not self._context: # should never happen - return - - logger.debug("Assistant response ended") - - # Report that the assistant has finished their response. - await self.push_frame(LLMFullResponseEndFrame()) - - # Report that equivalent of TTS (this is a speech-to-speech model) stopped. - await self.push_frame(TTSStoppedFrame()) - - # For an explanation of this hack, see _report_assistant_response_text_added. - self._context.flush_aggregated_assistant_text() - - # - # user transcription reporting - # - # 1. Text added - # 2. Ended - # - # Note: "started" does not need to be reported - # - - async def _report_user_transcription_text_added(self, text): - if not self._context: # should never happen - return - - logger.debug(f"User transcription text added: {text}") - - # Manually add new user transcription text to context. - # We can't rely on the user context aggregator to do this since it's upstream from the LLM. - self._context.buffer_user_text(text) - - # Report that some new user transcription text is available. - if self._send_transcription_frames: - await self.push_frame( - InterimTranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) - ) - - async def _report_user_transcription_ended(self): - if not self._context: # should never happen - return - - # Manually add user transcription to context (if any has been buffered). - # We can't rely on the user context aggregator to do this since it's upstream from the LLM. - transcription = self._context.flush_aggregated_user_text() - - if not transcription: - return - - logger.debug(f"User transcription ended") - - if self._send_transcription_frames: - await self.push_frame( - TranscriptionFrame(text=transcription, user_id="", timestamp=time_now_iso8601()) - ) - - # - # context - # - - def create_context_aggregator( - self, - context: OpenAILLMContext, - *, - user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), - assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), - ) -> AWSNovaSonicContextAggregatorPair: - """Create context aggregator pair for managing conversation context. - - Args: - context: The OpenAI LLM context to upgrade. - user_params: Parameters for the user context aggregator. - assistant_params: Parameters for the assistant context aggregator. - - Returns: - A pair of user and assistant context aggregators. - """ - context.set_llm_adapter(self.get_llm_adapter()) - - user = AWSNovaSonicUserContextAggregator(context=context, params=user_params) - assistant = AWSNovaSonicAssistantContextAggregator(context=context, params=assistant_params) - - return AWSNovaSonicContextAggregatorPair(user, assistant) - - # - # assistant response trigger (HACK) - # - - # Class variable - AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION = ( - "Start speaking when you hear the user say 'ready', but don't consider that 'ready' to be " - "a meaningful part of the conversation other than as a trigger for you to start speaking." - ) - - async def trigger_assistant_response(self): - """Trigger an assistant response by sending audio cue. - - Sends a pre-recorded "ready" audio trigger to prompt the assistant - to start speaking. This is useful for controlling conversation flow. - - Returns: - False if already triggering a response, True otherwise. - """ - if self._triggering_assistant_response: - return False - - self._triggering_assistant_response = True - - # Send the trigger audio, if we're fully connected and set up - if self._connected_time: - await self._send_assistant_response_trigger() - - async def _send_assistant_response_trigger(self): - if not self._connected_time: - # should never happen - return - - try: - logger.debug("Sending assistant response trigger...") - - chunk_duration = 0.02 # what we might get from InputAudioRawFrame - chunk_size = int( - chunk_duration - * self._params.input_sample_rate - * self._params.input_channel_count - * (self._params.input_sample_size / 8) - ) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes - - # Lead with a bit of blank audio, if needed. - # It seems like the LLM can't quite "hear" the first little bit of audio sent on a - # connection. - current_time = time.time() - max_blank_audio_duration = 0.5 - blank_audio_duration = ( - max_blank_audio_duration - (current_time - self._connected_time) - if self._connected_time is not None - and (current_time - self._connected_time) < max_blank_audio_duration - else None - ) - if blank_audio_duration: - logger.debug( - f"Leading assistant response trigger with {blank_audio_duration}s of blank audio" - ) - blank_audio_chunk = b"\x00" * chunk_size - num_chunks = int(blank_audio_duration / chunk_duration) - for _ in range(num_chunks): - await self._send_user_audio_event(blank_audio_chunk) - await asyncio.sleep(chunk_duration) - - # Send trigger audio - # NOTE: this audio *will* be transcribed and eventually make it into the context. That's OK: - # if we ever need to seed this service again with context it would make sense to include it - # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the - # context as well. - audio_chunks = [ - self._assistant_response_trigger_audio[i : i + chunk_size] - for i in range(0, len(self._assistant_response_trigger_audio), chunk_size) - ] - for chunk in audio_chunks: - await self._send_user_audio_event(chunk) - await asyncio.sleep(chunk_duration) - finally: - # We need to clean up in case sending the trigger was cancelled, e.g. in the case of a user interruption. - # (An asyncio.CancelledError would be raised in that case.) - self._triggering_assistant_response = False diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 0ce5ce033..05a24f337 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -10,358 +10,16 @@ This module provides specialized context aggregators and message handling for AW including conversation history management and role-specific message processing. """ -import copy -from dataclasses import dataclass, field -from enum import Enum - -from loguru import logger - -from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, - DataFrame, - Frame, - FunctionCallResultFrame, - InterruptionFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMMessagesAppendFrame, - LLMMessagesUpdateFrame, - LLMSetToolChoiceFrame, - LLMSetToolsFrame, - TextFrame, - UserImageRawFrame, -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) - - -class Role(Enum): - """Roles supported in AWS Nova Sonic conversations. - - Parameters: - SYSTEM: System-level messages (not used in conversation history). - USER: Messages sent by the user. - ASSISTANT: Messages sent by the assistant. - TOOL: Messages sent by tools (not used in conversation history). - """ - - SYSTEM = "SYSTEM" - USER = "USER" - ASSISTANT = "ASSISTANT" - TOOL = "TOOL" - - -@dataclass -class AWSNovaSonicConversationHistoryMessage: - """A single message in AWS Nova Sonic conversation history. - - Parameters: - role: The role of the message sender (USER or ASSISTANT only). - text: The text content of the message. - """ - - role: Role # only USER and ASSISTANT - text: str - - -@dataclass -class AWSNovaSonicConversationHistory: - """Complete conversation history for AWS Nova Sonic initialization. - - Parameters: - system_instruction: System-level instruction for the conversation. - messages: List of conversation messages between user and assistant. - """ - - system_instruction: str = None - messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) - - -class AWSNovaSonicLLMContext(OpenAILLMContext): - """Specialized LLM context for AWS Nova Sonic service. - - Extends OpenAI context with Nova Sonic-specific message handling, - conversation history management, and text buffering capabilities. - """ - - def __init__(self, messages=None, tools=None, **kwargs): - """Initialize AWS Nova Sonic LLM context. - - Args: - messages: Initial messages for the context. - tools: Available tools for the context. - **kwargs: Additional arguments passed to parent class. - """ - super().__init__(messages=messages, tools=tools, **kwargs) - self.__setup_local() - - def __setup_local(self, system_instruction: str = ""): - self._assistant_text = "" - self._user_text = "" - self._system_instruction = system_instruction - - @staticmethod - def upgrade_to_nova_sonic( - obj: OpenAILLMContext, system_instruction: str - ) -> "AWSNovaSonicLLMContext": - """Upgrade an OpenAI context to AWS Nova Sonic context. - - Args: - obj: The OpenAI context to upgrade. - system_instruction: System instruction for the context. - - Returns: - The upgraded AWS Nova Sonic context. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): - obj.__class__ = AWSNovaSonicLLMContext - obj.__setup_local(system_instruction) - return obj - - # NOTE: this method has the side-effect of updating _system_instruction from messages - def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: - """Get conversation history for initializing AWS Nova Sonic session. - - Processes stored messages and extracts system instruction and conversation - history in the format expected by AWS Nova Sonic. - - Returns: - Formatted conversation history with system instruction and messages. - """ - history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) - - # Bail if there are no messages - if not self.messages: - return history - - messages = copy.deepcopy(self.messages) - - # If we have a "system" message as our first message, let's pull that out into "instruction" - if messages[0].get("role") == "system": - system = messages.pop(0) - content = system.get("content") - if isinstance(content, str): - history.system_instruction = content - elif isinstance(content, list): - history.system_instruction = content[0].get("text") - if history.system_instruction: - self._system_instruction = history.system_instruction - - # Process remaining messages to fill out conversation history. - # Nova Sonic supports "user" and "assistant" messages in history. - for message in messages: - history_message = self.from_standard_message(message) - if history_message: - history.messages.append(history_message) - - return history - - def get_messages_for_persistent_storage(self): - """Get messages formatted for persistent storage. - - Returns: - List of messages including system instruction if present. - """ - messages = super().get_messages_for_persistent_storage() - # If we have a system instruction and messages doesn't already contain it, add it - if self._system_instruction and not (messages and messages[0].get("role") == "system"): - messages.insert(0, {"role": "system", "content": self._system_instruction}) - return messages - - def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: - """Convert standard message format to Nova Sonic format. - - Args: - message: Standard message dictionary to convert. - - Returns: - Nova Sonic conversation history message, or None if not convertible. - """ - role = message.get("role") - if message.get("role") == "user" or message.get("role") == "assistant": - content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): - if c.get("type") == "text": - content += " " + c.get("text") - else: - logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" - ) - # There won't be content if this is an assistant tool call entry. - # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation - # history - if content: - return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) - # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova - # Sonic conversation history - - def buffer_user_text(self, text): - """Buffer user text for later flushing to context. - - Args: - text: User text to buffer. - """ - self._user_text += f" {text}" if self._user_text else text - # logger.debug(f"User text buffered: {self._user_text}") - - def flush_aggregated_user_text(self) -> str: - """Flush buffered user text to context as a complete message. - - Returns: - The flushed user text, or empty string if no text was buffered. - """ - if not self._user_text: - return "" - user_text = self._user_text - message = { - "role": "user", - "content": [{"type": "text", "text": user_text}], - } - self._user_text = "" - self.add_message(message) - # logger.debug(f"Context updated (user): {self.get_messages_for_logging()}") - return user_text - - def buffer_assistant_text(self, text): - """Buffer assistant text for later flushing to context. - - Args: - text: Assistant text to buffer. - """ - self._assistant_text += text - # logger.debug(f"Assistant text buffered: {self._assistant_text}") - - def flush_aggregated_assistant_text(self): - """Flush buffered assistant text to context as a complete message.""" - if not self._assistant_text: - return - message = { - "role": "assistant", - "content": [{"type": "text", "text": self._assistant_text}], - } - self._assistant_text = "" - self.add_message(message) - # logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}") - - -@dataclass -class AWSNovaSonicMessagesUpdateFrame(DataFrame): - """Frame containing updated AWS Nova Sonic context. - - Parameters: - context: The updated AWS Nova Sonic LLM context. - """ - - context: AWSNovaSonicLLMContext - - -class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): - """Context aggregator for user messages in AWS Nova Sonic conversations. - - Extends the OpenAI user context aggregator to emit Nova Sonic-specific - context update frames. - """ - - async def process_frame( - self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM - ): - """Process frames and emit Nova Sonic-specific context updates. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - await super().process_frame(frame, direction) - - # Parent does not push LLMMessagesUpdateFrame - if isinstance(frame, LLMMessagesUpdateFrame): - await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) - - -class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Context aggregator for assistant messages in AWS Nova Sonic conversations. - - Provides specialized handling for assistant responses and function calls - in AWS Nova Sonic context, with custom frame processing logic. - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames with Nova Sonic-specific logic. - - Args: - frame: The frame to process. - direction: The direction the frame is traveling. - """ - # HACK: For now, disable the context aggregator by making it just pass through all frames - # that the parent handles (except the function call stuff, which we still need). - # For an explanation of this hack, see - # AWSNovaSonicLLMService._report_assistant_response_text_added. - if isinstance( - frame, - ( - InterruptionFrame, - LLMFullResponseStartFrame, - LLMFullResponseEndFrame, - TextFrame, - LLMMessagesAppendFrame, - LLMMessagesUpdateFrame, - LLMSetToolsFrame, - LLMSetToolChoiceFrame, - UserImageRawFrame, - BotStoppedSpeakingFrame, - ), - ): - await self.push_frame(frame, direction) - else: - await super().process_frame(frame, direction) - - async def handle_function_call_result(self, frame: FunctionCallResultFrame): - """Handle function call results for AWS Nova Sonic. - - Args: - frame: The function call result frame to handle. - """ - await super().handle_function_call_result(frame) - - # The standard function callback code path pushes the FunctionCallResultFrame from the LLM - # itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side - # context. Let's push a special frame to do that. - await self.push_frame( - AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) - - -@dataclass -class AWSNovaSonicContextAggregatorPair: - """Pair of user and assistant context aggregators for AWS Nova Sonic. - - Parameters: - _user: The user context aggregator. - _assistant: The assistant context aggregator. - """ - - _user: AWSNovaSonicUserContextAggregator - _assistant: AWSNovaSonicAssistantContextAggregator - - def user(self) -> AWSNovaSonicUserContextAggregator: - """Get the user context aggregator. - - Returns: - The user context aggregator instance. - """ - return self._user - - def assistant(self) -> AWSNovaSonicAssistantContextAggregator: - """Get the assistant context aggregator. - - Returns: - The assistant context aggregator instance. - """ - return self._assistant +import warnings + +from pipecat.services.aws.nova_sonic.context import * + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic.context are deprecated. " + "Please use the equivalent types from " + "pipecat.services.aws.nova_sonic.context instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/aws_nova_sonic/frames.py b/src/pipecat/services/aws_nova_sonic/frames.py index 7d4feb2ae..def5f26c4 100644 --- a/src/pipecat/services/aws_nova_sonic/frames.py +++ b/src/pipecat/services/aws_nova_sonic/frames.py @@ -6,20 +6,16 @@ """Custom frames for AWS Nova Sonic LLM service.""" -from dataclasses import dataclass +import warnings -from pipecat.frames.frames import DataFrame, FunctionCallResultFrame +from pipecat.services.aws.nova_sonic.frames import * - -@dataclass -class AWSNovaSonicFunctionCallResultFrame(DataFrame): - """Frame containing function call result for AWS Nova Sonic processing. - - This frame wraps a standard function call result frame to enable - AWS Nova Sonic-specific handling and context updates. - - Parameters: - result_frame: The underlying function call result frame. - """ - - result_frame: FunctionCallResultFrame +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.aws_nova_sonic.frames are deprecated. " + "Please use the equivalent types from " + "pipecat.services.aws.nova_sonic.frames instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/azure/realtime/__init__.py b/src/pipecat/services/azure/realtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/azure/realtime/llm.py b/src/pipecat/services/azure/realtime/llm.py new file mode 100644 index 000000000..fb420e10b --- /dev/null +++ b/src/pipecat/services/azure/realtime/llm.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Azure OpenAI Realtime LLM service implementation.""" + +from loguru import logger + +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService + +try: + from websockets.asyncio.client import connect as websocket_connect +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Azure Realtime, you need to `pip install pipecat-ai[openai]`.") + raise Exception(f"Missing module: {e}") + + +class AzureRealtimeLLMService(OpenAIRealtimeLLMService): + """Azure OpenAI Realtime LLM service with Azure-specific authentication. + + Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, + using Azure's authentication headers and endpoint format. Provides the same + real-time audio and text communication capabilities as the base OpenAI service. + """ + + def __init__( + self, + *, + api_key: str, + base_url: str, + **kwargs, + ): + """Initialize Azure Realtime LLM service. + + Args: + api_key: The API key for the Azure OpenAI service. + base_url: The full Azure WebSocket endpoint URL including api-version and deployment. + Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + **kwargs: Additional arguments passed to parent OpenAIRealtimeLLMService. + """ + super().__init__(base_url=base_url, api_key=api_key, **kwargs) + self.api_key = api_key + self.base_url = base_url + + async def _connect(self): + try: + if self._websocket: + # Here we assume that if we have a websocket, we are connected. We + # handle disconnections in the send/recv code paths. + return + + logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") + self._websocket = await websocket_connect( + uri=self.base_url, + additional_headers={ + "api-key": self.api_key, + }, + ) + self._receive_task = self.create_task(self._receive_task_handler()) + except Exception as e: + logger.error(f"{self} initialization error: {e}") + self._websocket = None diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index 5412c422c..b4e232c4a 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -28,13 +28,12 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.stt_service import STTService +from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt try: - import websockets from websockets.asyncio.client import connect as websocket_connect from websockets.protocol import State except ModuleNotFoundError as e: @@ -124,7 +123,7 @@ class CartesiaLiveOptions: return cls(**json.loads(json_str)) -class CartesiaSTTService(STTService): +class CartesiaSTTService(WebsocketSTTService): """Speech-to-text service using Cartesia Live API. Provides real-time speech transcription through WebSocket connection @@ -176,8 +175,7 @@ class CartesiaSTTService(STTService): self.set_model_name(merged_options.model) self._api_key = api_key self._base_url = base_url or "api.cartesia.ai" - self._connection = None - self._receiver_task = None + self._receive_task = None def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -214,6 +212,27 @@ class CartesiaSTTService(STTService): await super().cancel(frame) await self._disconnect() + async def start_metrics(self): + """Start performance metrics collection for transcription processing.""" + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle speech events. + + Args: + frame: The frame to process. + direction: Direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, UserStartedSpeakingFrame): + await self.start_metrics() + elif isinstance(frame, UserStoppedSpeakingFrame): + # Send finalize command to flush the transcription session + if self._websocket and self._websocket.state is State.OPEN: + await self._websocket.send("finalize") + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process audio data for speech-to-text transcription. @@ -224,45 +243,71 @@ class CartesiaSTTService(STTService): None - transcription results are handled via WebSocket responses. """ # If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again - if not self._connection or self._connection.state is State.CLOSED: + if not self._websocket or self._websocket.state is State.CLOSED: await self._connect() - await self._connection.send(audio) + await self._websocket.send(audio) yield None async def _connect(self): - params = self._settings.to_dict() - ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" - logger.debug(f"Connecting to Cartesia: {ws_url}") - headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} + await self._connect_websocket() + if self._websocket and not self._receive_task: + self._receive_task = asyncio.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): try: - self._connection = await websocket_connect(ws_url, additional_headers=headers) - # Setup the receiver task to handle the incoming messages from the Cartesia server - if self._receiver_task is None or self._receiver_task.done(): - self._receiver_task = asyncio.create_task(self._receive_messages()) - logger.debug(f"Connected to Cartesia") + if self._websocket and self._websocket.state is State.OPEN: + return + logger.debug("Connecting to Cartesia STT") + + params = self._settings.to_dict() + ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}" + headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key} + + self._websocket = await websocket_connect(ws_url, additional_headers=headers) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self}: unable to connect to Cartesia: {e}") - async def _receive_messages(self): + async def _disconnect_websocket(self): try: - while True: - if not self._connection or self._connection.state is State.CLOSED: - break - - message = await self._connection.recv() - try: - data = json.loads(message) - await self._process_response(data) - except json.JSONDecodeError: - logger.warning(f"Received non-JSON message: {message}") - except asyncio.CancelledError: - pass - except websockets.exceptions.ConnectionClosed as e: - logger.debug(f"WebSocket connection closed: {e}") + if self._websocket and self._websocket.state is State.OPEN: + logger.debug("Disconnecting from Cartesia STT") + await self._websocket.close() except Exception as e: - logger.error(f"Error in message receiver: {e}") + logger.error(f"{self} error closing websocket: {e}") + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _process_messages(self): + async for message in self._get_websocket(): + try: + data = json.loads(message) + await self._process_response(data) + except json.JSONDecodeError: + logger.warning(f"Received non-JSON message: {message}") + + async def _receive_messages(self): + while True: + await self._process_messages() + # Cartesia times out after 5 minutes of innactivity (no keepalive + # mechanism is available). So, we try to reconnect. + logger.debug(f"{self} Cartesia connection was disconnected (timeout?), reconnecting") + await self._connect_websocket() async def _process_response(self, data): if "type" in data: @@ -316,41 +361,3 @@ class CartesiaSTTService(STTService): language, ) ) - - async def _disconnect(self): - if self._receiver_task: - self._receiver_task.cancel() - try: - await self._receiver_task - except asyncio.CancelledError: - pass - except Exception as e: - logger.exception(f"Unexpected exception while cancelling task: {e}") - self._receiver_task = None - - if self._connection and self._connection.state is State.OPEN: - logger.debug("Disconnecting from Cartesia") - - await self._connection.close() - self._connection = None - - async def start_metrics(self): - """Start performance metrics collection for transcription processing.""" - await self.start_ttfb_metrics() - await self.start_processing_metrics() - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames and handle speech events. - - Args: - frame: The frame to process. - direction: Direction of frame flow in the pipeline. - """ - await super().process_frame(frame, direction) - - if isinstance(frame, UserStartedSpeakingFrame): - await self.start_metrics() - elif isinstance(frame, UserStoppedSpeakingFrame): - # Send finalize command to flush the transcription session - if self._connection and self._connection.state is State.OPEN: - await self._connection.send("finalize") diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 3b81da5d4..90f0ac3b4 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -344,10 +344,11 @@ class CartesiaTTSService(AudioContextWordTTSService): try: if self._websocket and self._websocket.state is State.OPEN: return - logger.debug("Connecting to Cartesia") + logger.debug("Connecting to Cartesia TTS") self._websocket = await websocket_connect( f"{self._url}?api_key={self._api_key}&cartesia_version={self._cartesia_version}" ) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -365,6 +366,7 @@ class CartesiaTTSService(AudioContextWordTTSService): finally: self._context_id = None self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 493bece80..f0b1a5baa 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -205,6 +205,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): additional_headers={"Authorization": f"Token {self._api_key}"}, ) logger.debug("Connected to Deepgram Flux Websocket") + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -225,6 +226,9 @@ class DeepgramFluxSTTService(WebsocketSTTService): await self._websocket.close() except Exception as e: logger.error(f"{self} error closing websocket: {e}") + finally: + self._websocket = None + await self._call_event_handler("on_disconnected") async def _send_close_stream(self) -> None: """Sends a CloseStream control message to the Deepgram Flux WebSocket API. diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 3d6d5bd4c..460b23d18 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -168,16 +168,24 @@ def build_elevenlabs_voice_settings( def calculate_word_times( - alignment_info: Mapping[str, Any], cumulative_time: float -) -> List[Tuple[str, float]]: + alignment_info: Mapping[str, Any], + cumulative_time: float, + partial_word: str = "", + partial_word_start_time: float = 0.0, +) -> tuple[List[Tuple[str, float]], str, float]: """Calculate word timestamps from character alignment information. Args: alignment_info: Character alignment data from ElevenLabs API. cumulative_time: Base time offset for this chunk. + partial_word: Partial word carried over from previous chunk. + partial_word_start_time: Start time of the partial word. Returns: - List of (word, timestamp) tuples. + Tuple of (word_times, new_partial_word, new_partial_word_start_time): + - word_times: List of (word, timestamp) tuples for complete words + - new_partial_word: Incomplete word at end of chunk (empty if chunk ends with space) + - new_partial_word_start_time: Start time of the incomplete word """ chars = alignment_info["chars"] char_start_times_ms = alignment_info["charStartTimesMs"] @@ -186,41 +194,37 @@ def calculate_word_times( logger.error( f"calculate_word_times: length mismatch - chars={len(chars)}, times={len(char_start_times_ms)}" ) - return [] + return ([], partial_word, partial_word_start_time) # Build words and track their start positions words = [] - word_start_indices = [] - current_word = "" - word_start_index = None + word_start_times = [] + current_word = partial_word # Start with any partial word from previous chunk + word_start_time = partial_word_start_time if partial_word else None for i, char in enumerate(chars): if char == " ": # End of current word if current_word: # Only add non-empty words words.append(current_word) - word_start_indices.append(word_start_index) + word_start_times.append(word_start_time) current_word = "" - word_start_index = None + word_start_time = None else: # Building a word - if word_start_index is None: # First character of new word - word_start_index = i + if word_start_time is None: # First character of new word + # Convert from milliseconds to seconds and add cumulative offset + word_start_time = cumulative_time + (char_start_times_ms[i] / 1000.0) current_word += char - # Handle the last word if there's no trailing space - if current_word and word_start_index is not None: - words.append(current_word) - word_start_indices.append(word_start_index) + # Build result for complete words + word_times = list(zip(words, word_start_times)) - # Calculate timestamps for each word - word_times = [] - for word, start_idx in zip(words, word_start_indices): - # Convert from milliseconds to seconds and add cumulative offset - start_time_seconds = cumulative_time + (char_start_times_ms[start_idx] / 1000.0) - word_times.append((word, start_time_seconds)) + # Return any incomplete word at the end of this chunk + new_partial_word = current_word if current_word else "" + new_partial_word_start_time = word_start_time if word_start_time is not None else 0.0 - return word_times + return (word_times, new_partial_word, new_partial_word_start_time) class ElevenLabsTTSService(AudioContextWordTTSService): @@ -332,6 +336,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService): # there's an interruption or TTSStoppedFrame. self._started = False self._cumulative_time = 0 + # Track partial words that span across alignment chunks + self._partial_word = "" + self._partial_word_start_time = 0.0 # Context management for v1 multi API self._context_id = None @@ -521,6 +528,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): url, max_size=16 * 1024 * 1024, additional_headers={"xi-api-key": self._api_key} ) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -543,6 +551,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._started = False self._context_id = None self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: @@ -570,6 +579,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): logger.error(f"Error closing context on interruption: {e}") self._context_id = None self._started = False + self._partial_word = "" + self._partial_word_start_time = 0.0 async def _receive_messages(self): """Handle incoming WebSocket messages from ElevenLabs.""" @@ -609,7 +620,14 @@ class ElevenLabsTTSService(AudioContextWordTTSService): if msg.get("alignment"): alignment = msg["alignment"] - word_times = calculate_word_times(alignment, self._cumulative_time) + word_times, self._partial_word, self._partial_word_start_time = ( + calculate_word_times( + alignment, + self._cumulative_time, + self._partial_word, + self._partial_word_start_time, + ) + ) if word_times: await self.add_word_timestamps(word_times) @@ -683,6 +701,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService): yield TTSStartedFrame() self._started = True self._cumulative_time = 0 + self._partial_word = "" + self._partial_word_start_time = 0.0 # If a context ID does not exist, create a new one and # register it. If an ID exists, that means the Pipeline is # configured for allow_interruptions=False, so continue @@ -756,6 +776,7 @@ class ElevenLabsHttpTTSService(WordTTSService): base_url: str = "https://api.elevenlabs.io", sample_rate: Optional[int] = None, params: Optional[InputParams] = None, + aggregate_sentences: Optional[bool] = True, **kwargs, ): """Initialize the ElevenLabs HTTP TTS service. @@ -768,10 +789,11 @@ class ElevenLabsHttpTTSService(WordTTSService): base_url: Base URL for ElevenLabs HTTP API. sample_rate: Audio sample rate. If None, uses default. params: Additional input parameters for voice customization. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to the parent service. """ super().__init__( - aggregate_sentences=True, + aggregate_sentences=aggregate_sentences, push_text_frames=False, push_stop_frames=True, sample_rate=sample_rate, @@ -809,6 +831,10 @@ class ElevenLabsHttpTTSService(WordTTSService): # Store previous text for context within a turn self._previous_text = "" + # Track partial words that span across alignment chunks + self._partial_word = "" + self._partial_word_start_time = 0.0 + def language_to_service_language(self, language: Language) -> Optional[str]: """Convert pipecat Language to ElevenLabs language code. @@ -836,6 +862,8 @@ class ElevenLabsHttpTTSService(WordTTSService): self._cumulative_time = 0 self._started = False self._previous_text = "" + self._partial_word = "" + self._partial_word_start_time = 0.0 logger.debug(f"{self}: Reset internal state") async def start(self, frame: StartFrame): @@ -870,11 +898,13 @@ class ElevenLabsHttpTTSService(WordTTSService): def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]: """Calculate word timing from character alignment data. + This method handles partial words that may span across multiple alignment chunks. + Args: alignment_info: Character timing data from ElevenLabs. Returns: - List of (word, timestamp) pairs. + List of (word, timestamp) pairs for complete words in this chunk. Example input data:: @@ -900,30 +930,28 @@ class ElevenLabsHttpTTSService(WordTTSService): # Build the words and find their start times words = [] word_start_times = [] - current_word = "" - first_char_idx = -1 + # Start with any partial word from previous chunk + current_word = self._partial_word + word_start_time = self._partial_word_start_time if self._partial_word else None for i, char in enumerate(chars): if char == " ": if current_word: # Only add non-empty words words.append(current_word) - # Use time of the first character of the word, offset by cumulative time - word_start_times.append( - self._cumulative_time + char_start_times[first_char_idx] - ) + word_start_times.append(word_start_time) current_word = "" - first_char_idx = -1 + word_start_time = None else: - if not current_word: # This is the first character of a new word - first_char_idx = i + if word_start_time is None: # First character of a new word + # Use time of the first character of the word, offset by cumulative time + word_start_time = self._cumulative_time + char_start_times[i] current_word += char - # Don't forget the last word if there's no trailing space - if current_word and first_char_idx >= 0: - words.append(current_word) - word_start_times.append(self._cumulative_time + char_start_times[first_char_idx]) + # Store any incomplete word at the end of this chunk + self._partial_word = current_word if current_word else "" + self._partial_word_start_time = word_start_time if word_start_time is not None else 0.0 - # Create word-time pairs + # Create word-time pairs for complete words only word_times = list(zip(words, word_start_times)) return word_times @@ -959,6 +987,9 @@ class ElevenLabsHttpTTSService(WordTTSService): if self._voice_settings: payload["voice_settings"] = self._voice_settings + if self._settings["apply_text_normalization"] is not None: + payload["apply_text_normalization"] = self._settings["apply_text_normalization"] + language = self._settings["language"] if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language: payload["language_code"] = language @@ -979,8 +1010,6 @@ class ElevenLabsHttpTTSService(WordTTSService): } if self._settings["optimize_streaming_latency"] is not None: params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"] - if self._settings["apply_text_normalization"] is not None: - params["apply_text_normalization"] = self._settings["apply_text_normalization"] try: await self.start_ttfb_metrics() @@ -1041,6 +1070,14 @@ class ElevenLabsHttpTTSService(WordTTSService): logger.error(f"Error processing response: {e}", exc_info=True) continue + # After processing all chunks, emit any remaining partial word + # since this is the end of the utterance + if self._partial_word: + final_word_time = [(self._partial_word, self._partial_word_start_time)] + await self.add_word_timestamps(final_word_time) + self._partial_word = "" + self._partial_word_start_time = 0.0 + # After processing all chunks, add the total utterance duration # to the cumulative time to ensure next utterance starts after this one if utterance_duration > 0: diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index b39b775e5..669d2ce97 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -225,6 +225,8 @@ class FishAudioTTSService(InterruptibleTTSService): start_message = {"event": "start", "request": {"text": "", **self._settings}} await self._websocket.send(ormsgpack.packb(start_message)) logger.debug("Sent start message to Fish Audio") + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"Fish Audio initialization error: {e}") self._websocket = None @@ -245,6 +247,7 @@ class FishAudioTTSService(InterruptibleTTSService): self._request_id = None self._started = False self._websocket = None + await self._call_event_handler("on_disconnected") async def flush_audio(self): """Flush any buffered audio by sending a flush event to Fish Audio.""" diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index 9705cf9f2..be69033aa 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -30,12 +30,15 @@ except ModuleNotFoundError as e: # These aliases are just here for backward compatibility, since we used to # define public-facing StartSensitivity and EndSensitivity enums in this # module. -warnings.warn( - "Importing StartSensitivity and EndSensitivity from " - "pipecat.services.gemini_multimodal_live.events is deprecated. " - "Please import them directly from google.genai.types instead.", - DeprecationWarning, - stacklevel=2, -) +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Importing StartSensitivity and EndSensitivity from " + "pipecat.services.gemini_multimodal_live.events is deprecated. " + "Please import them directly from google.genai.types instead.", + DeprecationWarning, + stacklevel=2, + ) + StartSensitivity = _StartSensitivity EndSensitivity = _EndSensitivity diff --git a/src/pipecat/services/gemini_multimodal_live/file_api.py b/src/pipecat/services/gemini_multimodal_live/file_api.py index 5ae7fdbb7..367d6797a 100644 --- a/src/pipecat/services/gemini_multimodal_live/file_api.py +++ b/src/pipecat/services/gemini_multimodal_live/file_api.py @@ -9,181 +9,31 @@ This module provides a client for Google's Gemini File API, enabling file uploads, metadata retrieval, listing, and deletion. Files uploaded through this API can be referenced in Gemini generative model calls. + +.. deprecated:: 0.0.90 + Importing GeminiFileAPI from this module is deprecated. + Import it from pipecat.services.google.gemini_live.file_api instead. """ -import mimetypes -from typing import Any, Dict, Optional +import warnings -import aiohttp from loguru import logger +try: + from pipecat.services.google.gemini_live.file_api import GeminiFileAPI as _GeminiFileAPI +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") + raise Exception(f"Missing module: {e}") -class GeminiFileAPI: - """Client for the Gemini File API. - - This class provides methods for uploading, fetching, listing, and deleting files - through Google's Gemini File API. - - Files uploaded through this API remain available for 48 hours and can be referenced - in calls to the Gemini generative models. Maximum file size is 2GB, with total - project storage limited to 20GB. - """ - - def __init__( - self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files" - ): - """Initialize the Gemini File API client. - - Args: - api_key: Google AI API key - base_url: Base URL for the Gemini File API (default is the v1beta endpoint) - """ - self._api_key = api_key - self._base_url = base_url - # Upload URL uses the /upload/ path - self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files" - - async def upload_file( - self, file_path: str, display_name: Optional[str] = None - ) -> Dict[str, Any]: - """Upload a file to the Gemini File API using the correct resumable upload protocol. - - Args: - file_path: Path to the file to upload - display_name: Optional display name for the file - - Returns: - File metadata including uri, name, and display_name - """ - logger.info(f"Uploading file: {file_path}") - - async with aiohttp.ClientSession() as session: - # Determine the file's MIME type - mime_type, _ = mimetypes.guess_type(file_path) - if not mime_type: - mime_type = "application/octet-stream" - - # Read the file - with open(file_path, "rb") as f: - file_data = f.read() - - # Create the metadata payload - metadata = {} - if display_name: - metadata = {"file": {"display_name": display_name}} - - # Step 1: Initial resumable request to get upload URL - headers = { - "X-Goog-Upload-Protocol": "resumable", - "X-Goog-Upload-Command": "start", - "X-Goog-Upload-Header-Content-Length": str(len(file_data)), - "X-Goog-Upload-Header-Content-Type": mime_type, - "Content-Type": "application/json", - } - - logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}") - async with session.post( - f"{self.upload_base_url}?key={self._api_key}", headers=headers, json=metadata - ) as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error initiating file upload: {error_text}") - raise Exception(f"Failed to initiate upload: {response.status} - {error_text}") - - # Get the upload URL from the response header - upload_url = response.headers.get("X-Goog-Upload-URL") - if not upload_url: - logger.error(f"Response headers: {dict(response.headers)}") - raise Exception("No upload URL in response headers") - - logger.debug(f"Got upload URL: {upload_url}") - - # Step 2: Upload the actual file data - upload_headers = { - "Content-Length": str(len(file_data)), - "X-Goog-Upload-Offset": "0", - "X-Goog-Upload-Command": "upload, finalize", - } - - logger.debug(f"Step 2: Uploading file data to {upload_url}") - async with session.post(upload_url, headers=upload_headers, data=file_data) as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error uploading file data: {error_text}") - raise Exception(f"Failed to upload file: {response.status} - {error_text}") - - file_info = await response.json() - logger.info(f"File uploaded successfully: {file_info.get('file', {}).get('name')}") - return file_info - - async def get_file(self, name: str) -> Dict[str, Any]: - """Get metadata for a file. - - Args: - name: File name (or full path) - - Returns: - File metadata - """ - # Extract just the name part if a full path is provided - if "/" in name: - name = name.split("/")[-1] - - async with aiohttp.ClientSession() as session: - async with session.get(f"{self._base_url}/{name}?key={self._api_key}") as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error getting file metadata: {error_text}") - raise Exception(f"Failed to get file metadata: {response.status}") - - file_info = await response.json() - return file_info - - async def list_files( - self, page_size: int = 10, page_token: Optional[str] = None - ) -> Dict[str, Any]: - """List uploaded files. - - Args: - page_size: Number of files to return per page - page_token: Token for pagination - - Returns: - List of files and next page token if available - """ - params = {"key": self._api_key, "pageSize": page_size} - - if page_token: - params["pageToken"] = page_token - - async with aiohttp.ClientSession() as session: - async with session.get(self._base_url, params=params) as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error listing files: {error_text}") - raise Exception(f"Failed to list files: {response.status}") - - result = await response.json() - return result - - async def delete_file(self, name: str) -> bool: - """Delete a file. - - Args: - name: File name (or full path) - - Returns: - True if deleted successfully - """ - # Extract just the name part if a full path is provided - if "/" in name: - name = name.split("/")[-1] - - async with aiohttp.ClientSession() as session: - async with session.delete(f"{self._base_url}/{name}?key={self._api_key}") as response: - if response.status != 200: - error_text = await response.text() - logger.error(f"Error deleting file: {error_text}") - raise Exception(f"Failed to delete file: {response.status}") - - return True +# These aliases are just here for backward compatibility, since we used to +# define public-facing StartSensitivity and EndSensitivity enums in this +# module. +warnings.warn( + "Importing GeminiFileAPI from " + "pipecat.services.gemini_multimodal_live.file_api is deprecated. " + "Please import it from pipecat.services.google.gemini_live.file_api instead.", + DeprecationWarning, + stacklevel=2, +) +GeminiFileAPI = _GeminiFileAPI diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 65b5eb781..e56e34e9d 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -4,1552 +4,54 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Google Gemini Multimodal Live API service implementation. +"""Google Gemini Live API service implementation. This module provides real-time conversational AI capabilities using Google's -Gemini Multimodal Live API, supporting both text and audio modalities with +Gemini Live API, supporting both text and audio modalities with voice transcription, streaming responses, and tool usage. + +.. deprecated:: 0.0.90 + This module is deprecated. Please use the equivalent types from + pipecat.services.google.gemini_live.llm instead. Note that the new type names + do not include 'Multimodal'. """ -import base64 -import io -import json -import random -import time -from dataclasses import dataclass -from enum import Enum -from typing import Any, Dict, List, Optional, Union +import warnings -from loguru import logger -from PIL import Image -from pydantic import BaseModel, Field +from pipecat.services.google.gemini_live.llm import ( + ContextWindowCompressionParams as _ContextWindowCompressionParams, +) +from pipecat.services.google.gemini_live.llm import ( + GeminiLiveAssistantContextAggregator, + GeminiLiveContext, + GeminiLiveContextAggregatorPair, + GeminiLiveLLMService, + GeminiLiveUserContextAggregator, + GeminiModalities, +) +from pipecat.services.google.gemini_live.llm import GeminiMediaResolution as _GeminiMediaResolution +from pipecat.services.google.gemini_live.llm import GeminiVADParams as _GeminiVADParams +from pipecat.services.google.gemini_live.llm import InputParams as _InputParams -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter -from pipecat.frames.frames import ( - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - CancelFrame, - EndFrame, - ErrorFrame, - Frame, - InputAudioRawFrame, - InputImageRawFrame, - InputTextRawFrame, - InterruptionFrame, - LLMContextFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMMessagesAppendFrame, - LLMSetToolsFrame, - LLMTextFrame, - LLMUpdateSettingsFrame, - StartFrame, - TranscriptionFrame, - TTSAudioRawFrame, - TTSStartedFrame, - TTSStoppedFrame, - TTSTextFrame, - UserImageRawFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, -) -from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantAggregatorParams, - LLMUserAggregatorParams, -) -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult -from pipecat.services.llm_service import FunctionCallFromLLM, LLMService -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) -from pipecat.transcriptions.language import Language -from pipecat.utils.string import match_endofsentence -from pipecat.utils.time import time_now_iso8601 -from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt - -from .file_api import GeminiFileAPI - -try: - from google.genai import Client - from google.genai.live import AsyncSession - from google.genai.types import ( - AudioTranscriptionConfig, - AutomaticActivityDetection, - Blob, - Content, - ContextWindowCompressionConfig, - EndSensitivity, - FileData, - FunctionResponse, - GenerationConfig, - GroundingMetadata, - HttpOptions, - LiveConnectConfig, - LiveServerMessage, - MediaResolution, - Modality, - Part, - ProactivityConfig, - RealtimeInputConfig, - SessionResumptionConfig, - SlidingWindow, - SpeechConfig, - StartSensitivity, - ThinkingConfig, - VoiceConfig, +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.gemini_multimodal_live.gemini are deprecated. " + "Please use the equivalent types from " + "pipecat.services.google.gemini_live.llm instead. Note that the new type " + "names do not include 'Multimodal' " + "(e.g. `GeminiMultimodalLiveLLMService` is now `GeminiLiveLLMService`).", + DeprecationWarning, + stacklevel=2, ) -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") - raise Exception(f"Missing module: {e}") - -# Connection management constants -MAX_CONSECUTIVE_FAILURES = 3 -CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds - - -def language_to_gemini_language(language: Language) -> Optional[str]: - """Maps a Language enum value to a Gemini Live supported language code. - - Source: - https://ai.google.dev/api/generate-content#MediaResolution - - Args: - language: The language enum value to convert. - - Returns: - The Gemini language code string, or None if the language is not supported. - """ - language_map = { - # Arabic - Language.AR: "ar-XA", - # Bengali - Language.BN_IN: "bn-IN", - # Chinese (Mandarin) - Language.CMN: "cmn-CN", - Language.CMN_CN: "cmn-CN", - Language.ZH: "cmn-CN", # Map general Chinese to Mandarin for Gemini - Language.ZH_CN: "cmn-CN", # Map Simplified Chinese to Mandarin for Gemini - # German - Language.DE: "de-DE", - Language.DE_DE: "de-DE", - # English - Language.EN: "en-US", # Default to US English (though not explicitly listed in supported codes) - Language.EN_US: "en-US", - Language.EN_AU: "en-AU", - Language.EN_GB: "en-GB", - Language.EN_IN: "en-IN", - # Spanish - Language.ES: "es-ES", # Default to Spain Spanish - Language.ES_ES: "es-ES", - Language.ES_US: "es-US", - # French - Language.FR: "fr-FR", # Default to France French - Language.FR_FR: "fr-FR", - Language.FR_CA: "fr-CA", - # Gujarati - Language.GU: "gu-IN", - Language.GU_IN: "gu-IN", - # Hindi - Language.HI: "hi-IN", - Language.HI_IN: "hi-IN", - # Indonesian - Language.ID: "id-ID", - Language.ID_ID: "id-ID", - # Italian - Language.IT: "it-IT", - Language.IT_IT: "it-IT", - # Japanese - Language.JA: "ja-JP", - Language.JA_JP: "ja-JP", - # Kannada - Language.KN: "kn-IN", - Language.KN_IN: "kn-IN", - # Korean - Language.KO: "ko-KR", - Language.KO_KR: "ko-KR", - # Malayalam - Language.ML: "ml-IN", - Language.ML_IN: "ml-IN", - # Marathi - Language.MR: "mr-IN", - Language.MR_IN: "mr-IN", - # Dutch - Language.NL: "nl-NL", - Language.NL_NL: "nl-NL", - # Polish - Language.PL: "pl-PL", - Language.PL_PL: "pl-PL", - # Portuguese (Brazil) - Language.PT_BR: "pt-BR", - # Russian - Language.RU: "ru-RU", - Language.RU_RU: "ru-RU", - # Tamil - Language.TA: "ta-IN", - Language.TA_IN: "ta-IN", - # Telugu - Language.TE: "te-IN", - Language.TE_IN: "te-IN", - # Thai - Language.TH: "th-TH", - Language.TH_TH: "th-TH", - # Turkish - Language.TR: "tr-TR", - Language.TR_TR: "tr-TR", - # Vietnamese - Language.VI: "vi-VN", - Language.VI_VN: "vi-VN", - } - return language_map.get(language) - - -class GeminiMultimodalLiveContext(OpenAILLMContext): - """Extended OpenAI context for Gemini Multimodal Live API. - - Provides Gemini-specific context management including system instruction - extraction and message format conversion for the Live API. - """ - - @staticmethod - def upgrade(obj: OpenAILLMContext) -> "GeminiMultimodalLiveContext": - """Upgrade an OpenAI context to Gemini context. - - Args: - obj: The OpenAI context to upgrade. - - Returns: - The upgraded Gemini context instance. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiMultimodalLiveContext): - logger.debug(f"Upgrading to Gemini Multimodal Live Context: {obj}") - obj.__class__ = GeminiMultimodalLiveContext - obj._restructure_from_openai_messages() - return obj - - def _restructure_from_openai_messages(self): - pass - - def extract_system_instructions(self): - """Extract system instructions from context messages. - - Returns: - Combined system instruction text from all system messages. - """ - system_instruction = "" - for item in self.messages: - if item.get("role") == "system": - content = item.get("content", "") - if content: - if system_instruction and not system_instruction.endswith("\n"): - system_instruction += "\n" - system_instruction += str(content) - return system_instruction - - def add_file_reference(self, file_uri: str, mime_type: str, text: Optional[str] = None): - """Add a file reference to the context. - - This adds a user message with a file reference that will be sent during context initialization. - - Args: - file_uri: URI of the uploaded file - mime_type: MIME type of the file - text: Optional text prompt to accompany the file - """ - # Create parts list with file reference - parts = [] - if text: - parts.append({"type": "text", "text": text}) - - # Add file reference part - parts.append( - {"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}} - ) - - # Add to messages - message = {"role": "user", "content": parts} - self.messages.append(message) - logger.info(f"Added file reference to context: {file_uri}") - - def get_messages_for_initializing_history(self) -> List[Content]: - """Get messages formatted for Gemini history initialization. - - Returns: - List of messages in Gemini format for conversation history. - """ - messages: List[Content] = [] - for item in self.messages: - role = item.get("role") - - if role == "system": - continue - - elif role == "assistant": - role = "model" - - content = item.get("content") - parts: List[Part] = [] - if isinstance(content, str): - parts = [Part(text=content)] - elif isinstance(content, list): - for part in content: - if part.get("type") == "text": - parts.append(Part(text=part.get("text"))) - elif part.get("type") == "file_data": - file_data = part.get("file_data", {}) - parts.append( - Part( - file_data=FileData( - mime_type=file_data.get("mime_type"), - file_uri=file_data.get("file_uri"), - ) - ) - ) - else: - logger.warning(f"Unsupported content type: {str(part)[:80]}") - else: - logger.warning(f"Unsupported content type: {str(content)[:80]}") - messages.append(Content(role=role, parts=parts)) - return messages - - -class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): - """User context aggregator for Gemini Multimodal Live. - - Extends OpenAI user aggregator to handle Gemini-specific message passing - while maintaining compatibility with the standard aggregation pipeline. - """ - - async def process_frame(self, frame, direction): - """Process incoming frames for user context aggregation. - - Args: - frame: The frame to process. - direction: The frame processing direction. - """ - await super().process_frame(frame, direction) - # kind of a hack just to pass the LLMMessagesAppendFrame through, but it's fine for now - if isinstance(frame, LLMMessagesAppendFrame): - await self.push_frame(frame, direction) - - -class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Assistant context aggregator for Gemini Multimodal Live. - - Handles assistant response aggregation while filtering out LLMTextFrames - to prevent duplicate context entries, as Gemini Live pushes both - LLMTextFrames and TTSTextFrames. - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames for assistant context aggregation. - - Args: - frame: The frame to process. - direction: The frame processing direction. - """ - # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, - # but the GeminiMultimodalLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We - # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames - # are process. This ensures that the context gets only one set of messages. - if not isinstance(frame, LLMTextFrame): - await super().process_frame(frame, direction) - - async def handle_user_image_frame(self, frame: UserImageRawFrame): - """Handle user image frames. - - Args: - frame: The user image frame to handle. - """ - # We don't want to store any images in the context. Revisit this later - # when the API evolves. - pass - - -@dataclass -class GeminiMultimodalLiveContextAggregatorPair: - """Pair of user and assistant context aggregators for Gemini Multimodal Live. - - Parameters: - _user: The user context aggregator instance. - _assistant: The assistant context aggregator instance. - """ - - _user: GeminiMultimodalLiveUserContextAggregator - _assistant: GeminiMultimodalLiveAssistantContextAggregator - - def user(self) -> GeminiMultimodalLiveUserContextAggregator: - """Get the user context aggregator. - - Returns: - The user context aggregator instance. - """ - return self._user - - def assistant(self) -> GeminiMultimodalLiveAssistantContextAggregator: - """Get the assistant context aggregator. - - Returns: - The assistant context aggregator instance. - """ - return self._assistant - - -class GeminiMultimodalModalities(Enum): - """Supported modalities for Gemini Multimodal Live. - - Parameters: - TEXT: Text responses. - AUDIO: Audio responses. - """ - - TEXT = "TEXT" - AUDIO = "AUDIO" - - -class GeminiMediaResolution(str, Enum): - """Media resolution options for Gemini Multimodal Live. - - Parameters: - UNSPECIFIED: Use default resolution setting. - LOW: Low resolution with 64 tokens. - MEDIUM: Medium resolution with 256 tokens. - HIGH: High resolution with zoomed reframing and 256 tokens. - """ - - UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default - LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens - MEDIUM = "MEDIA_RESOLUTION_MEDIUM" # 256 tokens - HIGH = "MEDIA_RESOLUTION_HIGH" # Zoomed reframing with 256 tokens - - -class GeminiVADParams(BaseModel): - """Voice Activity Detection parameters for Gemini Live. - - Parameters: - disabled: Whether to disable VAD. Defaults to None. - start_sensitivity: Sensitivity for speech start detection. Defaults to None. - end_sensitivity: Sensitivity for speech end detection. Defaults to None. - prefix_padding_ms: Prefix padding in milliseconds. Defaults to None. - silence_duration_ms: Silence duration threshold in milliseconds. Defaults to None. - """ - - disabled: Optional[bool] = Field(default=None) - start_sensitivity: Optional[StartSensitivity] = Field(default=None) - end_sensitivity: Optional[EndSensitivity] = Field(default=None) - prefix_padding_ms: Optional[int] = Field(default=None) - silence_duration_ms: Optional[int] = Field(default=None) - - -class ContextWindowCompressionParams(BaseModel): - """Parameters for context window compression in Gemini Live. - - Parameters: - enabled: Whether compression is enabled. Defaults to False. - trigger_tokens: Token count to trigger compression. None uses 80% of context window. - """ - - enabled: bool = Field(default=False) - trigger_tokens: Optional[int] = Field( - default=None - ) # None = use default (80% of context window) - - -class InputParams(BaseModel): - """Input parameters for Gemini Multimodal Live generation. - - Parameters: - frequency_penalty: Frequency penalty for generation (0.0-2.0). Defaults to None. - max_tokens: Maximum tokens to generate. Must be >= 1. Defaults to 4096. - presence_penalty: Presence penalty for generation (0.0-2.0). Defaults to None. - temperature: Sampling temperature (0.0-2.0). Defaults to None. - top_k: Top-k sampling parameter. Must be >= 0. Defaults to None. - top_p: Top-p sampling parameter (0.0-1.0). Defaults to None. - modalities: Response modalities. Defaults to AUDIO. - language: Language for generation. Defaults to EN_US. - media_resolution: Media resolution setting. Defaults to UNSPECIFIED. - vad: Voice activity detection parameters. Defaults to None. - context_window_compression: Context compression settings. Defaults to None. - thinking: Thinking settings. Defaults to None. - Note that these settings may require specifying a model that - supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". - enable_affective_dialog: Enable affective dialog, which allows Gemini - to adapt to expression and tone. Defaults to None. - Note that these settings may require specifying a model that - supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". - Also note that this setting may require specifying an API version that - supports it, e.g. HttpOptions(api_version="v1alpha"). - proactivity: Proactivity settings, which allows Gemini to proactively - decide how to behave, such as whether to avoid responding to - content that is not relevant. Defaults to None. - Note that these settings may require specifying a model that - supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". - Also note that this setting may require specifying an API version that - supports it, e.g. HttpOptions(api_version="v1alpha"). - extra: Additional parameters. Defaults to empty dict. - """ - - frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) - max_tokens: Optional[int] = Field(default=4096, ge=1) - presence_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) - temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) - top_k: Optional[int] = Field(default=None, ge=0) - top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) - modalities: Optional[GeminiMultimodalModalities] = Field( - default=GeminiMultimodalModalities.AUDIO - ) - language: Optional[Language] = Field(default=Language.EN_US) - media_resolution: Optional[GeminiMediaResolution] = Field( - default=GeminiMediaResolution.UNSPECIFIED - ) - vad: Optional[GeminiVADParams] = Field(default=None) - context_window_compression: Optional[ContextWindowCompressionParams] = Field(default=None) - thinking: Optional[ThinkingConfig] = Field(default=None) - enable_affective_dialog: Optional[bool] = Field(default=None) - proactivity: Optional[ProactivityConfig] = Field(default=None) - extra: Optional[Dict[str, Any]] = Field(default_factory=dict) - - -class GeminiMultimodalLiveLLMService(LLMService): - """Provides access to Google's Gemini Multimodal Live API. - - This service enables real-time conversations with Gemini, supporting both - text and audio modalities. It handles voice transcription, streaming audio - responses, and tool usage. - """ - - # Overriding the default adapter to use the Gemini one. - adapter_class = GeminiLLMAdapter - - def __init__( - self, - *, - api_key: str, - base_url: Optional[str] = None, - model="models/gemini-2.0-flash-live-001", - voice_id: str = "Charon", - start_audio_paused: bool = False, - start_video_paused: bool = False, - system_instruction: Optional[str] = None, - tools: Optional[Union[List[dict], ToolsSchema]] = None, - params: Optional[InputParams] = None, - inference_on_context_initialization: bool = True, - file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files", - http_options: Optional[HttpOptions] = None, - **kwargs, - ): - """Initialize the Gemini Multimodal Live LLM service. - - Args: - api_key: Google AI API key for authentication. - base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint. - - .. deprecated:: 0.0.90 - This parameter is deprecated and no longer has any effect. - Please use `http_options` to customize requests made by the - API client. - - model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001". - voice_id: TTS voice identifier. Defaults to "Charon". - start_audio_paused: Whether to start with audio input paused. Defaults to False. - start_video_paused: Whether to start with video input paused. Defaults to False. - system_instruction: System prompt for the model. Defaults to None. - tools: Tools/functions available to the model. Defaults to None. - params: Configuration parameters for the model. Defaults to InputParams(). - inference_on_context_initialization: Whether to generate a response when context - is first set. Defaults to True. - file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint. - http_options: HTTP options for the client. - **kwargs: Additional arguments passed to parent LLMService. - """ - # Check for deprecated parameter usage - if base_url is not None: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Parameter 'base_url' is deprecated and no longer has any effect. Please use 'http_options' to customize requests made by the API client.", - DeprecationWarning, - stacklevel=2, - ) - - super().__init__(base_url=base_url, **kwargs) - - params = params or InputParams() - - self._last_sent_time = 0 - self._base_url = base_url - self.set_model_name(model) - self._voice_id = voice_id - self._language_code = params.language - - self._system_instruction = system_instruction - self._tools = tools - self._inference_on_context_initialization = inference_on_context_initialization - self._needs_turn_complete_message = False - - self._audio_input_paused = start_audio_paused - self._video_input_paused = start_video_paused - self._context = None - self._create_client(api_key, http_options) - self._session: AsyncSession = None - self._connection_task = None - - self._disconnecting = False - self._run_llm_when_session_ready = False - - self._user_is_speaking = False - self._bot_is_speaking = False - self._user_audio_buffer = bytearray() - self._user_transcription_buffer = "" - self._last_transcription_sent = "" - self._bot_audio_buffer = bytearray() - self._bot_text_buffer = "" - self._llm_output_buffer = "" - - self._sample_rate = 24000 - - self._language = params.language - self._language_code = ( - language_to_gemini_language(params.language) if params.language else "en-US" - ) - self._vad_params = params.vad - - # Reconnection tracking - self._consecutive_failures = 0 - self._connection_start_time = None - - self._settings = { - "frequency_penalty": params.frequency_penalty, - "max_tokens": params.max_tokens, - "presence_penalty": params.presence_penalty, - "temperature": params.temperature, - "top_k": params.top_k, - "top_p": params.top_p, - "modalities": params.modalities, - "language": self._language_code, - "media_resolution": params.media_resolution, - "vad": params.vad, - "context_window_compression": params.context_window_compression.model_dump() - if params.context_window_compression - else {}, - "thinking": params.thinking or {}, - "enable_affective_dialog": params.enable_affective_dialog or False, - "proactivity": params.proactivity or {}, - "extra": params.extra if isinstance(params.extra, dict) else {}, - } - - # Initialize the File API client - self.file_api = GeminiFileAPI(api_key=api_key, base_url=file_api_base_url) - - # Grounding metadata tracking - self._search_result_buffer = "" - self._accumulated_grounding_metadata = None - - # Session resumption - self._session_resumption_handle: Optional[str] = None - - # Bookkeeping for ending gracefully (i.e. after the bot is finished) - self._end_frame_pending_bot_turn_finished: Optional[EndFrame] = None - - def _create_client(self, api_key: str, http_options: Optional[HttpOptions] = None): - self._client = Client(api_key=api_key, http_options=http_options) - - def can_generate_metrics(self) -> bool: - """Check if the service can generate usage metrics. - - Returns: - True as Gemini Live supports token usage metrics. - """ - return True - - def needs_mcp_alternate_schema(self) -> bool: - """Check if this LLM service requires alternate MCP schema. - - Google/Gemini has stricter JSON schema validation and requires - certain properties to be removed or modified for compatibility. - - Returns: - True for Google/Gemini services. - """ - return True - - def set_audio_input_paused(self, paused: bool): - """Set the audio input pause state. - - Args: - paused: Whether to pause audio input. - """ - self._audio_input_paused = paused - - def set_video_input_paused(self, paused: bool): - """Set the video input pause state. - - Args: - paused: Whether to pause video input. - """ - self._video_input_paused = paused - - def set_model_modalities(self, modalities: GeminiMultimodalModalities): - """Set the model response modalities. - - Args: - modalities: The modalities to use for responses. - """ - self._settings["modalities"] = modalities - - def set_language(self, language: Language): - """Set the language for generation. - - Args: - language: The language to use for generation. - """ - self._language = language - self._language_code = language_to_gemini_language(language) or "en-US" - self._settings["language"] = self._language_code - logger.info(f"Set Gemini language to: {self._language_code}") - - async def set_context(self, context: OpenAILLMContext): - """Set the context explicitly from outside the pipeline. - - This is useful when initializing a conversation because in server-side VAD mode we might not have a - way to trigger the pipeline. This sends the history to the server. The `inference_on_context_initialization` - flag controls whether to set the turnComplete flag when we do this. Without that flag, the model will - not respond. This is often what we want when setting the context at the beginning of a conversation. - - Args: - context: The OpenAI LLM context to set. - """ - if self._context: - logger.error( - "Context already set. Can only set up Gemini Multimodal Live context once." - ) - return - self._context = GeminiMultimodalLiveContext.upgrade(context) - await self._create_initial_response() - - # - # standard AIService frame handling - # - - async def start(self, frame: StartFrame): - """Start the service and establish connection. - - Args: - frame: The start frame. - """ - await super().start(frame) - await self._connect() - - async def stop(self, frame: EndFrame): - """Stop the service and close connections. - - Args: - frame: The end frame. - """ - await super().stop(frame) - await self._disconnect() - - async def cancel(self, frame: CancelFrame): - """Cancel the service and close connections. - - Args: - frame: The cancel frame. - """ - await super().cancel(frame) - await self._disconnect() - - # - # speech and interruption handling - # - - async def _handle_interruption(self): - await self._set_bot_is_speaking(False) - await self.push_frame(TTSStoppedFrame()) - await self.push_frame(LLMFullResponseEndFrame()) - - async def _handle_user_started_speaking(self, frame): - self._user_is_speaking = True - pass - - async def _handle_user_stopped_speaking(self, frame): - self._user_is_speaking = False - self._user_audio_buffer = bytearray() - await self.start_ttfb_metrics() - if self._needs_turn_complete_message: - self._needs_turn_complete_message = False - # NOTE: without this, the model ignores the context it's been - # seeded with before the user started speaking - await self._session.send_client_content(turn_complete=True) - - # - # frame processing - # - # StartFrame, StopFrame, CancelFrame implemented in base class - # - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames for the Gemini Live service. - - Args: - frame: The frame to process. - direction: The frame processing direction. - """ - # Defer EndFrame handling until after the bot turn is finished - if isinstance(frame, EndFrame): - if self._bot_is_speaking: - logger.debug("Deferring handling EndFrame until bot turn is finished") - self._end_frame_pending_bot_turn_finished = frame - return - - await super().process_frame(frame, direction) - - if isinstance(frame, TranscriptionFrame): - await self.push_frame(frame, direction) - elif isinstance(frame, OpenAILLMContextFrame): - context: GeminiMultimodalLiveContext = GeminiMultimodalLiveContext.upgrade( - frame.context - ) - # For now, we'll only trigger inference here when either: - # 1. We have not seen a context frame before - # 2. The last message is a tool call result - if not self._context: - self._context = context - if frame.context.tools: - self._tools = frame.context.tools - await self._create_initial_response() - elif context.messages and context.messages[-1].get("role") == "tool": - # Support just one tool call per context frame for now - tool_result_message = context.messages[-1] - await self._tool_result(tool_result_message) - elif isinstance(frame, LLMContextFrame): - raise NotImplementedError( - "Universal LLMContext is not yet supported for Gemini Multimodal Live." - ) - elif isinstance(frame, InputTextRawFrame): - await self._send_user_text(frame.text) - await self.push_frame(frame, direction) - elif isinstance(frame, InputAudioRawFrame): - await self._send_user_audio(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, InputImageRawFrame): - await self._send_user_video(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, InterruptionFrame): - await self._handle_interruption() - await self.push_frame(frame, direction) - elif isinstance(frame, UserStartedSpeakingFrame): - await self._handle_user_started_speaking(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, UserStoppedSpeakingFrame): - await self._handle_user_stopped_speaking(frame) - await self.push_frame(frame, direction) - elif isinstance(frame, BotStartedSpeakingFrame): - # Ignore this frame. Use the serverContent API message instead - await self.push_frame(frame, direction) - elif isinstance(frame, BotStoppedSpeakingFrame): - # ignore this frame. Use the serverContent.turnComplete API message - await self.push_frame(frame, direction) - elif isinstance(frame, LLMMessagesAppendFrame): - # NOTE: handling LLMMessagesAppendFrame here in the LLMService is - # unusual - typically this would be handled in the user context - # aggregator. Leaving this handling here so that user code that - # uses this frame *without* a user context aggregator still works - # (we have an example that does just that, actually). - await self._create_single_response(frame.messages) - elif isinstance(frame, LLMUpdateSettingsFrame): - await self._update_settings(frame.settings) - elif isinstance(frame, LLMSetToolsFrame): - await self._update_settings() - else: - await self.push_frame(frame, direction) - - async def _set_bot_is_speaking(self, speaking: bool): - if self._bot_is_speaking == speaking: - return - - self._bot_is_speaking = speaking - - if not self._bot_is_speaking and self._end_frame_pending_bot_turn_finished: - await self.queue_frame(self._end_frame_pending_bot_turn_finished) - self._end_frame_pending_bot_turn_finished = None - - async def _connect(self, session_resumption_handle: Optional[str] = None): - """Establish client connection to Gemini Live API.""" - if self._session: - # Here we assume that if we have a client, we are connected. We - # handle disconnections in the send/recv code paths. - return - - if session_resumption_handle: - logger.info( - f"Connecting to Gemini service with session_resumption_handle: {session_resumption_handle}" - ) - else: - logger.info("Connecting to Gemini service") - try: - # Assemble basic configuration - config = LiveConnectConfig( - generation_config=GenerationConfig( - frequency_penalty=self._settings["frequency_penalty"], - max_output_tokens=self._settings["max_tokens"], - presence_penalty=self._settings["presence_penalty"], - temperature=self._settings["temperature"], - top_k=self._settings["top_k"], - top_p=self._settings["top_p"], - response_modalities=[Modality(self._settings["modalities"].value)], - speech_config=SpeechConfig( - voice_config=VoiceConfig( - prebuilt_voice_config={"voice_name": self._voice_id} - ), - language_code=self._settings["language"], - ), - media_resolution=MediaResolution(self._settings["media_resolution"].value), - ), - input_audio_transcription=AudioTranscriptionConfig(), - output_audio_transcription=AudioTranscriptionConfig(), - session_resumption=SessionResumptionConfig(handle=session_resumption_handle), - ) - - # Add context window compression to configuration, if enabled - if self._settings.get("context_window_compression", {}).get("enabled", False): - compression_config = ContextWindowCompressionConfig() - - # Add sliding window (always true if compression is enabled) - compression_config.sliding_window = SlidingWindow() - - # Add trigger_tokens if specified - trigger_tokens = self._settings.get("context_window_compression", {}).get( - "trigger_tokens" - ) - if trigger_tokens is not None: - compression_config.trigger_tokens = trigger_tokens - - config.context_window_compression = compression_config - - # Add thinking configuration to configuration, if provided - if self._settings.get("thinking"): - config.thinking_config = self._settings["thinking"] - - # Add affective dialog setting, if provided - if self._settings.get("enable_affective_dialog", False): - config.enable_affective_dialog = self._settings["enable_affective_dialog"] - - # Add proactivity configuration to configuration, if provided - if self._settings.get("proactivity"): - config.proactivity = self._settings["proactivity"] - - # Add VAD configuration to configuration, if provided - if self._settings.get("vad"): - vad_config = AutomaticActivityDetection() - vad_params = self._settings["vad"] - has_vad_settings = False - - # Only add parameters that are explicitly set - if vad_params.disabled is not None: - vad_config.disabled = vad_params.disabled - has_vad_settings = True - - if vad_params.start_sensitivity: - vad_config.start_of_speech_sensitivity = vad_params.start_sensitivity - has_vad_settings = True - - if vad_params.end_sensitivity: - vad_config.end_of_speech_sensitivity = vad_params.end_sensitivity - has_vad_settings = True - - if vad_params.prefix_padding_ms is not None: - vad_config.prefix_padding_ms = vad_params.prefix_padding_ms - has_vad_settings = True - - if vad_params.silence_duration_ms is not None: - vad_config.silence_duration_ms = vad_params.silence_duration_ms - has_vad_settings = True - - # Only add automatic_activity_detection if we have VAD settings - if has_vad_settings: - config.realtime_input_config = RealtimeInputConfig( - automatic_activity_detection=vad_config - ) - - # Add system instruction to configuration, if provided - system_instruction = self._system_instruction or "" - if self._context and hasattr(self._context, "extract_system_instructions"): - system_instruction += "\n" + self._context.extract_system_instructions() - if system_instruction: - logger.debug(f"Setting system instruction: {system_instruction}") - config.system_instruction = system_instruction - - # Add tools to configuration, if provided - if self._tools: - logger.debug(f"Setting tools: {self._tools}") - config.tools = self.get_llm_adapter().from_standard_tools(self._tools) - - # Start the connection - self._connection_task = self.create_task(self._connection_task_handler(config=config)) - - except Exception as e: - await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}", fatal=True)) - - async def _connection_task_handler(self, config: LiveConnectConfig): - async with self._client.aio.live.connect(model=self._model_name, config=config) as session: - logger.info("Connected to Gemini service") - - # Mark connection start time - self._connection_start_time = time.time() - - await self._handle_session_ready(session) - - while True: - try: - turn = self._session.receive() - async for message in turn: - # Reset failure counter if connection has been stable - self._check_and_reset_failure_counter() - - if message.server_content and message.server_content.model_turn: - await self._handle_msg_model_turn(message) - elif ( - message.server_content - and message.server_content.turn_complete - and message.usage_metadata - ): - await self._handle_msg_turn_complete(message) - await self._handle_msg_usage_metadata(message) - elif message.server_content and message.server_content.input_transcription: - await self._handle_msg_input_transcription(message) - elif message.server_content and message.server_content.output_transcription: - await self._handle_msg_output_transcription(message) - elif message.server_content and message.server_content.grounding_metadata: - await self._handle_msg_grounding_metadata(message) - elif message.tool_call: - await self._handle_msg_tool_call(message) - elif message.session_resumption_update: - self._handle_msg_resumption_update(message) - except Exception as e: - if not self._disconnecting: - should_reconnect = await self._handle_connection_error(e) - if should_reconnect: - await self._reconnect() - return # Exit this connection handler, _reconnect will start a new one - break - - def _check_and_reset_failure_counter(self): - """Check if connection has been stable long enough to reset the failure counter. - - If the connection has been active for longer than the established threshold - and there are accumulated failures, reset the counter to 0. - """ - if ( - self._connection_start_time - and self._consecutive_failures > 0 - and time.time() - self._connection_start_time >= CONNECTION_ESTABLISHED_THRESHOLD - ): - logger.info( - f"Connection stable for {CONNECTION_ESTABLISHED_THRESHOLD}s, " - f"resetting failure counter from {self._consecutive_failures} to 0" - ) - self._consecutive_failures = 0 - - async def _handle_connection_error(self, error: Exception) -> bool: - """Handle a connection error and determine if reconnection should be attempted. - - Args: - error: The exception that caused the connection error. - - Returns: - True if reconnection should be attempted, False if a fatal error should be pushed. - """ - self._consecutive_failures += 1 - logger.warning( - f"Connection error (failure {self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES}): {error}" - ) - - if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: - logger.error( - f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, " - "treating as fatal error" - ) - await self.push_error( - ErrorFrame(error=f"{self} Error in receive loop: {error}", fatal=True) - ) - return False - else: - logger.info( - f"Attempting reconnection ({self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES})" - ) - return True - - async def _reconnect(self): - """Reconnect to Gemini Live API.""" - await self._disconnect() - await self._connect(session_resumption_handle=self._session_resumption_handle) - - async def _disconnect(self): - """Disconnect from Gemini Live API and clean up resources.""" - logger.info("Disconnecting from Gemini service") - try: - self._disconnecting = True - await self.stop_all_metrics() - if self._connection_task: - await self.cancel_task(self._connection_task, timeout=1.0) - self._connection_task = None - if self._session: - await self._session.close() - self._session = None - self._disconnecting = False - except Exception as e: - logger.error(f"{self} error disconnecting: {e}") - - async def _send_user_audio(self, frame): - """Send user audio frame to Gemini Live API.""" - if self._audio_input_paused or self._disconnecting or not self._session: - return - - # Send all audio to Gemini - try: - await self._session.send_realtime_input( - audio=Blob(data=frame.audio, mime_type=f"audio/pcm;rate={frame.sample_rate}") - ) - except Exception as e: - await self._handle_send_error(e) - - # Manage a buffer of audio to use for transcription - audio = frame.audio - if self._user_is_speaking: - self._user_audio_buffer.extend(audio) - else: - # Keep 1/2 second of audio in the buffer even when not speaking. - self._user_audio_buffer.extend(audio) - length = int((frame.sample_rate * frame.num_channels * 2) * 0.5) - self._user_audio_buffer = self._user_audio_buffer[-length:] - - async def _send_user_text(self, text: str): - """Send user text via Gemini Live API's realtime input stream. - - This method sends text through the realtimeInput stream (via TextInputMessage) - rather than the clientContent stream. This ensures text input is synchronized - with audio and video inputs, preventing temporal misalignment that can occur - when different modalities are processed through separate API pathways. - - For realtimeInput, turn completion is automatically inferred by the API based - on user activity, so no explicit turnComplete signal is needed. - - Args: - text: The text to send as user input. - """ - if self._disconnecting or not self._session: - return - - try: - await self._session.send_realtime_input(text=text) - except Exception as e: - await self._handle_send_error(e) - - async def _send_user_video(self, frame): - """Send user video frame to Gemini Live API.""" - if self._video_input_paused or self._disconnecting or not self._session: - return - - now = time.time() - if now - self._last_sent_time < 1: - return # Ignore if less than 1 second has passed - - self._last_sent_time = now # Update last sent time - logger.debug(f"Sending video frame to Gemini: {frame}") - - buffer = io.BytesIO() - Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") - data = base64.b64encode(buffer.getvalue()).decode("utf-8") - - try: - await self._session.send_realtime_input(video=Blob(data=data, mime_type="image/jpeg")) - except Exception as e: - await self._handle_send_error(e) - - async def _create_initial_response(self): - """Create initial response based on context history.""" - if self._disconnecting: - return - - if not self._session: - self._run_llm_when_session_ready = True - return - - messages = self._context.get_messages_for_initializing_history() - if not messages: - return - - logger.debug(f"Creating initial response: {messages}") - - await self.start_ttfb_metrics() - - try: - await self._session.send_client_content( - turns=messages, turn_complete=self._inference_on_context_initialization - ) - except Exception as e: - await self._handle_send_error(e) - - # If we're generating a response right away upon initializing - # conversation history, set a flag saying that we need a turn complete - # message when the user stops speaking. - if not self._inference_on_context_initialization: - self._needs_turn_complete_message = True - - async def _create_single_response(self, messages_list): - """Create a single response from a list of messages.""" - if self._disconnecting or not self._session: - return - - # Create a throwaway context just for the purpose of getting messages - # in the right format - context = GeminiMultimodalLiveContext.upgrade(OpenAILLMContext(messages=messages_list)) - messages = context.get_messages_for_initializing_history() - - if not messages: - return - - logger.debug(f"Creating response: {messages}") - - await self.start_ttfb_metrics() - - try: - await self._session.send_client_content(turns=messages, turn_complete=True) - except Exception as e: - await self._handle_send_error(e) - - @traced_gemini_live(operation="llm_tool_result") - async def _tool_result(self, tool_result_message): - """Send tool result back to the API.""" - if self._disconnecting or not self._session: - return - - # For now we're shoving the name into the tool_call_id field, so this - # will work until we revisit that. - id = tool_result_message.get("tool_call_id") - name = tool_result_message.get("tool_call_name") - result = json.loads(tool_result_message.get("content") or "") - response = FunctionResponse(name=name, id=id, response=result) - - try: - await self._session.send_tool_response(function_responses=response) - except Exception as e: - await self._handle_send_error(e) - - @traced_gemini_live(operation="llm_setup") - async def _handle_session_ready(self, session: AsyncSession): - """Handle the session being ready.""" - self._session = session - # If we were just waititng for the session to be ready to run the LLM, - # do that now. - if self._run_llm_when_session_ready: - self._run_llm_when_session_ready = False - await self._create_initial_response() - - async def _handle_msg_model_turn(self, msg: LiveServerMessage): - """Handle the model turn message.""" - part = msg.server_content.model_turn.parts[0] - if not part: - return - - await self.stop_ttfb_metrics() - - # part.text is added when `modalities` is set to TEXT; otherwise, it's None - text = part.text - if text: - if not self._bot_text_buffer: - await self.push_frame(LLMFullResponseStartFrame()) - - self._bot_text_buffer += text - self._search_result_buffer += text # Also accumulate for grounding - await self.push_frame(LLMTextFrame(text=text)) - - # Check for grounding metadata in server content - if msg.server_content and msg.server_content.grounding_metadata: - self._accumulated_grounding_metadata = msg.server_content.grounding_metadata - - inline_data = part.inline_data - if not inline_data: - return - if inline_data.mime_type != f"audio/pcm;rate={self._sample_rate}": - logger.warning(f"Unrecognized server_content format {inline_data.mime_type}") - return - - audio = inline_data.data - if not audio: - return - - if not self._bot_is_speaking: - await self._set_bot_is_speaking(True) - await self.push_frame(TTSStartedFrame()) - await self.push_frame(LLMFullResponseStartFrame()) - - self._bot_audio_buffer.extend(audio) - frame = TTSAudioRawFrame( - audio=audio, - sample_rate=self._sample_rate, - num_channels=1, - ) - await self.push_frame(frame) - - @traced_gemini_live(operation="llm_tool_call") - async def _handle_msg_tool_call(self, message: LiveServerMessage): - """Handle tool call messages.""" - function_calls = message.tool_call.function_calls - if not function_calls: - return - if not self._context: - logger.error("Function calls are not supported without a context object.") - - function_calls_llm = [ - FunctionCallFromLLM( - context=self._context, - tool_call_id=f.id, - function_name=f.name, - arguments=f.args, - ) - for f in function_calls - ] - - await self.run_function_calls(function_calls_llm) - - @traced_gemini_live(operation="llm_response") - async def _handle_msg_turn_complete(self, message: LiveServerMessage): - """Handle the turn complete message.""" - await self._set_bot_is_speaking(False) - text = self._bot_text_buffer - - # Trace the complete LLM response (this will be handled by the decorator) - # The decorator will extract the output text and usage metadata from the message - - self._bot_text_buffer = "" - self._llm_output_buffer = "" - - # Process grounding metadata if we have accumulated any - if self._accumulated_grounding_metadata: - await self._process_grounding_metadata( - self._accumulated_grounding_metadata, self._search_result_buffer - ) - - # Reset grounding tracking for next response - self._search_result_buffer = "" - self._accumulated_grounding_metadata = None - - # Only push the TTSStoppedFrame if the bot is outputting audio - # when text is found, modalities is set to TEXT and no audio - # is produced. - if not text: - await self.push_frame(TTSStoppedFrame()) - - await self.push_frame(LLMFullResponseEndFrame()) - - @traced_stt - async def _handle_user_transcription( - self, transcript: str, is_final: bool, language: Optional[Language] = None - ): - """Handle a transcription result with tracing.""" - pass - - async def _handle_msg_input_transcription(self, message: LiveServerMessage): - """Handle the input transcription message. - - Gemini Live sends user transcriptions in either single words or multi-word - phrases. As a result, we have to aggregate the input transcription. This handler - aggregates into sentences, splitting on the end of sentence markers. - """ - if not message.server_content.input_transcription: - return - - text = message.server_content.input_transcription.text - - if not text: - return - - # Strip leading space from sentence starts if buffer is empty - if text.startswith(" ") and not self._user_transcription_buffer: - text = text.lstrip() - - # Accumulate text in the buffer - self._user_transcription_buffer += text - - # Check for complete sentences - while True: - eos_end_marker = match_endofsentence(self._user_transcription_buffer) - if not eos_end_marker: - break - - # Extract the complete sentence - complete_sentence = self._user_transcription_buffer[:eos_end_marker] - # Keep the remainder for the next chunk - self._user_transcription_buffer = self._user_transcription_buffer[eos_end_marker:] - - # Send a TranscriptionFrame with the complete sentence - logger.debug(f"[Transcription:user] [{complete_sentence}]") - await self._handle_user_transcription( - complete_sentence, True, self._settings["language"] - ) - await self.push_frame( - TranscriptionFrame( - text=complete_sentence, - user_id="", - timestamp=time_now_iso8601(), - result=message, - ), - FrameDirection.UPSTREAM, - ) - - async def _handle_msg_output_transcription(self, message: LiveServerMessage): - """Handle the output transcription message.""" - if not message.server_content.output_transcription: - return - - # This is the output transcription text when modalities is set to AUDIO. - # In this case, we push LLMTextFrame and TTSTextFrame to be handled by the - # downstream assistant context aggregator. - text = message.server_content.output_transcription.text - - if not text: - return - - # Accumulate text for grounding as well - self._search_result_buffer += text - - # Check for grounding metadata in server content - if message.server_content and message.server_content.grounding_metadata: - self._accumulated_grounding_metadata = message.server_content.grounding_metadata - # Collect text for tracing - self._llm_output_buffer += text - - await self.push_frame(LLMTextFrame(text=text)) - await self.push_frame(TTSTextFrame(text=text)) - - async def _handle_msg_grounding_metadata(self, message: LiveServerMessage): - """Handle dedicated grounding metadata messages.""" - if message.server_content and message.server_content.grounding_metadata: - grounding_metadata = message.server_content.grounding_metadata - # Process the grounding metadata immediately - await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer) - - async def _process_grounding_metadata( - self, grounding_metadata: GroundingMetadata, search_result: str = "" - ): - """Process grounding metadata and emit LLMSearchResponseFrame.""" - if not grounding_metadata: - return - - # Extract rendered content for search suggestions - rendered_content = None - if ( - grounding_metadata.search_entry_point - and grounding_metadata.search_entry_point.rendered_content - ): - rendered_content = grounding_metadata.search_entry_point.rendered_content - - # Convert grounding chunks and supports to LLMSearchOrigin format - origins = [] - - if grounding_metadata.grounding_chunks and grounding_metadata.grounding_supports: - # Create a mapping of chunk indices to origins - chunk_to_origin: Dict[int, LLMSearchOrigin] = {} - - for index, chunk in enumerate(grounding_metadata.grounding_chunks): - if chunk.web: - origin = LLMSearchOrigin( - site_uri=chunk.web.uri, site_title=chunk.web.title, results=[] - ) - chunk_to_origin[index] = origin - origins.append(origin) - - # Add grounding support results to the appropriate origins - for support in grounding_metadata.grounding_supports: - if support.segment and support.grounding_chunk_indices: - text = support.segment.text or "" - confidence_scores = support.confidence_scores or [] - - # Add this result to all origins referenced by this support - for chunk_index in support.grounding_chunk_indices: - if chunk_index in chunk_to_origin: - result = LLMSearchResult(text=text, confidence=confidence_scores) - chunk_to_origin[chunk_index].results.append(result) - - # Create and push the search response frame - search_frame = LLMSearchResponseFrame( - search_result=search_result, origins=origins, rendered_content=rendered_content - ) - - await self.push_frame(search_frame) - - async def _handle_msg_usage_metadata(self, message: LiveServerMessage): - """Handle the usage metadata message.""" - if not message.usage_metadata: - return - - usage = message.usage_metadata - - # Ensure we have valid integers for all token counts - prompt_tokens = usage.prompt_token_count or 0 - completion_tokens = usage.response_token_count or 0 - total_tokens = usage.total_token_count or (prompt_tokens + completion_tokens) - - tokens = LLMTokenUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) - - await self.start_llm_usage_metrics(tokens) - - def _handle_msg_resumption_update(self, message: LiveServerMessage): - update = message.session_resumption_update - if update.resumable and update.new_handle: - self._session_resumption_handle = update.new_handle - - async def _handle_send_error(self, error: Exception): - # In server-to-server contexts, a WebSocket error should be quite rare. - # Given how hard it is to recover from a send-side error with proper - # state management, and that exponential backoff for retries can have - # cost/stability implications for a service cluster, let's just treat a - # send-side error as fatal. - if not self._disconnecting: - await self.push_error(ErrorFrame(error=f"{self} Send error: {error}", fatal=True)) - - def create_context_aggregator( - self, - context: OpenAILLMContext, - *, - user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), - assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), - ) -> GeminiMultimodalLiveContextAggregatorPair: - """Create an instance of GeminiMultimodalLiveContextAggregatorPair from an OpenAILLMContext. - - Constructor keyword arguments for both the user and assistant aggregators can be provided. - - Args: - context: The LLM context to use. - user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams(). - assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams(). - - Returns: - GeminiMultimodalLiveContextAggregatorPair: A pair of context - aggregators, one for the user and one for the assistant, - encapsulated in an GeminiMultimodalLiveContextAggregatorPair. - """ - context.set_llm_adapter(self.get_llm_adapter()) - - GeminiMultimodalLiveContext.upgrade(context) - user = GeminiMultimodalLiveUserContextAggregator(context, params=user_params) - - assistant_params.expect_stripped_words = False - assistant = GeminiMultimodalLiveAssistantContextAggregator(context, params=assistant_params) - return GeminiMultimodalLiveContextAggregatorPair(_user=user, _assistant=assistant) +GeminiMultimodalLiveContext = GeminiLiveContext +GeminiMultimodalLiveUserContextAggregator = GeminiLiveUserContextAggregator +GeminiMultimodalLiveAssistantContextAggregator = GeminiLiveAssistantContextAggregator +GeminiMultimodalLiveContextAggregatorPair = GeminiLiveContextAggregatorPair +GeminiMultimodalModalities = GeminiModalities +GeminiMediaResolution = _GeminiMediaResolution +GeminiVADParams = _GeminiVADParams +ContextWindowCompressionParams = _ContextWindowCompressionParams +InputParams = _InputParams +GeminiMultimodalLiveLLMService = GeminiLiveLLMService diff --git a/src/pipecat/services/google/__init__.py b/src/pipecat/services/google/__init__.py index ec187000f..08e295b9e 100644 --- a/src/pipecat/services/google/__init__.py +++ b/src/pipecat/services/google/__init__.py @@ -9,6 +9,7 @@ import sys from pipecat.services import DeprecatedModuleProxy from .frames import * +from .gemini_live import * from .image import * from .llm import * from .llm_openai import * diff --git a/src/pipecat/services/google/gemini_live/__init__.py b/src/pipecat/services/google/gemini_live/__init__.py new file mode 100644 index 000000000..142ca2a83 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/__init__.py @@ -0,0 +1,3 @@ +from .file_api import GeminiFileAPI +from .llm import GeminiLiveLLMService +from .llm_vertex import GeminiLiveVertexLLMService diff --git a/src/pipecat/services/google/gemini_live/file_api.py b/src/pipecat/services/google/gemini_live/file_api.py new file mode 100644 index 000000000..5ae7fdbb7 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/file_api.py @@ -0,0 +1,189 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Gemini File API client for uploading and managing files. + +This module provides a client for Google's Gemini File API, enabling file +uploads, metadata retrieval, listing, and deletion. Files uploaded through +this API can be referenced in Gemini generative model calls. +""" + +import mimetypes +from typing import Any, Dict, Optional + +import aiohttp +from loguru import logger + + +class GeminiFileAPI: + """Client for the Gemini File API. + + This class provides methods for uploading, fetching, listing, and deleting files + through Google's Gemini File API. + + Files uploaded through this API remain available for 48 hours and can be referenced + in calls to the Gemini generative models. Maximum file size is 2GB, with total + project storage limited to 20GB. + """ + + def __init__( + self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files" + ): + """Initialize the Gemini File API client. + + Args: + api_key: Google AI API key + base_url: Base URL for the Gemini File API (default is the v1beta endpoint) + """ + self._api_key = api_key + self._base_url = base_url + # Upload URL uses the /upload/ path + self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files" + + async def upload_file( + self, file_path: str, display_name: Optional[str] = None + ) -> Dict[str, Any]: + """Upload a file to the Gemini File API using the correct resumable upload protocol. + + Args: + file_path: Path to the file to upload + display_name: Optional display name for the file + + Returns: + File metadata including uri, name, and display_name + """ + logger.info(f"Uploading file: {file_path}") + + async with aiohttp.ClientSession() as session: + # Determine the file's MIME type + mime_type, _ = mimetypes.guess_type(file_path) + if not mime_type: + mime_type = "application/octet-stream" + + # Read the file + with open(file_path, "rb") as f: + file_data = f.read() + + # Create the metadata payload + metadata = {} + if display_name: + metadata = {"file": {"display_name": display_name}} + + # Step 1: Initial resumable request to get upload URL + headers = { + "X-Goog-Upload-Protocol": "resumable", + "X-Goog-Upload-Command": "start", + "X-Goog-Upload-Header-Content-Length": str(len(file_data)), + "X-Goog-Upload-Header-Content-Type": mime_type, + "Content-Type": "application/json", + } + + logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}") + async with session.post( + f"{self.upload_base_url}?key={self._api_key}", headers=headers, json=metadata + ) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error initiating file upload: {error_text}") + raise Exception(f"Failed to initiate upload: {response.status} - {error_text}") + + # Get the upload URL from the response header + upload_url = response.headers.get("X-Goog-Upload-URL") + if not upload_url: + logger.error(f"Response headers: {dict(response.headers)}") + raise Exception("No upload URL in response headers") + + logger.debug(f"Got upload URL: {upload_url}") + + # Step 2: Upload the actual file data + upload_headers = { + "Content-Length": str(len(file_data)), + "X-Goog-Upload-Offset": "0", + "X-Goog-Upload-Command": "upload, finalize", + } + + logger.debug(f"Step 2: Uploading file data to {upload_url}") + async with session.post(upload_url, headers=upload_headers, data=file_data) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error uploading file data: {error_text}") + raise Exception(f"Failed to upload file: {response.status} - {error_text}") + + file_info = await response.json() + logger.info(f"File uploaded successfully: {file_info.get('file', {}).get('name')}") + return file_info + + async def get_file(self, name: str) -> Dict[str, Any]: + """Get metadata for a file. + + Args: + name: File name (or full path) + + Returns: + File metadata + """ + # Extract just the name part if a full path is provided + if "/" in name: + name = name.split("/")[-1] + + async with aiohttp.ClientSession() as session: + async with session.get(f"{self._base_url}/{name}?key={self._api_key}") as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error getting file metadata: {error_text}") + raise Exception(f"Failed to get file metadata: {response.status}") + + file_info = await response.json() + return file_info + + async def list_files( + self, page_size: int = 10, page_token: Optional[str] = None + ) -> Dict[str, Any]: + """List uploaded files. + + Args: + page_size: Number of files to return per page + page_token: Token for pagination + + Returns: + List of files and next page token if available + """ + params = {"key": self._api_key, "pageSize": page_size} + + if page_token: + params["pageToken"] = page_token + + async with aiohttp.ClientSession() as session: + async with session.get(self._base_url, params=params) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error listing files: {error_text}") + raise Exception(f"Failed to list files: {response.status}") + + result = await response.json() + return result + + async def delete_file(self, name: str) -> bool: + """Delete a file. + + Args: + name: File name (or full path) + + Returns: + True if deleted successfully + """ + # Extract just the name part if a full path is provided + if "/" in name: + name = name.split("/")[-1] + + async with aiohttp.ClientSession() as session: + async with session.delete(f"{self._base_url}/{name}?key={self._api_key}") as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Error deleting file: {error_text}") + raise Exception(f"Failed to delete file: {response.status}") + + return True diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py new file mode 100644 index 000000000..4b5f05209 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -0,0 +1,1582 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Google Gemini Live API service implementation. + +This module provides real-time conversational AI capabilities using Google's +Gemini Live API, supporting both text and audio modalities with +voice transcription, streaming responses, and tool usage. +""" + +import base64 +import io +import json +import random +import time +import uuid +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from loguru import logger +from PIL import Image +from pydantic import BaseModel, Field + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + InputAudioRawFrame, + InputImageRawFrame, + InputTextRawFrame, + InterruptionFrame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesAppendFrame, + LLMSetToolsFrame, + LLMTextFrame, + LLMUpdateSettingsFrame, + StartFrame, + TranscriptionFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, + TTSTextFrame, + UserImageRawFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, +) +from pipecat.metrics.metrics import LLMTokenUsage +from pipecat.processors.aggregators.llm_response import ( + LLMAssistantAggregatorParams, + LLMUserAggregatorParams, +) +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult +from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) +from pipecat.transcriptions.language import Language +from pipecat.utils.string import match_endofsentence +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt + +from .file_api import GeminiFileAPI + +try: + from google.genai import Client + from google.genai.live import AsyncSession + from google.genai.types import ( + AudioTranscriptionConfig, + AutomaticActivityDetection, + Blob, + Content, + ContextWindowCompressionConfig, + EndSensitivity, + FileData, + FunctionResponse, + GenerationConfig, + GroundingMetadata, + HttpOptions, + LiveConnectConfig, + LiveServerMessage, + MediaResolution, + Modality, + Part, + ProactivityConfig, + RealtimeInputConfig, + SessionResumptionConfig, + SlidingWindow, + SpeechConfig, + StartSensitivity, + ThinkingConfig, + VoiceConfig, + ) +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.") + raise Exception(f"Missing module: {e}") + + +# Connection management constants +MAX_CONSECUTIVE_FAILURES = 3 +CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds + + +def language_to_gemini_language(language: Language) -> Optional[str]: + """Maps a Language enum value to a Gemini Live supported language code. + + Source: + https://ai.google.dev/api/generate-content#MediaResolution + + Args: + language: The language enum value to convert. + + Returns: + The Gemini language code string, or None if the language is not supported. + """ + language_map = { + # Arabic + Language.AR: "ar-XA", + # Bengali + Language.BN_IN: "bn-IN", + # Chinese (Mandarin) + Language.CMN: "cmn-CN", + Language.CMN_CN: "cmn-CN", + Language.ZH: "cmn-CN", # Map general Chinese to Mandarin for Gemini + Language.ZH_CN: "cmn-CN", # Map Simplified Chinese to Mandarin for Gemini + # German + Language.DE: "de-DE", + Language.DE_DE: "de-DE", + # English + Language.EN: "en-US", # Default to US English (though not explicitly listed in supported codes) + Language.EN_US: "en-US", + Language.EN_AU: "en-AU", + Language.EN_GB: "en-GB", + Language.EN_IN: "en-IN", + # Spanish + Language.ES: "es-ES", # Default to Spain Spanish + Language.ES_ES: "es-ES", + Language.ES_US: "es-US", + # French + Language.FR: "fr-FR", # Default to France French + Language.FR_FR: "fr-FR", + Language.FR_CA: "fr-CA", + # Gujarati + Language.GU: "gu-IN", + Language.GU_IN: "gu-IN", + # Hindi + Language.HI: "hi-IN", + Language.HI_IN: "hi-IN", + # Indonesian + Language.ID: "id-ID", + Language.ID_ID: "id-ID", + # Italian + Language.IT: "it-IT", + Language.IT_IT: "it-IT", + # Japanese + Language.JA: "ja-JP", + Language.JA_JP: "ja-JP", + # Kannada + Language.KN: "kn-IN", + Language.KN_IN: "kn-IN", + # Korean + Language.KO: "ko-KR", + Language.KO_KR: "ko-KR", + # Malayalam + Language.ML: "ml-IN", + Language.ML_IN: "ml-IN", + # Marathi + Language.MR: "mr-IN", + Language.MR_IN: "mr-IN", + # Dutch + Language.NL: "nl-NL", + Language.NL_NL: "nl-NL", + # Polish + Language.PL: "pl-PL", + Language.PL_PL: "pl-PL", + # Portuguese (Brazil) + Language.PT_BR: "pt-BR", + # Russian + Language.RU: "ru-RU", + Language.RU_RU: "ru-RU", + # Tamil + Language.TA: "ta-IN", + Language.TA_IN: "ta-IN", + # Telugu + Language.TE: "te-IN", + Language.TE_IN: "te-IN", + # Thai + Language.TH: "th-TH", + Language.TH_TH: "th-TH", + # Turkish + Language.TR: "tr-TR", + Language.TR_TR: "tr-TR", + # Vietnamese + Language.VI: "vi-VN", + Language.VI_VN: "vi-VN", + } + return language_map.get(language) + + +class GeminiLiveContext(OpenAILLMContext): + """Extended OpenAI context for Gemini Live API. + + Provides Gemini-specific context management including system instruction + extraction and message format conversion for the Live API. + """ + + @staticmethod + def upgrade(obj: OpenAILLMContext) -> "GeminiLiveContext": + """Upgrade an OpenAI context to Gemini context. + + Args: + obj: The OpenAI context to upgrade. + + Returns: + The upgraded Gemini context instance. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiLiveContext): + logger.debug(f"Upgrading to Gemini Live Context: {obj}") + obj.__class__ = GeminiLiveContext + obj._restructure_from_openai_messages() + return obj + + def _restructure_from_openai_messages(self): + pass + + def extract_system_instructions(self): + """Extract system instructions from context messages. + + Returns: + Combined system instruction text from all system messages. + """ + system_instruction = "" + for item in self.messages: + if item.get("role") == "system": + content = item.get("content", "") + if content: + if system_instruction and not system_instruction.endswith("\n"): + system_instruction += "\n" + system_instruction += str(content) + return system_instruction + + def add_file_reference(self, file_uri: str, mime_type: str, text: Optional[str] = None): + """Add a file reference to the context. + + This adds a user message with a file reference that will be sent during context initialization. + + Args: + file_uri: URI of the uploaded file + mime_type: MIME type of the file + text: Optional text prompt to accompany the file + """ + # Create parts list with file reference + parts = [] + if text: + parts.append({"type": "text", "text": text}) + + # Add file reference part + parts.append( + {"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}} + ) + + # Add to messages + message = {"role": "user", "content": parts} + self.messages.append(message) + logger.info(f"Added file reference to context: {file_uri}") + + def get_messages_for_initializing_history(self) -> List[Content]: + """Get messages formatted for Gemini history initialization. + + Returns: + List of messages in Gemini format for conversation history. + """ + messages: List[Content] = [] + for item in self.messages: + role = item.get("role") + + if role == "system": + continue + + elif role == "assistant": + role = "model" + + content = item.get("content") + parts: List[Part] = [] + if isinstance(content, str): + parts = [Part(text=content)] + elif isinstance(content, list): + for part in content: + if part.get("type") == "text": + parts.append(Part(text=part.get("text"))) + elif part.get("type") == "file_data": + file_data = part.get("file_data", {}) + parts.append( + Part( + file_data=FileData( + mime_type=file_data.get("mime_type"), + file_uri=file_data.get("file_uri"), + ) + ) + ) + else: + logger.warning(f"Unsupported content type: {str(part)[:80]}") + else: + logger.warning(f"Unsupported content type: {str(content)[:80]}") + messages.append(Content(role=role, parts=parts)) + return messages + + +class GeminiLiveUserContextAggregator(OpenAIUserContextAggregator): + """User context aggregator for Gemini Live. + + Extends OpenAI user aggregator to handle Gemini-specific message passing + while maintaining compatibility with the standard aggregation pipeline. + """ + + async def process_frame(self, frame, direction): + """Process incoming frames for user context aggregation. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ + await super().process_frame(frame, direction) + # kind of a hack just to pass the LLMMessagesAppendFrame through, but it's fine for now + if isinstance(frame, LLMMessagesAppendFrame): + await self.push_frame(frame, direction) + + +class GeminiLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Assistant context aggregator for Gemini Live. + + Handles assistant response aggregation while filtering out LLMTextFrames + to prevent duplicate context entries, as Gemini Live pushes both + LLMTextFrames and TTSTextFrames. + """ + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames for assistant context aggregation. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the GeminiLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. + if not isinstance(frame, LLMTextFrame): + await super().process_frame(frame, direction) + + async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle user image frames. + + Args: + frame: The user image frame to handle. + """ + # We don't want to store any images in the context. Revisit this later + # when the API evolves. + pass + + +@dataclass +class GeminiLiveContextAggregatorPair: + """Pair of user and assistant context aggregators for Gemini Live. + + Parameters: + _user: The user context aggregator instance. + _assistant: The assistant context aggregator instance. + """ + + _user: GeminiLiveUserContextAggregator + _assistant: GeminiLiveAssistantContextAggregator + + def user(self) -> GeminiLiveUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ + return self._user + + def assistant(self) -> GeminiLiveAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ + return self._assistant + + +class GeminiModalities(Enum): + """Supported modalities for Gemini Live. + + Parameters: + TEXT: Text responses. + AUDIO: Audio responses. + """ + + TEXT = "TEXT" + AUDIO = "AUDIO" + + +class GeminiMediaResolution(str, Enum): + """Media resolution options for Gemini Live. + + Parameters: + UNSPECIFIED: Use default resolution setting. + LOW: Low resolution with 64 tokens. + MEDIUM: Medium resolution with 256 tokens. + HIGH: High resolution with zoomed reframing and 256 tokens. + """ + + UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default + LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens + MEDIUM = "MEDIA_RESOLUTION_MEDIUM" # 256 tokens + HIGH = "MEDIA_RESOLUTION_HIGH" # Zoomed reframing with 256 tokens + + +class GeminiVADParams(BaseModel): + """Voice Activity Detection parameters for Gemini Live. + + Parameters: + disabled: Whether to disable VAD. Defaults to None. + start_sensitivity: Sensitivity for speech start detection. Defaults to None. + end_sensitivity: Sensitivity for speech end detection. Defaults to None. + prefix_padding_ms: Prefix padding in milliseconds. Defaults to None. + silence_duration_ms: Silence duration threshold in milliseconds. Defaults to None. + """ + + disabled: Optional[bool] = Field(default=None) + start_sensitivity: Optional[StartSensitivity] = Field(default=None) + end_sensitivity: Optional[EndSensitivity] = Field(default=None) + prefix_padding_ms: Optional[int] = Field(default=None) + silence_duration_ms: Optional[int] = Field(default=None) + + +class ContextWindowCompressionParams(BaseModel): + """Parameters for context window compression in Gemini Live. + + Parameters: + enabled: Whether compression is enabled. Defaults to False. + trigger_tokens: Token count to trigger compression. None uses 80% of context window. + """ + + enabled: bool = Field(default=False) + trigger_tokens: Optional[int] = Field( + default=None + ) # None = use default (80% of context window) + + +class InputParams(BaseModel): + """Input parameters for Gemini Live generation. + + Parameters: + frequency_penalty: Frequency penalty for generation (0.0-2.0). Defaults to None. + max_tokens: Maximum tokens to generate. Must be >= 1. Defaults to 4096. + presence_penalty: Presence penalty for generation (0.0-2.0). Defaults to None. + temperature: Sampling temperature (0.0-2.0). Defaults to None. + top_k: Top-k sampling parameter. Must be >= 0. Defaults to None. + top_p: Top-p sampling parameter (0.0-1.0). Defaults to None. + modalities: Response modalities. Defaults to AUDIO. + language: Language for generation. Defaults to EN_US. + media_resolution: Media resolution setting. Defaults to UNSPECIFIED. + vad: Voice activity detection parameters. Defaults to None. + context_window_compression: Context compression settings. Defaults to None. + thinking: Thinking settings. Defaults to None. + Note that these settings may require specifying a model that + supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". + enable_affective_dialog: Enable affective dialog, which allows Gemini + to adapt to expression and tone. Defaults to None. + Note that these settings may require specifying a model that + supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". + Also note that this setting may require specifying an API version that + supports it, e.g. HttpOptions(api_version="v1alpha"). + proactivity: Proactivity settings, which allows Gemini to proactively + decide how to behave, such as whether to avoid responding to + content that is not relevant. Defaults to None. + Note that these settings may require specifying a model that + supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025". + Also note that this setting may require specifying an API version that + supports it, e.g. HttpOptions(api_version="v1alpha"). + extra: Additional parameters. Defaults to empty dict. + """ + + frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) + max_tokens: Optional[int] = Field(default=4096, ge=1) + presence_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) + temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) + top_k: Optional[int] = Field(default=None, ge=0) + top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) + modalities: Optional[GeminiModalities] = Field(default=GeminiModalities.AUDIO) + language: Optional[Language] = Field(default=Language.EN_US) + media_resolution: Optional[GeminiMediaResolution] = Field( + default=GeminiMediaResolution.UNSPECIFIED + ) + vad: Optional[GeminiVADParams] = Field(default=None) + context_window_compression: Optional[ContextWindowCompressionParams] = Field(default=None) + thinking: Optional[ThinkingConfig] = Field(default=None) + enable_affective_dialog: Optional[bool] = Field(default=None) + proactivity: Optional[ProactivityConfig] = Field(default=None) + extra: Optional[Dict[str, Any]] = Field(default_factory=dict) + + +class GeminiLiveLLMService(LLMService): + """Provides access to Google's Gemini Live API. + + This service enables real-time conversations with Gemini, supporting both + text and audio modalities. It handles voice transcription, streaming audio + responses, and tool usage. + """ + + # Overriding the default adapter to use the Gemini one. + adapter_class = GeminiLLMAdapter + + def __init__( + self, + *, + api_key: str, + base_url: Optional[str] = None, + model="models/gemini-2.0-flash-live-001", + voice_id: str = "Charon", + start_audio_paused: bool = False, + start_video_paused: bool = False, + system_instruction: Optional[str] = None, + tools: Optional[Union[List[dict], ToolsSchema]] = None, + params: Optional[InputParams] = None, + inference_on_context_initialization: bool = True, + file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files", + http_options: Optional[HttpOptions] = None, + **kwargs, + ): + """Initialize the Gemini Live LLM service. + + Args: + api_key: Google AI API key for authentication. + base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint. + + .. deprecated:: 0.0.90 + This parameter is deprecated and no longer has any effect. + Please use `http_options` to customize requests made by the + API client. + + model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001". + voice_id: TTS voice identifier. Defaults to "Charon". + start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_video_paused: Whether to start with video input paused. Defaults to False. + system_instruction: System prompt for the model. Defaults to None. + tools: Tools/functions available to the model. Defaults to None. + params: Configuration parameters for the model. Defaults to InputParams(). + inference_on_context_initialization: Whether to generate a response when context + is first set. Defaults to True. + file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint. + http_options: HTTP options for the client. + **kwargs: Additional arguments passed to parent LLMService. + """ + # Check for deprecated parameter usage + if base_url is not None: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'base_url' is deprecated and no longer has any effect. Please use 'http_options' to customize requests made by the API client.", + DeprecationWarning, + stacklevel=2, + ) + + super().__init__(base_url=base_url, **kwargs) + + params = params or InputParams() + + self._last_sent_time = 0 + self._base_url = base_url + self.set_model_name(model) + self._voice_id = voice_id + self._language_code = params.language + + self._system_instruction = system_instruction + self._tools = tools + self._inference_on_context_initialization = inference_on_context_initialization + self._needs_turn_complete_message = False + + self._audio_input_paused = start_audio_paused + self._video_input_paused = start_video_paused + self._context = None + self._api_key = api_key + self._http_options = http_options + self._session: AsyncSession = None + self._connection_task = None + + self._disconnecting = False + self._run_llm_when_session_ready = False + + self._user_is_speaking = False + self._bot_is_speaking = False + self._user_audio_buffer = bytearray() + self._user_transcription_buffer = "" + self._last_transcription_sent = "" + self._bot_audio_buffer = bytearray() + self._bot_text_buffer = "" + self._llm_output_buffer = "" + + self._sample_rate = 24000 + + self._language = params.language + self._language_code = ( + language_to_gemini_language(params.language) if params.language else "en-US" + ) + self._vad_params = params.vad + + # Reconnection tracking + self._consecutive_failures = 0 + self._connection_start_time = None + + self._settings = { + "frequency_penalty": params.frequency_penalty, + "max_tokens": params.max_tokens, + "presence_penalty": params.presence_penalty, + "temperature": params.temperature, + "top_k": params.top_k, + "top_p": params.top_p, + "modalities": params.modalities, + "language": self._language_code, + "media_resolution": params.media_resolution, + "vad": params.vad, + "context_window_compression": params.context_window_compression.model_dump() + if params.context_window_compression + else {}, + "thinking": params.thinking or {}, + "enable_affective_dialog": params.enable_affective_dialog or False, + "proactivity": params.proactivity or {}, + "extra": params.extra if isinstance(params.extra, dict) else {}, + } + + self._file_api_base_url = file_api_base_url + self._file_api: Optional[GeminiFileAPI] = None + + # Grounding metadata tracking + self._search_result_buffer = "" + self._accumulated_grounding_metadata = None + + # Session resumption + self._session_resumption_handle: Optional[str] = None + + # Bookkeeping for ending gracefully (i.e. after the bot is finished) + self._end_frame_pending_bot_turn_finished: Optional[EndFrame] = None + + # Initialize the API client. Subclasses can override this if needed. + self.create_client() + + def create_client(self): + """Create the Gemini API client instance. Subclasses can override this.""" + self._client = Client(api_key=self._api_key, http_options=self._http_options) + + @property + def file_api(self) -> GeminiFileAPI: + """Get the Gemini File API client instance. Subclasses can override this. + + Returns: + The Gemini File API client. + """ + if not self._file_api: + self._file_api = GeminiFileAPI(api_key=self._api_key, base_url=self._file_api_base_url) + return self._file_api + + def can_generate_metrics(self) -> bool: + """Check if the service can generate usage metrics. + + Returns: + True as Gemini Live supports token usage metrics. + """ + return True + + def needs_mcp_alternate_schema(self) -> bool: + """Check if this LLM service requires alternate MCP schema. + + Google/Gemini has stricter JSON schema validation and requires + certain properties to be removed or modified for compatibility. + + Returns: + True for Google/Gemini services. + """ + return True + + def set_audio_input_paused(self, paused: bool): + """Set the audio input pause state. + + Args: + paused: Whether to pause audio input. + """ + self._audio_input_paused = paused + + def set_video_input_paused(self, paused: bool): + """Set the video input pause state. + + Args: + paused: Whether to pause video input. + """ + self._video_input_paused = paused + + def set_model_modalities(self, modalities: GeminiModalities): + """Set the model response modalities. + + Args: + modalities: The modalities to use for responses. + """ + self._settings["modalities"] = modalities + + def set_language(self, language: Language): + """Set the language for generation. + + Args: + language: The language to use for generation. + """ + self._language = language + self._language_code = language_to_gemini_language(language) or "en-US" + self._settings["language"] = self._language_code + logger.info(f"Set Gemini language to: {self._language_code}") + + async def set_context(self, context: OpenAILLMContext): + """Set the context explicitly from outside the pipeline. + + This is useful when initializing a conversation because in server-side VAD mode we might not have a + way to trigger the pipeline. This sends the history to the server. The `inference_on_context_initialization` + flag controls whether to set the turnComplete flag when we do this. Without that flag, the model will + not respond. This is often what we want when setting the context at the beginning of a conversation. + + Args: + context: The OpenAI LLM context to set. + """ + if self._context: + logger.error("Context already set. Can only set up Gemini Live context once.") + return + self._context = GeminiLiveContext.upgrade(context) + await self._create_initial_response() + + # + # standard AIService frame handling + # + + async def start(self, frame: StartFrame): + """Start the service and establish connection. + + Args: + frame: The start frame. + """ + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the service and close connections. + + Args: + frame: The end frame. + """ + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel the service and close connections. + + Args: + frame: The cancel frame. + """ + await super().cancel(frame) + await self._disconnect() + + # + # speech and interruption handling + # + + async def _handle_interruption(self): + await self._set_bot_is_speaking(False) + await self.push_frame(TTSStoppedFrame()) + await self.push_frame(LLMFullResponseEndFrame()) + + async def _handle_user_started_speaking(self, frame): + self._user_is_speaking = True + pass + + async def _handle_user_stopped_speaking(self, frame): + self._user_is_speaking = False + self._user_audio_buffer = bytearray() + await self.start_ttfb_metrics() + if self._needs_turn_complete_message: + self._needs_turn_complete_message = False + # NOTE: without this, the model ignores the context it's been + # seeded with before the user started speaking + await self._session.send_client_content(turn_complete=True) + + # + # frame processing + # + # StartFrame, StopFrame, CancelFrame implemented in base class + # + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames for the Gemini Live service. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ + # Defer EndFrame handling until after the bot turn is finished + if isinstance(frame, EndFrame): + if self._bot_is_speaking: + logger.debug("Deferring handling EndFrame until bot turn is finished") + self._end_frame_pending_bot_turn_finished = frame + return + + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + await self.push_frame(frame, direction) + elif isinstance(frame, OpenAILLMContextFrame): + context: GeminiLiveContext = GeminiLiveContext.upgrade(frame.context) + # For now, we'll only trigger inference here when either: + # 1. We have not seen a context frame before + # 2. The last message is a tool call result + if not self._context: + self._context = context + if frame.context.tools: + self._tools = frame.context.tools + await self._create_initial_response() + elif context.messages and context.messages[-1].get("role") == "tool": + # Support just one tool call per context frame for now + tool_result_message = context.messages[-1] + await self._tool_result(tool_result_message) + elif isinstance(frame, LLMContextFrame): + raise NotImplementedError("Universal LLMContext is not yet supported for Gemini Live.") + elif isinstance(frame, InputTextRawFrame): + await self._send_user_text(frame.text) + await self.push_frame(frame, direction) + elif isinstance(frame, InputAudioRawFrame): + await self._send_user_audio(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, InputImageRawFrame): + await self._send_user_video(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, InterruptionFrame): + await self._handle_interruption() + await self.push_frame(frame, direction) + elif isinstance(frame, UserStartedSpeakingFrame): + await self._handle_user_started_speaking(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, UserStoppedSpeakingFrame): + await self._handle_user_stopped_speaking(frame) + await self.push_frame(frame, direction) + elif isinstance(frame, BotStartedSpeakingFrame): + # Ignore this frame. Use the serverContent API message instead + await self.push_frame(frame, direction) + elif isinstance(frame, BotStoppedSpeakingFrame): + # ignore this frame. Use the serverContent.turnComplete API message + await self.push_frame(frame, direction) + elif isinstance(frame, LLMMessagesAppendFrame): + # NOTE: handling LLMMessagesAppendFrame here in the LLMService is + # unusual - typically this would be handled in the user context + # aggregator. Leaving this handling here so that user code that + # uses this frame *without* a user context aggregator still works + # (we have an example that does just that, actually). + await self._create_single_response(frame.messages) + elif isinstance(frame, LLMUpdateSettingsFrame): + await self._update_settings(frame.settings) + elif isinstance(frame, LLMSetToolsFrame): + await self._update_settings() + else: + await self.push_frame(frame, direction) + + async def _set_bot_is_speaking(self, speaking: bool): + if self._bot_is_speaking == speaking: + return + + self._bot_is_speaking = speaking + + if not self._bot_is_speaking and self._end_frame_pending_bot_turn_finished: + await self.queue_frame(self._end_frame_pending_bot_turn_finished) + self._end_frame_pending_bot_turn_finished = None + + async def _connect(self, session_resumption_handle: Optional[str] = None): + """Establish client connection to Gemini Live API.""" + if self._session: + # Here we assume that if we have a client, we are connected. We + # handle disconnections in the send/recv code paths. + return + + if session_resumption_handle: + logger.info( + f"Connecting to Gemini service with session_resumption_handle: {session_resumption_handle}" + ) + else: + logger.info("Connecting to Gemini service") + try: + # Assemble basic configuration + config = LiveConnectConfig( + generation_config=GenerationConfig( + frequency_penalty=self._settings["frequency_penalty"], + max_output_tokens=self._settings["max_tokens"], + presence_penalty=self._settings["presence_penalty"], + temperature=self._settings["temperature"], + top_k=self._settings["top_k"], + top_p=self._settings["top_p"], + response_modalities=[Modality(self._settings["modalities"].value)], + speech_config=SpeechConfig( + voice_config=VoiceConfig( + prebuilt_voice_config={"voice_name": self._voice_id} + ), + language_code=self._settings["language"], + ), + media_resolution=MediaResolution(self._settings["media_resolution"].value), + ), + input_audio_transcription=AudioTranscriptionConfig(), + output_audio_transcription=AudioTranscriptionConfig(), + session_resumption=SessionResumptionConfig(handle=session_resumption_handle), + ) + + # Add context window compression to configuration, if enabled + if self._settings.get("context_window_compression", {}).get("enabled", False): + compression_config = ContextWindowCompressionConfig() + + # Add sliding window (always true if compression is enabled) + compression_config.sliding_window = SlidingWindow() + + # Add trigger_tokens if specified + trigger_tokens = self._settings.get("context_window_compression", {}).get( + "trigger_tokens" + ) + if trigger_tokens is not None: + compression_config.trigger_tokens = trigger_tokens + + config.context_window_compression = compression_config + + # Add thinking configuration to configuration, if provided + if self._settings.get("thinking"): + config.thinking_config = self._settings["thinking"] + + # Add affective dialog setting, if provided + if self._settings.get("enable_affective_dialog", False): + config.enable_affective_dialog = self._settings["enable_affective_dialog"] + + # Add proactivity configuration to configuration, if provided + if self._settings.get("proactivity"): + config.proactivity = self._settings["proactivity"] + + # Add VAD configuration to configuration, if provided + if self._settings.get("vad"): + vad_config = AutomaticActivityDetection() + vad_params = self._settings["vad"] + has_vad_settings = False + + # Only add parameters that are explicitly set + if vad_params.disabled is not None: + vad_config.disabled = vad_params.disabled + has_vad_settings = True + + if vad_params.start_sensitivity: + vad_config.start_of_speech_sensitivity = vad_params.start_sensitivity + has_vad_settings = True + + if vad_params.end_sensitivity: + vad_config.end_of_speech_sensitivity = vad_params.end_sensitivity + has_vad_settings = True + + if vad_params.prefix_padding_ms is not None: + vad_config.prefix_padding_ms = vad_params.prefix_padding_ms + has_vad_settings = True + + if vad_params.silence_duration_ms is not None: + vad_config.silence_duration_ms = vad_params.silence_duration_ms + has_vad_settings = True + + # Only add automatic_activity_detection if we have VAD settings + if has_vad_settings: + config.realtime_input_config = RealtimeInputConfig( + automatic_activity_detection=vad_config + ) + + # Add system instruction to configuration, if provided + system_instruction = self._system_instruction or "" + if self._context and hasattr(self._context, "extract_system_instructions"): + system_instruction += "\n" + self._context.extract_system_instructions() + if system_instruction: + logger.debug(f"Setting system instruction: {system_instruction}") + config.system_instruction = system_instruction + + # Add tools to configuration, if provided + if self._tools: + logger.debug(f"Setting tools: {self._tools}") + config.tools = self.get_llm_adapter().from_standard_tools(self._tools) + + # Start the connection + self._connection_task = self.create_task(self._connection_task_handler(config=config)) + + except Exception as e: + await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}", fatal=True)) + + async def _connection_task_handler(self, config: LiveConnectConfig): + async with self._client.aio.live.connect(model=self._model_name, config=config) as session: + logger.info("Connected to Gemini service") + + # Mark connection start time + self._connection_start_time = time.time() + + await self._handle_session_ready(session) + + while True: + try: + turn = self._session.receive() + async for message in turn: + # Reset failure counter if connection has been stable + self._check_and_reset_failure_counter() + + if message.server_content and message.server_content.model_turn: + await self._handle_msg_model_turn(message) + elif ( + message.server_content + and message.server_content.turn_complete + and message.usage_metadata + ): + await self._handle_msg_turn_complete(message) + await self._handle_msg_usage_metadata(message) + elif message.server_content and message.server_content.input_transcription: + await self._handle_msg_input_transcription(message) + elif message.server_content and message.server_content.output_transcription: + await self._handle_msg_output_transcription(message) + elif message.server_content and message.server_content.grounding_metadata: + await self._handle_msg_grounding_metadata(message) + elif message.tool_call: + await self._handle_msg_tool_call(message) + elif message.session_resumption_update: + self._handle_msg_resumption_update(message) + except Exception as e: + if not self._disconnecting: + should_reconnect = await self._handle_connection_error(e) + if should_reconnect: + await self._reconnect() + return # Exit this connection handler, _reconnect will start a new one + break + + def _check_and_reset_failure_counter(self): + """Check if connection has been stable long enough to reset the failure counter. + + If the connection has been active for longer than the established threshold + and there are accumulated failures, reset the counter to 0. + """ + if ( + self._connection_start_time + and self._consecutive_failures > 0 + and time.time() - self._connection_start_time >= CONNECTION_ESTABLISHED_THRESHOLD + ): + logger.info( + f"Connection stable for {CONNECTION_ESTABLISHED_THRESHOLD}s, " + f"resetting failure counter from {self._consecutive_failures} to 0" + ) + self._consecutive_failures = 0 + + async def _handle_connection_error(self, error: Exception) -> bool: + """Handle a connection error and determine if reconnection should be attempted. + + Args: + error: The exception that caused the connection error. + + Returns: + True if reconnection should be attempted, False if a fatal error should be pushed. + """ + self._consecutive_failures += 1 + logger.warning( + f"Connection error (failure {self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES}): {error}" + ) + + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + logger.error( + f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, " + "treating as fatal error" + ) + await self.push_error( + ErrorFrame(error=f"{self} Error in receive loop: {error}", fatal=True) + ) + return False + else: + logger.info( + f"Attempting reconnection ({self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES})" + ) + return True + + async def _reconnect(self): + """Reconnect to Gemini Live API.""" + await self._disconnect() + await self._connect(session_resumption_handle=self._session_resumption_handle) + + async def _disconnect(self): + """Disconnect from Gemini Live API and clean up resources.""" + logger.info("Disconnecting from Gemini service") + try: + self._disconnecting = True + await self.stop_all_metrics() + if self._connection_task: + await self.cancel_task(self._connection_task, timeout=1.0) + self._connection_task = None + if self._session: + await self._session.close() + self._session = None + self._disconnecting = False + except Exception as e: + logger.error(f"{self} error disconnecting: {e}") + + async def _send_user_audio(self, frame): + """Send user audio frame to Gemini Live API.""" + if self._audio_input_paused or self._disconnecting or not self._session: + return + + # Send all audio to Gemini + try: + await self._session.send_realtime_input( + audio=Blob(data=frame.audio, mime_type=f"audio/pcm;rate={frame.sample_rate}") + ) + except Exception as e: + await self._handle_send_error(e) + + # Manage a buffer of audio to use for transcription + audio = frame.audio + if self._user_is_speaking: + self._user_audio_buffer.extend(audio) + else: + # Keep 1/2 second of audio in the buffer even when not speaking. + self._user_audio_buffer.extend(audio) + length = int((frame.sample_rate * frame.num_channels * 2) * 0.5) + self._user_audio_buffer = self._user_audio_buffer[-length:] + + async def _send_user_text(self, text: str): + """Send user text via Gemini Live API's realtime input stream. + + This method sends text through the realtimeInput stream (via TextInputMessage) + rather than the clientContent stream. This ensures text input is synchronized + with audio and video inputs, preventing temporal misalignment that can occur + when different modalities are processed through separate API pathways. + + For realtimeInput, turn completion is automatically inferred by the API based + on user activity, so no explicit turnComplete signal is needed. + + Args: + text: The text to send as user input. + """ + if self._disconnecting or not self._session: + return + + try: + await self._session.send_realtime_input(text=text) + except Exception as e: + await self._handle_send_error(e) + + async def _send_user_video(self, frame): + """Send user video frame to Gemini Live API.""" + if self._video_input_paused or self._disconnecting or not self._session: + return + + now = time.time() + if now - self._last_sent_time < 1: + return # Ignore if less than 1 second has passed + + self._last_sent_time = now # Update last sent time + logger.debug(f"Sending video frame to Gemini: {frame}") + + buffer = io.BytesIO() + Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") + data = base64.b64encode(buffer.getvalue()).decode("utf-8") + + try: + await self._session.send_realtime_input(video=Blob(data=data, mime_type="image/jpeg")) + except Exception as e: + await self._handle_send_error(e) + + async def _create_initial_response(self): + """Create initial response based on context history.""" + if self._disconnecting: + return + + if not self._session: + self._run_llm_when_session_ready = True + return + + messages = self._context.get_messages_for_initializing_history() + if not messages: + return + + logger.debug(f"Creating initial response: {messages}") + + await self.start_ttfb_metrics() + + try: + await self._session.send_client_content( + turns=messages, turn_complete=self._inference_on_context_initialization + ) + except Exception as e: + await self._handle_send_error(e) + + # If we're generating a response right away upon initializing + # conversation history, set a flag saying that we need a turn complete + # message when the user stops speaking. + if not self._inference_on_context_initialization: + self._needs_turn_complete_message = True + + async def _create_single_response(self, messages_list): + """Create a single response from a list of messages.""" + if self._disconnecting or not self._session: + return + + # Create a throwaway context just for the purpose of getting messages + # in the right format + context = GeminiLiveContext.upgrade(OpenAILLMContext(messages=messages_list)) + messages = context.get_messages_for_initializing_history() + + if not messages: + return + + logger.debug(f"Creating response: {messages}") + + await self.start_ttfb_metrics() + + try: + await self._session.send_client_content(turns=messages, turn_complete=True) + except Exception as e: + await self._handle_send_error(e) + + @traced_gemini_live(operation="llm_tool_result") + async def _tool_result(self, tool_result_message): + """Send tool result back to the API.""" + if self._disconnecting or not self._session: + return + + # For now we're shoving the name into the tool_call_id field, so this + # will work until we revisit that. + id = tool_result_message.get("tool_call_id") + name = tool_result_message.get("tool_call_name") + result = json.loads(tool_result_message.get("content") or "") + response = FunctionResponse(name=name, id=id, response=result) + + try: + await self._session.send_tool_response(function_responses=response) + except Exception as e: + await self._handle_send_error(e) + + @traced_gemini_live(operation="llm_setup") + async def _handle_session_ready(self, session: AsyncSession): + """Handle the session being ready.""" + self._session = session + # If we were just waititng for the session to be ready to run the LLM, + # do that now. + if self._run_llm_when_session_ready: + self._run_llm_when_session_ready = False + await self._create_initial_response() + + async def _handle_msg_model_turn(self, msg: LiveServerMessage): + """Handle the model turn message.""" + part = msg.server_content.model_turn.parts[0] + if not part: + return + + await self.stop_ttfb_metrics() + + # part.text is added when `modalities` is set to TEXT; otherwise, it's None + text = part.text + if text: + if not self._bot_text_buffer: + await self.push_frame(LLMFullResponseStartFrame()) + + self._bot_text_buffer += text + self._search_result_buffer += text # Also accumulate for grounding + await self.push_frame(LLMTextFrame(text=text)) + + # Check for grounding metadata in server content + if msg.server_content and msg.server_content.grounding_metadata: + self._accumulated_grounding_metadata = msg.server_content.grounding_metadata + + inline_data = part.inline_data + if not inline_data: + return + + # Check if mime type matches expected format + expected_mime_type = f"audio/pcm;rate={self._sample_rate}" + if inline_data.mime_type == expected_mime_type: + # Perfect match, continue processing + pass + elif inline_data.mime_type == "audio/pcm": + # Sample rate not provided in mime type, assume default + if not hasattr(self, "_sample_rate_warning_logged"): + logger.warning( + f"Sample rate not provided in mime type '{inline_data.mime_type}', assuming rate of {self._sample_rate}" + ) + self._sample_rate_warning_logged = True + else: + # Unrecognized format + logger.warning(f"Unrecognized server_content format {inline_data.mime_type}") + return + + audio = inline_data.data + if not audio: + return + + if not self._bot_is_speaking: + await self._set_bot_is_speaking(True) + await self.push_frame(TTSStartedFrame()) + await self.push_frame(LLMFullResponseStartFrame()) + + self._bot_audio_buffer.extend(audio) + frame = TTSAudioRawFrame( + audio=audio, + sample_rate=self._sample_rate, + num_channels=1, + ) + await self.push_frame(frame) + + @traced_gemini_live(operation="llm_tool_call") + async def _handle_msg_tool_call(self, message: LiveServerMessage): + """Handle tool call messages.""" + function_calls = message.tool_call.function_calls + if not function_calls: + return + if not self._context: + logger.error("Function calls are not supported without a context object.") + + function_calls_llm = [ + FunctionCallFromLLM( + context=self._context, + tool_call_id=( + # NOTE: when using Vertex AI we don't get server-provided + # tool call IDs here + f.id or str(uuid.uuid4()) + ), + function_name=f.name, + arguments=f.args, + ) + for f in function_calls + ] + + await self.run_function_calls(function_calls_llm) + + @traced_gemini_live(operation="llm_response") + async def _handle_msg_turn_complete(self, message: LiveServerMessage): + """Handle the turn complete message.""" + await self._set_bot_is_speaking(False) + text = self._bot_text_buffer + + # Trace the complete LLM response (this will be handled by the decorator) + # The decorator will extract the output text and usage metadata from the message + + self._bot_text_buffer = "" + self._llm_output_buffer = "" + + # Process grounding metadata if we have accumulated any + if self._accumulated_grounding_metadata: + await self._process_grounding_metadata( + self._accumulated_grounding_metadata, self._search_result_buffer + ) + + # Reset grounding tracking for next response + self._search_result_buffer = "" + self._accumulated_grounding_metadata = None + + # Only push the TTSStoppedFrame if the bot is outputting audio + # when text is found, modalities is set to TEXT and no audio + # is produced. + if not text: + await self.push_frame(TTSStoppedFrame()) + + await self.push_frame(LLMFullResponseEndFrame()) + + @traced_stt + async def _handle_user_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing.""" + pass + + async def _handle_msg_input_transcription(self, message: LiveServerMessage): + """Handle the input transcription message. + + Gemini Live sends user transcriptions in either single words or multi-word + phrases. As a result, we have to aggregate the input transcription. This handler + aggregates into sentences, splitting on the end of sentence markers. + """ + if not message.server_content.input_transcription: + return + + text = message.server_content.input_transcription.text + + if not text: + return + + # Strip leading space from sentence starts if buffer is empty + if text.startswith(" ") and not self._user_transcription_buffer: + text = text.lstrip() + + # Accumulate text in the buffer + self._user_transcription_buffer += text + + # Check for complete sentences + while True: + eos_end_marker = match_endofsentence(self._user_transcription_buffer) + if not eos_end_marker: + break + + # Extract the complete sentence + complete_sentence = self._user_transcription_buffer[:eos_end_marker] + # Keep the remainder for the next chunk + self._user_transcription_buffer = self._user_transcription_buffer[eos_end_marker:] + + # Send a TranscriptionFrame with the complete sentence + logger.debug(f"[Transcription:user] [{complete_sentence}]") + await self._handle_user_transcription( + complete_sentence, True, self._settings["language"] + ) + await self.push_frame( + TranscriptionFrame( + text=complete_sentence, + user_id="", + timestamp=time_now_iso8601(), + result=message, + ), + FrameDirection.UPSTREAM, + ) + + async def _handle_msg_output_transcription(self, message: LiveServerMessage): + """Handle the output transcription message.""" + if not message.server_content.output_transcription: + return + + # This is the output transcription text when modalities is set to AUDIO. + # In this case, we push LLMTextFrame and TTSTextFrame to be handled by the + # downstream assistant context aggregator. + text = message.server_content.output_transcription.text + + if not text: + return + + # Accumulate text for grounding as well + self._search_result_buffer += text + + # Check for grounding metadata in server content + if message.server_content and message.server_content.grounding_metadata: + self._accumulated_grounding_metadata = message.server_content.grounding_metadata + # Collect text for tracing + self._llm_output_buffer += text + + await self.push_frame(LLMTextFrame(text=text)) + await self.push_frame(TTSTextFrame(text=text)) + + async def _handle_msg_grounding_metadata(self, message: LiveServerMessage): + """Handle dedicated grounding metadata messages.""" + if message.server_content and message.server_content.grounding_metadata: + grounding_metadata = message.server_content.grounding_metadata + # Process the grounding metadata immediately + await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer) + + async def _process_grounding_metadata( + self, grounding_metadata: GroundingMetadata, search_result: str = "" + ): + """Process grounding metadata and emit LLMSearchResponseFrame.""" + if not grounding_metadata: + return + + # Extract rendered content for search suggestions + rendered_content = None + if ( + grounding_metadata.search_entry_point + and grounding_metadata.search_entry_point.rendered_content + ): + rendered_content = grounding_metadata.search_entry_point.rendered_content + + # Convert grounding chunks and supports to LLMSearchOrigin format + origins = [] + + if grounding_metadata.grounding_chunks and grounding_metadata.grounding_supports: + # Create a mapping of chunk indices to origins + chunk_to_origin: Dict[int, LLMSearchOrigin] = {} + + for index, chunk in enumerate(grounding_metadata.grounding_chunks): + if chunk.web: + origin = LLMSearchOrigin( + site_uri=chunk.web.uri, site_title=chunk.web.title, results=[] + ) + chunk_to_origin[index] = origin + origins.append(origin) + + # Add grounding support results to the appropriate origins + for support in grounding_metadata.grounding_supports: + if support.segment and support.grounding_chunk_indices: + text = support.segment.text or "" + confidence_scores = support.confidence_scores or [] + + # Add this result to all origins referenced by this support + for chunk_index in support.grounding_chunk_indices: + if chunk_index in chunk_to_origin: + result = LLMSearchResult(text=text, confidence=confidence_scores) + chunk_to_origin[chunk_index].results.append(result) + + # Create and push the search response frame + search_frame = LLMSearchResponseFrame( + search_result=search_result, origins=origins, rendered_content=rendered_content + ) + + await self.push_frame(search_frame) + + async def _handle_msg_usage_metadata(self, message: LiveServerMessage): + """Handle the usage metadata message.""" + if not message.usage_metadata: + return + + usage = message.usage_metadata + + # Ensure we have valid integers for all token counts + prompt_tokens = usage.prompt_token_count or 0 + completion_tokens = usage.response_token_count or 0 + total_tokens = usage.total_token_count or (prompt_tokens + completion_tokens) + + tokens = LLMTokenUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + + await self.start_llm_usage_metrics(tokens) + + def _handle_msg_resumption_update(self, message: LiveServerMessage): + update = message.session_resumption_update + if update.resumable and update.new_handle: + self._session_resumption_handle = update.new_handle + + async def _handle_send_error(self, error: Exception): + # In server-to-server contexts, a WebSocket error should be quite rare. + # Given how hard it is to recover from a send-side error with proper + # state management, and that exponential backoff for retries can have + # cost/stability implications for a service cluster, let's just treat a + # send-side error as fatal. + if not self._disconnecting: + await self.push_error(ErrorFrame(error=f"{self} Send error: {error}", fatal=True)) + + def create_context_aggregator( + self, + context: OpenAILLMContext, + *, + user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), + assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), + ) -> GeminiLiveContextAggregatorPair: + """Create an instance of GeminiLiveContextAggregatorPair from an OpenAILLMContext. + + Constructor keyword arguments for both the user and assistant aggregators can be provided. + + Args: + context: The LLM context to use. + user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams(). + assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams(). + + Returns: + GeminiLiveContextAggregatorPair: A pair of context + aggregators, one for the user and one for the assistant, + encapsulated in an GeminiLiveContextAggregatorPair. + """ + context.set_llm_adapter(self.get_llm_adapter()) + + GeminiLiveContext.upgrade(context) + user = GeminiLiveUserContextAggregator(context, params=user_params) + + assistant_params.expect_stripped_words = False + assistant = GeminiLiveAssistantContextAggregator(context, params=assistant_params) + return GeminiLiveContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/google/gemini_live/llm_vertex.py b/src/pipecat/services/google/gemini_live/llm_vertex.py new file mode 100644 index 000000000..a38154755 --- /dev/null +++ b/src/pipecat/services/google/gemini_live/llm_vertex.py @@ -0,0 +1,184 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Service for accessing Gemini Live via Google Vertex AI. + +This module provides integration with Google's Gemini Live model via +Vertex AI, supporting both text and audio modalities with voice transcription, +streaming responses, and tool usage. +""" + +import json +from typing import List, Optional, Union + +from loguru import logger + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.services.google.gemini_live.llm import ( + GeminiLiveLLMService, + HttpOptions, + InputParams, +) + +try: + from google.auth import default + from google.auth.exceptions import GoogleAuthError + from google.auth.transport.requests import Request + from google.genai import Client + from google.oauth2 import service_account + +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Google Vertex AI, you need to `pip install pipecat-ai[google]`.") + raise Exception(f"Missing module: {e}") + + +class GeminiLiveVertexLLMService(GeminiLiveLLMService): + """Provides access to Google's Gemini Live model via Vertex AI. + + This service enables real-time conversations with Gemini, supporting both + text and audio modalities. It handles voice transcription, streaming audio + responses, and tool usage. + """ + + def __init__( + self, + *, + credentials: Optional[str] = None, + credentials_path: Optional[str] = None, + location: str, + project_id: str, + model="google/gemini-2.0-flash-live-preview-04-09", + voice_id: str = "Charon", + start_audio_paused: bool = False, + start_video_paused: bool = False, + system_instruction: Optional[str] = None, + tools: Optional[Union[List[dict], ToolsSchema]] = None, + params: Optional[InputParams] = None, + inference_on_context_initialization: bool = True, + file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files", + http_options: Optional[HttpOptions] = None, + **kwargs, + ): + """Initialize the service for accessing Gemini Live via Google Vertex AI. + + Args: + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. + location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + project_id: Google Cloud project ID. + model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-preview-04-09". + voice_id: TTS voice identifier. Defaults to "Charon". + start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_video_paused: Whether to start with video input paused. Defaults to False. + system_instruction: System prompt for the model. Defaults to None. + tools: Tools/functions available to the model. Defaults to None. + params: Configuration parameters for the model along with Vertex AI + location and project ID. + inference_on_context_initialization: Whether to generate a response when context + is first set. Defaults to True. + file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint. + http_options: HTTP options for the client. + **kwargs: Additional arguments passed to parent GeminiLiveLLMService. + """ + # Check if user incorrectly passed api_key, which is used by parent + # class but not here. + if "api_key" in kwargs: + logger.error( + "GeminiLiveVertexLLMService does not accept 'api_key' parameter. " + "Use 'credentials' or 'credentials_path' instead for Vertex AI authentication." + ) + raise ValueError( + "Invalid parameter 'api_key'. Use 'credentials' or 'credentials_path' for Vertex AI authentication." + ) + + # These need to be set before calling super().__init__() because + # super().__init__() invokes create_client(), which needs these. + self._credentials = self._get_credentials(credentials, credentials_path) + self._project_id = project_id + self._location = location + + # Call parent constructor with the obtained API key + super().__init__( + # api_key is required by parent class, but actually not used with + # Vertex + api_key="dummy", + model=model, + voice_id=voice_id, + start_audio_paused=start_audio_paused, + start_video_paused=start_video_paused, + system_instruction=system_instruction, + tools=tools, + params=params, + inference_on_context_initialization=inference_on_context_initialization, + file_api_base_url=file_api_base_url, + http_options=http_options, + **kwargs, + ) + + def create_client(self): + """Create the Gemini client instance.""" + self._client = Client( + vertexai=True, + credentials=self._credentials, + project=self._project_id, + location=self._location, + ) + + @property + def file_api(self): + """Gemini File API is not supported with Vertex AI.""" + raise NotImplementedError( + "When using Vertex AI, the recommended approach is to use Google Cloud Storage for file handling. The Gemini File API is not directly supported in this context." + ) + + @staticmethod + def _get_credentials(credentials: Optional[str], credentials_path: Optional[str]) -> str: + """Retrieve Credentials using Google service account credentials JSON. + + Supports multiple authentication methods: + 1. Direct JSON credentials string + 2. Path to service account JSON file + 3. Default application credentials (ADC) + + Args: + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. + + Returns: + OAuth token for API authentication. + + Raises: + ValueError: If no valid credentials are provided or found. + """ + creds: Optional[service_account.Credentials] = None + + if credentials: + # Parse and load credentials from JSON string + creds = service_account.Credentials.from_service_account_info( + json.loads(credentials), + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + elif credentials_path: + # Load credentials from JSON file + creds = service_account.Credentials.from_service_account_file( + credentials_path, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + else: + try: + creds, project_id = default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + except GoogleAuthError: + pass + + if not creds: + raise ValueError("No valid credentials provided.") + + creds.refresh(Request()) # Ensure token is up-to-date, lifetime is 1 hour. + + return creds diff --git a/src/pipecat/services/google/llm_openai.py b/src/pipecat/services/google/llm_openai.py index 81d124cb7..fde62154e 100644 --- a/src/pipecat/services/google/llm_openai.py +++ b/src/pipecat/services/google/llm_openai.py @@ -94,9 +94,9 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): async for chunk in chunk_stream: if chunk.usage: tokens = LLMTokenUsage( - prompt_tokens=chunk.usage.prompt_tokens, - completion_tokens=chunk.usage.completion_tokens, - total_tokens=chunk.usage.total_tokens, + prompt_tokens=chunk.usage.prompt_tokens or 0, + completion_tokens=chunk.usage.completion_tokens or 0, + total_tokens=chunk.usage.total_tokens or 0, ) await self.start_llm_usage_metrics(tokens) diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py index ee66c1f20..49adb2e9b 100644 --- a/src/pipecat/services/google/llm_vertex.py +++ b/src/pipecat/services/google/llm_vertex.py @@ -53,12 +53,44 @@ class GoogleVertexLLMService(OpenAILLMService): Parameters: location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + + .. deprecated:: 0.0.90 + Use `location` as a direct argument to + `GoogleVertexLLMService.__init__()` instead. + project_id: Google Cloud project ID. + + .. deprecated:: 0.0.90 + Use `project_id` as a direct argument to + `GoogleVertexLLMService.__init__()` instead. """ # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations - location: str = "us-east4" - project_id: str + location: Optional[str] = None + project_id: Optional[str] = None + + def __init__(self, **kwargs): + """Initializes the InputParams.""" + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + if "location" in kwargs and kwargs["location"] is not None: + warnings.warn( + "GoogleVertexLLMService.InputParams.location is deprecated. " + "Please provide 'location' as a direct argument to GoogleVertexLLMService.__init__() instead.", + DeprecationWarning, + stacklevel=2, + ) + + if "project_id" in kwargs and kwargs["project_id"] is not None: + warnings.warn( + "GoogleVertexLLMService.InputParams.project_id is deprecated. " + "Please provide 'project_id' as a direct argument to GoogleVertexLLMService.__init__() instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(**kwargs) def __init__( self, @@ -66,7 +98,8 @@ class GoogleVertexLLMService(OpenAILLMService): credentials: Optional[str] = None, credentials_path: Optional[str] = None, model: str = "google/gemini-2.0-flash-001", - params: Optional[InputParams] = None, + location: Optional[str] = None, + project_id: Optional[str] = None, **kwargs, ): """Initializes the VertexLLMService. @@ -75,33 +108,60 @@ class GoogleVertexLLMService(OpenAILLMService): credentials: JSON string of service account credentials. credentials_path: Path to the service account JSON file. model: Model identifier (e.g., "google/gemini-2.0-flash-001"). - params: Vertex AI input parameters including location and project. + location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + project_id: Google Cloud project ID. **kwargs: Additional arguments passed to OpenAILLMService. """ - params = params or OpenAILLMService.InputParams() - base_url = self._get_base_url(params) + # Handle deprecated InputParams fields + if "params" in kwargs and isinstance(kwargs["params"], GoogleVertexLLMService.InputParams): + params = kwargs["params"] + # Extract location and project_id from params if not provided + # directly, for backward compatibility + if project_id is None: + project_id = params.project_id + if location is None: + location = params.location + # Convert to base InputParams + params = OpenAILLMService.InputParams( + **params.model_dump(exclude={"location", "project_id"}, exclude_unset=True) + ) + kwargs["params"] = params + + # Validate project_id and location parameters + # NOTE: once we remove Vertex-spcific InputParams class, we can update + # __init__() signature as follows: + # - location: str = "us-east4", + # - project_id: str, + # But for now, we need them as-is to maintain proper backward + # compatibility. + if project_id is None: + raise ValueError("project_id is required") + if location is None: + # If location is not provided, default to "us-east4". + # Note: this is legacy behavior; ideally location would be + # required. + logger.warning("location is not provided. Defaulting to 'us-east4'.") + location = "us-east4" # Default location if not provided + + base_url = self._get_base_url(location, project_id) self._api_key = self._get_api_token(credentials, credentials_path) super().__init__( api_key=self._api_key, base_url=base_url, model=model, - params=params, **kwargs, ) @staticmethod - def _get_base_url(params: InputParams) -> str: + def _get_base_url(location: str, project_id: str) -> str: """Construct the base URL for Vertex AI API.""" # Determine the correct API host based on location - if params.location == "global": + if location == "global": api_host = "aiplatform.googleapis.com" else: - api_host = f"{params.location}-aiplatform.googleapis.com" - return ( - f"https://{api_host}/v1/" - f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" - ) + api_host = f"{location}-aiplatform.googleapis.com" + return f"https://{api_host}/v1/projects/{project_id}/locations/{location}/endpoints/openapi" @staticmethod def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 31ae597f7..b9e56f55b 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -730,6 +730,8 @@ class GoogleSTTService(STTService): self._request_queue = asyncio.Queue() self._streaming_task = self.create_task(self._stream_audio()) + await self._call_event_handler("on_connected") + async def _disconnect(self): """Clean up streaming recognition resources.""" if self._streaming_task: @@ -737,6 +739,8 @@ class GoogleSTTService(STTService): await self.cancel_task(self._streaming_task) self._streaming_task = None + await self._call_event_handler("on_disconnected") + async def _request_generator(self): """Generates requests for the streaming recognize method.""" recognizer_path = f"projects/{self._project_id}/locations/{self._location}/recognizers/_" diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index 0629a8909..2701c5d05 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -42,7 +42,7 @@ class HumeTTSService(TTSService): """Hume Octave Text-to-Speech service. Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint - using the Python SDK and emits `TTSAudioRawFrame`s suitable for Pipecat transports. + using the Python SDK and emits ``TTSAudioRawFrame`` frames suitable for Pipecat transports. Supported features: @@ -78,7 +78,7 @@ class HumeTTSService(TTSService): Args: api_key: Hume API key. If omitted, reads the ``HUME_API_KEY`` environment variable. - voice_id: ID of the voice to use (ID-only; names are not supported here). + voice_id: ID of the voice to use. Only voice IDs are supported; voice names are not. params: Optional synthesis controls (acting instructions, speed, trailing silence). sample_rate: Output sample rate for emitted PCM frames. Defaults to 48_000 (Hume). **kwargs: Additional arguments passed to the parent class. diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index a602789fd..9f9fef5fc 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -222,6 +222,7 @@ class LmntTTSService(InterruptibleTTSService): # Send initialization message await self._websocket.send(json.dumps(init_msg)) + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -243,6 +244,7 @@ class LmntTTSService(InterruptibleTTSService): finally: self._started = False self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): """Get the WebSocket connection if available.""" diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 46d805086..6ccdfe17f 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -293,6 +293,8 @@ class NeuphonicTTSService(InterruptibleTTSService): headers = {"x-api-key": self._api_key} self._websocket = await websocket_connect(url, additional_headers=headers) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -311,6 +313,7 @@ class NeuphonicTTSService(InterruptibleTTSService): finally: self._started = False self._websocket = None + await self._call_event_handler("on_disconnected") async def _receive_messages(self): """Receive and process messages from Neuphonic WebSocket.""" diff --git a/src/pipecat/services/openai/__init__.py b/src/pipecat/services/openai/__init__.py index 4decac126..d913c5ee0 100644 --- a/src/pipecat/services/openai/__init__.py +++ b/src/pipecat/services/openai/__init__.py @@ -10,6 +10,7 @@ from pipecat.services import DeprecatedModuleProxy from .image import * from .llm import * +from .realtime import * from .stt import * from .tts import * diff --git a/src/pipecat/services/openai/realtime/__init__.py b/src/pipecat/services/openai/realtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py new file mode 100644 index 000000000..cb1c0a9f5 --- /dev/null +++ b/src/pipecat/services/openai/realtime/context.py @@ -0,0 +1,272 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""OpenAI Realtime LLM context and aggregator implementations.""" + +import copy +import json + +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + FunctionCallResultFrame, + InterimTranscriptionFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMTextFrame, + TranscriptionFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) + +from . import events +from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame + + +class OpenAIRealtimeLLMContext(OpenAILLMContext): + """OpenAI Realtime LLM context with session management and message conversion. + + Extends the standard OpenAI LLM context to support real-time session properties, + instruction management, and conversion between standard message formats and + realtime conversation items. + """ + + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize the OpenAIRealtimeLLMContext. + + Args: + messages: Initial conversation messages. Defaults to None. + tools: Available function tools. Defaults to None. + **kwargs: Additional arguments passed to parent OpenAILLMContext. + """ + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self): + self.llm_needs_settings_update = True + self.llm_needs_initial_messages = True + self._session_instructions = "" + + return + + @staticmethod + def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": + """Upgrade a standard OpenAI LLM context to a realtime context. + + Args: + obj: The OpenAILLMContext instance to upgrade. + + Returns: + The upgraded OpenAIRealtimeLLMContext instance. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): + obj.__class__ = OpenAIRealtimeLLMContext + obj.__setup_local() + return obj + + # todo + # - finish implementing all frames + + def from_standard_message(self, message): + """Convert a standard message format to a realtime conversation item. + + Args: + message: The standard message dictionary to convert. + + Returns: + A ConversationItem instance for the realtime API. + """ + if message.get("role") == "user": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + return events.ConversationItem( + role="user", + type="message", + content=[events.ItemContent(type="input_text", text=content)], + ) + if message.get("role") == "assistant" and message.get("tool_calls"): + tc = message.get("tool_calls")[0] + return events.ConversationItem( + type="function_call", + call_id=tc["id"], + name=tc["function"]["name"], + arguments=tc["function"]["arguments"], + ) + logger.error(f"Unhandled message type in from_standard_message: {message}") + + def get_messages_for_initializing_history(self): + """Get conversation items for initializing the realtime session history. + + Converts the context's messages to a format suitable for the realtime API, + handling system instructions and conversation history packaging. + + Returns: + List of conversation items for session initialization. + """ + # We can't load a long conversation history into the openai realtime api yet. (The API/model + # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So + # our general strategy until this is fixed is just to put everything into a first "user" + # message as a single input. + if not self.messages: + return [] + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into session + # "instructions" + if messages[0].get("role") == "system": + self.llm_needs_settings_update = True + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + self._session_instructions = content + elif isinstance(content, list): + self._session_instructions = content[0].get("text") + if not messages: + return [] + + # If we have just a single "user" item, we can just send it normally + if len(messages) == 1 and messages[0].get("role") == "user": + return [self.from_standard_message(messages[0])] + + # Otherwise, let's pack everything into a single "user" message with a bit of + # explanation for the LLM + intro_text = """ + This is a previously saved conversation. Please treat this conversation history as a + starting point for the current conversation.""" + + trailing_text = """ + This is the end of the previously saved conversation. Please continue the conversation + from here. If the last message is a user instruction or question, act on that instruction + or answer the question. If the last message is an assistant response, simple say that you + are ready to continue the conversation.""" + + return [ + { + "role": "user", + "type": "message", + "content": [ + { + "type": "input_text", + "text": "\n\n".join( + [intro_text, json.dumps(messages, indent=2), trailing_text] + ), + } + ], + } + ] + + def add_user_content_item_as_message(self, item): + """Add a user content item as a standard message to the context. + + Args: + item: The conversation item to add as a user message. + """ + message = { + "role": "user", + "content": [{"type": "text", "text": item.content[0].transcript}], + } + self.add_message(message) + + +class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): + """User context aggregator for OpenAI Realtime API. + + Handles user input frames and generates appropriate context updates + for the realtime conversation, including message updates and tool settings. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + + async def process_frame( + self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM + ): + """Process incoming frames and handle realtime-specific frame types. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + # Parent does not push LLMMessagesUpdateFrame. This ensures that in a typical pipeline, + # messages are only processed by the user context aggregator, which is generally what we want. But + # we also need to send new messages over the websocket, so the openai realtime API has them + # in its context. + if isinstance(frame, LLMMessagesUpdateFrame): + await self.push_frame(RealtimeMessagesUpdateFrame(context=self._context)) + + # Parent also doesn't push the LLMSetToolsFrame. + if isinstance(frame, LLMSetToolsFrame): + await self.push_frame(frame, direction) + + async def push_aggregation(self): + """Push user input aggregation. + + Currently ignores all user input coming into the pipeline as realtime + audio input is handled directly by the service. + """ + # for the moment, ignore all user input coming into the pipeline. + # todo: think about whether/how to fix this to allow for text input from + # upstream (transport/transcription, or other sources) + pass + + +class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Assistant context aggregator for OpenAI Realtime API. + + Handles assistant output frames from the realtime service, filtering + out duplicate text frames and managing function call results. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. + # OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames, + # so we need to ignore pushing those as well, as they're also TextFrames. + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process assistant frames, filtering out duplicate text content. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)): + await super().process_frame(frame, direction) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call result and notify the realtime service. + + Args: + frame: The function call result frame to handle. + """ + await super().handle_function_call_result(frame) + + # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, + # so we didn't have a chance to add the result to the openai realtime api context. Let's push a + # special frame to do that. + await self.push_frame( + RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py new file mode 100644 index 000000000..200e59d68 --- /dev/null +++ b/src/pipecat/services/openai/realtime/events.py @@ -0,0 +1,1106 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Event models and data structures for OpenAI Realtime API communication.""" + +import json +import uuid +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field + +# +# session properties +# + + +class AudioFormat(BaseModel): + """Base class for audio format configuration.""" + + type: str + + +class PCMAudioFormat(AudioFormat): + """PCM audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcm". + rate: Sample rate, always 24000 for PCM. + """ + + type: Literal["audio/pcm"] = "audio/pcm" + rate: Literal[24000] = 24000 + + +class PCMUAudioFormat(AudioFormat): + """PCMU (G.711 μ-law) audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcmu". + """ + + type: Literal["audio/pcmu"] = "audio/pcmu" + + +class PCMAAudioFormat(AudioFormat): + """PCMA (G.711 A-law) audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcma". + """ + + type: Literal["audio/pcma"] = "audio/pcma" + + +class InputAudioTranscription(BaseModel): + """Configuration for audio transcription settings.""" + + model: str = "gpt-4o-transcribe" + language: Optional[str] + prompt: Optional[str] + + def __init__( + self, + model: Optional[str] = "gpt-4o-transcribe", + language: Optional[str] = None, + prompt: Optional[str] = None, + ): + """Initialize InputAudioTranscription. + + Args: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ + super().__init__(model=model, language=language, prompt=prompt) + + +class TurnDetection(BaseModel): + """Server-side voice activity detection configuration. + + Parameters: + type: Detection type, must be "server_vad". + threshold: Voice activity detection threshold (0.0-1.0). Defaults to 0.5. + prefix_padding_ms: Padding before speech starts in milliseconds. Defaults to 300. + silence_duration_ms: Silence duration to detect speech end in milliseconds. Defaults to 500. + """ + + type: Optional[Literal["server_vad"]] = "server_vad" + threshold: Optional[float] = 0.5 + prefix_padding_ms: Optional[int] = 300 + silence_duration_ms: Optional[int] = 500 + + +class SemanticTurnDetection(BaseModel): + """Semantic-based turn detection configuration. + + Parameters: + type: Detection type, must be "semantic_vad". + eagerness: Turn detection eagerness level. Can be "low", "medium", "high", or "auto". + create_response: Whether to automatically create responses on turn detection. + interrupt_response: Whether to interrupt ongoing responses on turn detection. + """ + + type: Optional[Literal["semantic_vad"]] = "semantic_vad" + eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None + create_response: Optional[bool] = None + interrupt_response: Optional[bool] = None + + +class InputAudioNoiseReduction(BaseModel): + """Input audio noise reduction configuration. + + Parameters: + type: Noise reduction type for different microphone scenarios. + """ + + type: Optional[Literal["near_field", "far_field"]] + + +class AudioInput(BaseModel): + """Audio input configuration. + + Parameters: + format: The format of the input audio. + transcription: Configuration for input audio transcription. + noise_reduction: Configuration for input audio noise reduction. + turn_detection: Configuration for turn detection, or False to disable. + """ + + format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None + transcription: Optional[InputAudioTranscription] = None + noise_reduction: Optional[InputAudioNoiseReduction] = None + turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = None + + +class AudioOutput(BaseModel): + """Audio output configuration. + + Parameters: + format: The format of the output audio. + voice: The voice the model uses to respond. + speed: The speed of the model's spoken response. + """ + + format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None + voice: Optional[str] = None + speed: Optional[float] = None + + +class AudioConfiguration(BaseModel): + """Audio configuration for input and output. + + Parameters: + input: Configuration for input audio. + output: Configuration for output audio. + """ + + input: Optional[AudioInput] = None + output: Optional[AudioOutput] = None + + +class SessionProperties(BaseModel): + """Configuration properties for an OpenAI Realtime session. + + Parameters: + type: The type of session, always "realtime". + object: Object type identifier, always "realtime.session". + id: Unique identifier for the session. + model: The Realtime model used for this session. + output_modalities: The set of modalities the model can respond with. + instructions: System instructions for the assistant. + audio: Configuration for input and output audio. + tools: Available function tools for the assistant. + tool_choice: Tool usage strategy ("auto", "none", or "required"). + max_output_tokens: Maximum tokens in response or "inf" for unlimited. + tracing: Configuration options for tracing. + prompt: Reference to a prompt template and its variables. + expires_at: Session expiration timestamp. + include: Additional fields to include in server outputs. + """ + + type: Optional[Literal["realtime"]] = "realtime" + object: Optional[Literal["realtime.session"]] = None + id: Optional[str] = None + model: Optional[str] = None + output_modalities: Optional[List[Literal["text", "audio"]]] = None + instructions: Optional[str] = None + audio: Optional[AudioConfiguration] = None + tools: Optional[List[Dict]] = None + tool_choice: Optional[Literal["auto", "none", "required"]] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + tracing: Optional[Union[Literal["auto"], Dict]] = None + prompt: Optional[Dict] = None + expires_at: Optional[int] = None + include: Optional[List[str]] = None + + +# +# context +# + + +class ItemContent(BaseModel): + """Content within a conversation item. + + Parameters: + type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). + text: Text content for text-based items. + audio: Base64-encoded audio data for audio items. + transcript: Transcribed text for audio items. + """ + + type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] + text: Optional[str] = None + audio: Optional[str] = None # base64-encoded audio + transcript: Optional[str] = None + + +class ConversationItem(BaseModel): + """A conversation item in the realtime session. + + Parameters: + id: Unique identifier for the item, auto-generated if not provided. + object: Object type identifier for the realtime API. + type: Item type (message, function_call, or function_call_output). + status: Current status of the item. + role: Speaker role for message items (user, assistant, or system). + content: Content list for message items. + call_id: Function call identifier for function_call items. + name: Function name for function_call items. + arguments: Function arguments as JSON string for function_call items. + output: Function output as JSON string for function_call_output items. + """ + + id: str = Field(default_factory=lambda: str(uuid.uuid4().hex)) + object: Optional[Literal["realtime.item"]] = None + type: Literal["message", "function_call", "function_call_output"] + status: Optional[Literal["completed", "in_progress", "incomplete"]] = None + # role and content are present for message items + role: Optional[Literal["user", "assistant", "system"]] = None + content: Optional[List[ItemContent]] = None + # these four fields are present for function_call items + call_id: Optional[str] = None + name: Optional[str] = None + arguments: Optional[str] = None + output: Optional[str] = None + + +class RealtimeConversation(BaseModel): + """A realtime conversation session. + + Parameters: + id: Unique identifier for the conversation. + object: Object type identifier, always "realtime.conversation". + """ + + id: str + object: Literal["realtime.conversation"] + + +class ResponseProperties(BaseModel): + """Properties for configuring assistant responses. + + Parameters: + output_modalities: Output modalities for the response. Must be either ["text"] or ["audio"]. Defaults to ["audio"]. + instructions: Specific instructions for this response. + audio: Audio configuration for this response. + tools: Available tools for this response. + tool_choice: Tool usage strategy for this response. + temperature: Sampling temperature for this response. + max_output_tokens: Maximum tokens for this response. + """ + + output_modalities: Optional[List[Literal["text", "audio"]]] = ["audio"] + instructions: Optional[str] = None + audio: Optional[AudioConfiguration] = None + tools: Optional[List[Dict]] = None + tool_choice: Optional[Literal["auto", "none", "required"]] = None + temperature: Optional[float] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + + +# +# error class +# +class RealtimeError(BaseModel): + """Error information from the realtime API. + + Parameters: + type: Error type identifier. + code: Specific error code. + message: Human-readable error message. + param: Parameter name that caused the error, if applicable. + event_id: Event ID associated with the error, if applicable. + """ + + type: str + code: Optional[str] = "" + message: str + param: Optional[str] = None + event_id: Optional[str] = None + + +# +# client events +# + + +class ClientEvent(BaseModel): + """Base class for client events sent to the realtime API. + + Parameters: + event_id: Unique identifier for the event, auto-generated if not provided. + """ + + event_id: str = Field(default_factory=lambda: str(uuid.uuid4())) + + +class SessionUpdateEvent(ClientEvent): + """Event to update session properties. + + Parameters: + type: Event type, always "session.update". + session: Updated session properties. + """ + + type: Literal["session.update"] = "session.update" + session: SessionProperties + + def model_dump(self, *args, **kwargs) -> Dict[str, Any]: + """Serialize the event to a dictionary. + + Handles special serialization for turn_detection where False becomes null. + + Args: + *args: Positional arguments passed to parent model_dump. + **kwargs: Keyword arguments passed to parent model_dump. + + Returns: + Dictionary representation of the event. + """ + dump = super().model_dump(*args, **kwargs) + + # Handle turn_detection in audio.input so that False becomes null + if "audio" in dump["session"] and dump["session"]["audio"]: + if "input" in dump["session"]["audio"] and dump["session"]["audio"]["input"]: + if "turn_detection" in dump["session"]["audio"]["input"]: + if dump["session"]["audio"]["input"]["turn_detection"] is False: + dump["session"]["audio"]["input"]["turn_detection"] = None + + return dump + + +class InputAudioBufferAppendEvent(ClientEvent): + """Event to append audio data to the input buffer. + + Parameters: + type: Event type, always "input_audio_buffer.append". + audio: Base64-encoded audio data to append. + """ + + type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" + audio: str # base64-encoded audio + + +class InputAudioBufferCommitEvent(ClientEvent): + """Event to commit the current input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.commit". + """ + + type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" + + +class InputAudioBufferClearEvent(ClientEvent): + """Event to clear the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.clear". + """ + + type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" + + +class ConversationItemCreateEvent(ClientEvent): + """Event to create a new conversation item. + + Parameters: + type: Event type, always "conversation.item.create". + previous_item_id: ID of the item to insert after, if any. + item: The conversation item to create. + """ + + type: Literal["conversation.item.create"] = "conversation.item.create" + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemTruncateEvent(ClientEvent): + """Event to truncate a conversation item's audio content. + + Parameters: + type: Event type, always "conversation.item.truncate". + item_id: ID of the item to truncate. + content_index: Index of the content to truncate within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + + type: Literal["conversation.item.truncate"] = "conversation.item.truncate" + item_id: str + content_index: int + audio_end_ms: int + + +class ConversationItemDeleteEvent(ClientEvent): + """Event to delete a conversation item. + + Parameters: + type: Event type, always "conversation.item.delete". + item_id: ID of the item to delete. + """ + + type: Literal["conversation.item.delete"] = "conversation.item.delete" + item_id: str + + +class ConversationItemRetrieveEvent(ClientEvent): + """Event to retrieve a conversation item by ID. + + Parameters: + type: Event type, always "conversation.item.retrieve". + item_id: ID of the item to retrieve. + """ + + type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" + item_id: str + + +class ResponseCreateEvent(ClientEvent): + """Event to create a new assistant response. + + Parameters: + type: Event type, always "response.create". + response: Optional response configuration properties. + """ + + type: Literal["response.create"] = "response.create" + response: Optional[ResponseProperties] = None + + +class ResponseCancelEvent(ClientEvent): + """Event to cancel the current assistant response. + + Parameters: + type: Event type, always "response.cancel". + """ + + type: Literal["response.cancel"] = "response.cancel" + + +# +# server events +# + + +class ServerEvent(BaseModel): + """Base class for server events received from the realtime API. + + Parameters: + event_id: Unique identifier for the event. + type: Type of the server event. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + event_id: str + type: str + + +class SessionCreatedEvent(ServerEvent): + """Event indicating a session has been created. + + Parameters: + type: Event type, always "session.created". + session: The created session properties. + """ + + type: Literal["session.created"] + session: SessionProperties + + +class SessionUpdatedEvent(ServerEvent): + """Event indicating a session has been updated. + + Parameters: + type: Event type, always "session.updated". + session: The updated session properties. + """ + + type: Literal["session.updated"] + session: SessionProperties + + +class ConversationCreated(ServerEvent): + """Event indicating a conversation has been created. + + Parameters: + type: Event type, always "conversation.created". + conversation: The created conversation. + """ + + type: Literal["conversation.created"] + conversation: RealtimeConversation + + +class ConversationItemAdded(ServerEvent): + """Event indicating a conversation item has been added. + + Parameters: + type: Event type, always "conversation.item.added". + previous_item_id: ID of the previous item, if any. + item: The added conversation item. + """ + + type: Literal["conversation.item.added"] + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemDone(ServerEvent): + """Event indicating a conversation item is done processing. + + Parameters: + type: Event type, always "conversation.item.done". + previous_item_id: ID of the previous item, if any. + item: The completed conversation item. + """ + + type: Literal["conversation.item.done"] + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemInputAudioTranscriptionDelta(ServerEvent): + """Event containing incremental input audio transcription. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.delta". + item_id: ID of the conversation item being transcribed. + content_index: Index of the content within the item. + delta: Incremental transcription text. + """ + + type: Literal["conversation.item.input_audio_transcription.delta"] + item_id: str + content_index: int + delta: str + + +class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): + """Event indicating input audio transcription is complete. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.completed". + item_id: ID of the conversation item that was transcribed. + content_index: Index of the content within the item. + transcript: Complete transcription text. + """ + + type: Literal["conversation.item.input_audio_transcription.completed"] + item_id: str + content_index: int + transcript: str + + +class ConversationItemInputAudioTranscriptionFailed(ServerEvent): + """Event indicating input audio transcription failed. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.failed". + item_id: ID of the conversation item that failed transcription. + content_index: Index of the content within the item. + error: Error details for the transcription failure. + """ + + type: Literal["conversation.item.input_audio_transcription.failed"] + item_id: str + content_index: int + error: RealtimeError + + +class ConversationItemTruncated(ServerEvent): + """Event indicating a conversation item has been truncated. + + Parameters: + type: Event type, always "conversation.item.truncated". + item_id: ID of the truncated conversation item. + content_index: Index of the content within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + + type: Literal["conversation.item.truncated"] + item_id: str + content_index: int + audio_end_ms: int + + +class ConversationItemDeleted(ServerEvent): + """Event indicating a conversation item has been deleted. + + Parameters: + type: Event type, always "conversation.item.deleted". + item_id: ID of the deleted conversation item. + """ + + type: Literal["conversation.item.deleted"] + item_id: str + + +class ConversationItemRetrieved(ServerEvent): + """Event containing a retrieved conversation item. + + Parameters: + type: Event type, always "conversation.item.retrieved". + item: The retrieved conversation item. + """ + + type: Literal["conversation.item.retrieved"] + item: ConversationItem + + +class ResponseCreated(ServerEvent): + """Event indicating an assistant response has been created. + + Parameters: + type: Event type, always "response.created". + response: The created response object. + """ + + type: Literal["response.created"] + response: "Response" + + +class ResponseDone(ServerEvent): + """Event indicating an assistant response is complete. + + Parameters: + type: Event type, always "response.done". + response: The completed response object. + """ + + type: Literal["response.done"] + response: "Response" + + +class ResponseOutputItemAdded(ServerEvent): + """Event indicating an output item has been added to a response. + + Parameters: + type: Event type, always "response.output_item.added". + response_id: ID of the response. + output_index: Index of the output item. + item: The added conversation item. + """ + + type: Literal["response.output_item.added"] + response_id: str + output_index: int + item: ConversationItem + + +class ResponseOutputItemDone(ServerEvent): + """Event indicating an output item is complete. + + Parameters: + type: Event type, always "response.output_item.done". + response_id: ID of the response. + output_index: Index of the output item. + item: The completed conversation item. + """ + + type: Literal["response.output_item.done"] + response_id: str + output_index: int + item: ConversationItem + + +class ResponseContentPartAdded(ServerEvent): + """Event indicating a content part has been added to a response. + + Parameters: + type: Event type, always "response.content_part.added". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The added content part. + """ + + type: Literal["response.content_part.added"] + response_id: str + item_id: str + output_index: int + content_index: int + part: ItemContent + + +class ResponseContentPartDone(ServerEvent): + """Event indicating a content part is complete. + + Parameters: + type: Event type, always "response.content_part.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The completed content part. + """ + + type: Literal["response.content_part.done"] + response_id: str + item_id: str + output_index: int + content_index: int + part: ItemContent + + +class ResponseTextDelta(ServerEvent): + """Event containing incremental text from a response. + + Parameters: + type: Event type, always "response.output_text.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental text content. + """ + + type: Literal["response.output_text.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseTextDone(ServerEvent): + """Event indicating text content is complete. + + Parameters: + type: Event type, always "response.output_text.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + text: Complete text content. + """ + + type: Literal["response.output_text.done"] + response_id: str + item_id: str + output_index: int + content_index: int + text: str + + +class ResponseAudioTranscriptDelta(ServerEvent): + """Event containing incremental audio transcript from a response. + + Parameters: + type: Event type, always "response.output_audio_transcript.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental transcript text. + """ + + type: Literal["response.output_audio_transcript.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseAudioTranscriptDone(ServerEvent): + """Event indicating audio transcript is complete. + + Parameters: + type: Event type, always "response.output_audio_transcript.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + transcript: Complete transcript text. + """ + + type: Literal["response.output_audio_transcript.done"] + response_id: str + item_id: str + output_index: int + content_index: int + transcript: str + + +class ResponseAudioDelta(ServerEvent): + """Event containing incremental audio data from a response. + + Parameters: + type: Event type, always "response.output_audio.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Base64-encoded incremental audio data. + """ + + type: Literal["response.output_audio.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str # base64-encoded audio + + +class ResponseAudioDone(ServerEvent): + """Event indicating audio content is complete. + + Parameters: + type: Event type, always "response.output_audio.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + """ + + type: Literal["response.output_audio.done"] + response_id: str + item_id: str + output_index: int + content_index: int + + +class ResponseFunctionCallArgumentsDelta(ServerEvent): + """Event containing incremental function call arguments. + + Parameters: + type: Event type, always "response.function_call_arguments.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + delta: Incremental function arguments as JSON. + """ + + type: Literal["response.function_call_arguments.delta"] + response_id: str + item_id: str + output_index: int + call_id: str + delta: str + + +class ResponseFunctionCallArgumentsDone(ServerEvent): + """Event indicating function call arguments are complete. + + Parameters: + type: Event type, always "response.function_call_arguments.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + arguments: Complete function arguments as JSON string. + """ + + type: Literal["response.function_call_arguments.done"] + response_id: str + item_id: str + output_index: int + call_id: str + arguments: str + + +class InputAudioBufferSpeechStarted(ServerEvent): + """Event indicating speech has started in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_started". + audio_start_ms: Start time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + + type: Literal["input_audio_buffer.speech_started"] + audio_start_ms: int + item_id: str + + +class InputAudioBufferSpeechStopped(ServerEvent): + """Event indicating speech has stopped in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_stopped". + audio_end_ms: End time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + + type: Literal["input_audio_buffer.speech_stopped"] + audio_end_ms: int + item_id: str + + +class InputAudioBufferCommitted(ServerEvent): + """Event indicating the input audio buffer has been committed. + + Parameters: + type: Event type, always "input_audio_buffer.committed". + previous_item_id: ID of the previous item, if any. + item_id: ID of the committed conversation item. + """ + + type: Literal["input_audio_buffer.committed"] + previous_item_id: Optional[str] = None + item_id: str + + +class InputAudioBufferCleared(ServerEvent): + """Event indicating the input audio buffer has been cleared. + + Parameters: + type: Event type, always "input_audio_buffer.cleared". + """ + + type: Literal["input_audio_buffer.cleared"] + + +class ErrorEvent(ServerEvent): + """Event indicating an error occurred. + + Parameters: + type: Event type, always "error". + error: Error details. + """ + + type: Literal["error"] + error: RealtimeError + + +class RateLimitsUpdated(ServerEvent): + """Event indicating rate limits have been updated. + + Parameters: + type: Event type, always "rate_limits.updated". + rate_limits: List of rate limit information. + """ + + type: Literal["rate_limits.updated"] + rate_limits: List[Dict[str, Any]] + + +class CachedTokensDetails(BaseModel): + """Details about cached tokens. + + Parameters: + text_tokens: Number of cached text tokens. + audio_tokens: Number of cached audio tokens. + """ + + text_tokens: Optional[int] = 0 + audio_tokens: Optional[int] = 0 + + +class TokenDetails(BaseModel): + """Detailed token usage information. + + Parameters: + cached_tokens: Number of cached tokens used. Defaults to 0. + text_tokens: Number of text tokens used. Defaults to 0. + audio_tokens: Number of audio tokens used. Defaults to 0. + cached_tokens_details: Detailed breakdown of cached tokens. + image_tokens: Number of image tokens used (for input only). + """ + + cached_tokens: Optional[int] = 0 + text_tokens: Optional[int] = 0 + audio_tokens: Optional[int] = 0 + cached_tokens_details: Optional[CachedTokensDetails] = None + image_tokens: Optional[int] = 0 + + class Config: + """Pydantic configuration for TokenDetails.""" + + extra = "allow" + + +class Usage(BaseModel): + """Token usage statistics for a response. + + Parameters: + total_tokens: Total number of tokens used. + input_tokens: Number of input tokens used. + output_tokens: Number of output tokens used. + input_token_details: Detailed breakdown of input token usage. + output_token_details: Detailed breakdown of output token usage. + """ + + total_tokens: int + input_tokens: int + output_tokens: int + input_token_details: TokenDetails + output_token_details: TokenDetails + + +class Response(BaseModel): + """A complete assistant response. + + Parameters: + id: Unique identifier for the response. + object: Object type, always "realtime.response". + status: Current status of the response. + status_details: Additional status information. + output: List of conversation items in the response. + conversation_id: Which conversation the response is added to. + output_modalities: The set of modalities the model used to respond. + max_output_tokens: Maximum number of output tokens used. + audio: Audio configuration for the response. + usage: Token usage statistics for the response. + voice: The voice the model used to respond. + temperature: Sampling temperature used for the response. + output_audio_format: The format of output audio. + """ + + id: str + object: Literal["realtime.response"] + status: Literal["completed", "in_progress", "incomplete", "cancelled", "failed"] + status_details: Any + output: List[ConversationItem] + output_modalities: Optional[List[Literal["text", "audio"]]] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + audio: Optional[AudioConfiguration] = None + usage: Optional[Usage] = None + voice: Optional[str] = None + temperature: Optional[float] = None + output_audio_format: Optional[str] = None + + +_server_event_types = { + "error": ErrorEvent, + "session.created": SessionCreatedEvent, + "session.updated": SessionUpdatedEvent, + "conversation.created": ConversationCreated, + "input_audio_buffer.committed": InputAudioBufferCommitted, + "input_audio_buffer.cleared": InputAudioBufferCleared, + "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, + "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, + "conversation.item.added": ConversationItemAdded, + "conversation.item.done": ConversationItemDone, + "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, + "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, + "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, + "conversation.item.truncated": ConversationItemTruncated, + "conversation.item.deleted": ConversationItemDeleted, + "conversation.item.retrieved": ConversationItemRetrieved, + "response.created": ResponseCreated, + "response.done": ResponseDone, + "response.output_item.added": ResponseOutputItemAdded, + "response.output_item.done": ResponseOutputItemDone, + "response.content_part.added": ResponseContentPartAdded, + "response.content_part.done": ResponseContentPartDone, + "response.output_text.delta": ResponseTextDelta, + "response.output_text.done": ResponseTextDone, + "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta, + "response.output_audio_transcript.done": ResponseAudioTranscriptDone, + "response.output_audio.delta": ResponseAudioDelta, + "response.output_audio.done": ResponseAudioDone, + "response.function_call_arguments.delta": ResponseFunctionCallArgumentsDelta, + "response.function_call_arguments.done": ResponseFunctionCallArgumentsDone, + "rate_limits.updated": RateLimitsUpdated, +} + + +def parse_server_event(str): + """Parse a server event from JSON string. + + Args: + str: JSON string containing the server event. + + Returns: + Parsed server event object of the appropriate type. + + Raises: + Exception: If the event type is unimplemented or parsing fails. + """ + try: + event = json.loads(str) + event_type = event["type"] + if event_type not in _server_event_types: + raise Exception(f"Unimplemented server event type: {event_type}") + return _server_event_types[event_type].model_validate(event) + except Exception as e: + raise Exception(f"{e} \n\n{str}") diff --git a/src/pipecat/services/openai/realtime/frames.py b/src/pipecat/services/openai/realtime/frames.py new file mode 100644 index 000000000..8617c6efd --- /dev/null +++ b/src/pipecat/services/openai/realtime/frames.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Custom frame types for OpenAI Realtime API integration.""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from pipecat.frames.frames import DataFrame, FunctionCallResultFrame + +if TYPE_CHECKING: + from pipecat.services.openai.realtime.context import OpenAIRealtimeLLMContext + + +@dataclass +class RealtimeMessagesUpdateFrame(DataFrame): + """Frame indicating that the realtime context messages have been updated. + + Parameters: + context: The updated OpenAI realtime LLM context. + """ + + context: "OpenAIRealtimeLLMContext" + + +@dataclass +class RealtimeFunctionCallResultFrame(DataFrame): + """Frame containing function call results for the realtime service. + + Parameters: + result_frame: The function call result frame to send to the realtime API. + """ + + result_frame: FunctionCallResultFrame diff --git a/src/pipecat/services/openai_realtime/openai.py b/src/pipecat/services/openai/realtime/llm.py similarity index 100% rename from src/pipecat/services/openai_realtime/openai.py rename to src/pipecat/services/openai/realtime/llm.py diff --git a/src/pipecat/services/openai_realtime/__init__.py b/src/pipecat/services/openai_realtime/__init__.py index 6f3154f1c..f1fbefeda 100644 --- a/src/pipecat/services/openai_realtime/__init__.py +++ b/src/pipecat/services/openai_realtime/__init__.py @@ -1,9 +1,27 @@ -from .azure import AzureRealtimeLLMService -from .events import ( +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import warnings + +from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService +from pipecat.services.openai.realtime.events import ( InputAudioNoiseReduction, InputAudioTranscription, SemanticTurnDetection, SessionProperties, TurnDetection, ) -from .openai import OpenAIRealtimeLLMService +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/azure.py b/src/pipecat/services/openai_realtime/azure.py index aed53b94c..dae6ef496 100644 --- a/src/pipecat/services/openai_realtime/azure.py +++ b/src/pipecat/services/openai_realtime/azure.py @@ -1,67 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Azure OpenAI Realtime LLM service implementation.""" -from loguru import logger +import warnings -from .openai import OpenAIRealtimeLLMService +from pipecat.services.azure.realtime.llm import * -try: - from websockets.asyncio.client import connect as websocket_connect -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.azure are deprecated. " + "Please use the equivalent types from " + "pipecat.services.azure.realtime.llm instead.", + DeprecationWarning, + stacklevel=2, ) - raise Exception(f"Missing module: {e}") - - -class AzureRealtimeLLMService(OpenAIRealtimeLLMService): - """Azure OpenAI Realtime LLM service with Azure-specific authentication. - - Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, - using Azure's authentication headers and endpoint format. Provides the same - real-time audio and text communication capabilities as the base OpenAI service. - """ - - def __init__( - self, - *, - api_key: str, - base_url: str, - **kwargs, - ): - """Initialize Azure Realtime LLM service. - - Args: - api_key: The API key for the Azure OpenAI service. - base_url: The full Azure WebSocket endpoint URL including api-version and deployment. - Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" - **kwargs: Additional arguments passed to parent OpenAIRealtimeLLMService. - """ - super().__init__(base_url=base_url, api_key=api_key, **kwargs) - self.api_key = api_key - self.base_url = base_url - - async def _connect(self): - try: - if self._websocket: - # Here we assume that if we have a websocket, we are connected. We - # handle disconnections in the send/recv code paths. - return - - logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") - self._websocket = await websocket_connect( - uri=self.base_url, - additional_headers={ - "api-key": self.api_key, - }, - ) - self._receive_task = self.create_task(self._receive_task_handler()) - except Exception as e: - logger.error(f"{self} initialization error: {e}") - self._websocket = None diff --git a/src/pipecat/services/openai_realtime/context.py b/src/pipecat/services/openai_realtime/context.py index cb1c0a9f5..58f1cfe75 100644 --- a/src/pipecat/services/openai_realtime/context.py +++ b/src/pipecat/services/openai_realtime/context.py @@ -1,272 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """OpenAI Realtime LLM context and aggregator implementations.""" -import copy -import json +import warnings -from loguru import logger +from pipecat.services.openai.realtime.context import * -from pipecat.frames.frames import ( - Frame, - FunctionCallResultFrame, - InterimTranscriptionFrame, - LLMMessagesUpdateFrame, - LLMSetToolsFrame, - LLMTextFrame, - TranscriptionFrame, -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) - -from . import events -from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame - - -class OpenAIRealtimeLLMContext(OpenAILLMContext): - """OpenAI Realtime LLM context with session management and message conversion. - - Extends the standard OpenAI LLM context to support real-time session properties, - instruction management, and conversion between standard message formats and - realtime conversation items. - """ - - def __init__(self, messages=None, tools=None, **kwargs): - """Initialize the OpenAIRealtimeLLMContext. - - Args: - messages: Initial conversation messages. Defaults to None. - tools: Available function tools. Defaults to None. - **kwargs: Additional arguments passed to parent OpenAILLMContext. - """ - super().__init__(messages=messages, tools=tools, **kwargs) - self.__setup_local() - - def __setup_local(self): - self.llm_needs_settings_update = True - self.llm_needs_initial_messages = True - self._session_instructions = "" - - return - - @staticmethod - def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": - """Upgrade a standard OpenAI LLM context to a realtime context. - - Args: - obj: The OpenAILLMContext instance to upgrade. - - Returns: - The upgraded OpenAIRealtimeLLMContext instance. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): - obj.__class__ = OpenAIRealtimeLLMContext - obj.__setup_local() - return obj - - # todo - # - finish implementing all frames - - def from_standard_message(self, message): - """Convert a standard message format to a realtime conversation item. - - Args: - message: The standard message dictionary to convert. - - Returns: - A ConversationItem instance for the realtime API. - """ - if message.get("role") == "user": - content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): - if c.get("type") == "text": - content += " " + c.get("text") - else: - logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" - ) - return events.ConversationItem( - role="user", - type="message", - content=[events.ItemContent(type="input_text", text=content)], - ) - if message.get("role") == "assistant" and message.get("tool_calls"): - tc = message.get("tool_calls")[0] - return events.ConversationItem( - type="function_call", - call_id=tc["id"], - name=tc["function"]["name"], - arguments=tc["function"]["arguments"], - ) - logger.error(f"Unhandled message type in from_standard_message: {message}") - - def get_messages_for_initializing_history(self): - """Get conversation items for initializing the realtime session history. - - Converts the context's messages to a format suitable for the realtime API, - handling system instructions and conversation history packaging. - - Returns: - List of conversation items for session initialization. - """ - # We can't load a long conversation history into the openai realtime api yet. (The API/model - # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So - # our general strategy until this is fixed is just to put everything into a first "user" - # message as a single input. - if not self.messages: - return [] - - messages = copy.deepcopy(self.messages) - - # If we have a "system" message as our first message, let's pull that out into session - # "instructions" - if messages[0].get("role") == "system": - self.llm_needs_settings_update = True - system = messages.pop(0) - content = system.get("content") - if isinstance(content, str): - self._session_instructions = content - elif isinstance(content, list): - self._session_instructions = content[0].get("text") - if not messages: - return [] - - # If we have just a single "user" item, we can just send it normally - if len(messages) == 1 and messages[0].get("role") == "user": - return [self.from_standard_message(messages[0])] - - # Otherwise, let's pack everything into a single "user" message with a bit of - # explanation for the LLM - intro_text = """ - This is a previously saved conversation. Please treat this conversation history as a - starting point for the current conversation.""" - - trailing_text = """ - This is the end of the previously saved conversation. Please continue the conversation - from here. If the last message is a user instruction or question, act on that instruction - or answer the question. If the last message is an assistant response, simple say that you - are ready to continue the conversation.""" - - return [ - { - "role": "user", - "type": "message", - "content": [ - { - "type": "input_text", - "text": "\n\n".join( - [intro_text, json.dumps(messages, indent=2), trailing_text] - ), - } - ], - } - ] - - def add_user_content_item_as_message(self, item): - """Add a user content item as a standard message to the context. - - Args: - item: The conversation item to add as a user message. - """ - message = { - "role": "user", - "content": [{"type": "text", "text": item.content[0].transcript}], - } - self.add_message(message) - - -class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): - """User context aggregator for OpenAI Realtime API. - - Handles user input frames and generates appropriate context updates - for the realtime conversation, including message updates and tool settings. - - Args: - context: The OpenAI realtime LLM context. - **kwargs: Additional arguments passed to parent aggregator. - """ - - async def process_frame( - self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM - ): - """Process incoming frames and handle realtime-specific frame types. - - Args: - frame: The frame to process. - direction: The direction of frame flow in the pipeline. - """ - await super().process_frame(frame, direction) - # Parent does not push LLMMessagesUpdateFrame. This ensures that in a typical pipeline, - # messages are only processed by the user context aggregator, which is generally what we want. But - # we also need to send new messages over the websocket, so the openai realtime API has them - # in its context. - if isinstance(frame, LLMMessagesUpdateFrame): - await self.push_frame(RealtimeMessagesUpdateFrame(context=self._context)) - - # Parent also doesn't push the LLMSetToolsFrame. - if isinstance(frame, LLMSetToolsFrame): - await self.push_frame(frame, direction) - - async def push_aggregation(self): - """Push user input aggregation. - - Currently ignores all user input coming into the pipeline as realtime - audio input is handled directly by the service. - """ - # for the moment, ignore all user input coming into the pipeline. - # todo: think about whether/how to fix this to allow for text input from - # upstream (transport/transcription, or other sources) - pass - - -class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Assistant context aggregator for OpenAI Realtime API. - - Handles assistant output frames from the realtime service, filtering - out duplicate text frames and managing function call results. - - Args: - context: The OpenAI realtime LLM context. - **kwargs: Additional arguments passed to parent aggregator. - """ - - # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, - # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We - # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames - # are process. This ensures that the context gets only one set of messages. - # OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames, - # so we need to ignore pushing those as well, as they're also TextFrames. - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process assistant frames, filtering out duplicate text content. - - Args: - frame: The frame to process. - direction: The direction of frame flow in the pipeline. - """ - if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)): - await super().process_frame(frame, direction) - - async def handle_function_call_result(self, frame: FunctionCallResultFrame): - """Handle function call result and notify the realtime service. - - Args: - frame: The function call result frame to handle. - """ - await super().handle_function_call_result(frame) - - # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, - # so we didn't have a chance to add the result to the openai realtime api context. Let's push a - # special frame to do that. - await self.push_frame( - RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.context are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.context instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/events.py b/src/pipecat/services/openai_realtime/events.py index 200e59d68..53b4b0dff 100644 --- a/src/pipecat/services/openai_realtime/events.py +++ b/src/pipecat/services/openai_realtime/events.py @@ -1,1106 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Event models and data structures for OpenAI Realtime API communication.""" -import json -import uuid -from typing import Any, Dict, List, Literal, Optional, Union - -from pydantic import BaseModel, ConfigDict, Field - -# -# session properties -# - - -class AudioFormat(BaseModel): - """Base class for audio format configuration.""" - - type: str - - -class PCMAudioFormat(AudioFormat): - """PCM audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcm". - rate: Sample rate, always 24000 for PCM. - """ - - type: Literal["audio/pcm"] = "audio/pcm" - rate: Literal[24000] = 24000 - - -class PCMUAudioFormat(AudioFormat): - """PCMU (G.711 μ-law) audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcmu". - """ - - type: Literal["audio/pcmu"] = "audio/pcmu" - - -class PCMAAudioFormat(AudioFormat): - """PCMA (G.711 A-law) audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcma". - """ - - type: Literal["audio/pcma"] = "audio/pcma" - - -class InputAudioTranscription(BaseModel): - """Configuration for audio transcription settings.""" - - model: str = "gpt-4o-transcribe" - language: Optional[str] - prompt: Optional[str] - - def __init__( - self, - model: Optional[str] = "gpt-4o-transcribe", - language: Optional[str] = None, - prompt: Optional[str] = None, - ): - """Initialize InputAudioTranscription. - - Args: - model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). - language: Optional language code for transcription. - prompt: Optional transcription hint text. - """ - super().__init__(model=model, language=language, prompt=prompt) - - -class TurnDetection(BaseModel): - """Server-side voice activity detection configuration. - - Parameters: - type: Detection type, must be "server_vad". - threshold: Voice activity detection threshold (0.0-1.0). Defaults to 0.5. - prefix_padding_ms: Padding before speech starts in milliseconds. Defaults to 300. - silence_duration_ms: Silence duration to detect speech end in milliseconds. Defaults to 500. - """ - - type: Optional[Literal["server_vad"]] = "server_vad" - threshold: Optional[float] = 0.5 - prefix_padding_ms: Optional[int] = 300 - silence_duration_ms: Optional[int] = 500 - - -class SemanticTurnDetection(BaseModel): - """Semantic-based turn detection configuration. - - Parameters: - type: Detection type, must be "semantic_vad". - eagerness: Turn detection eagerness level. Can be "low", "medium", "high", or "auto". - create_response: Whether to automatically create responses on turn detection. - interrupt_response: Whether to interrupt ongoing responses on turn detection. - """ - - type: Optional[Literal["semantic_vad"]] = "semantic_vad" - eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None - create_response: Optional[bool] = None - interrupt_response: Optional[bool] = None - - -class InputAudioNoiseReduction(BaseModel): - """Input audio noise reduction configuration. - - Parameters: - type: Noise reduction type for different microphone scenarios. - """ - - type: Optional[Literal["near_field", "far_field"]] - - -class AudioInput(BaseModel): - """Audio input configuration. - - Parameters: - format: The format of the input audio. - transcription: Configuration for input audio transcription. - noise_reduction: Configuration for input audio noise reduction. - turn_detection: Configuration for turn detection, or False to disable. - """ - - format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None - transcription: Optional[InputAudioTranscription] = None - noise_reduction: Optional[InputAudioNoiseReduction] = None - turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = None - - -class AudioOutput(BaseModel): - """Audio output configuration. - - Parameters: - format: The format of the output audio. - voice: The voice the model uses to respond. - speed: The speed of the model's spoken response. - """ - - format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None - voice: Optional[str] = None - speed: Optional[float] = None - - -class AudioConfiguration(BaseModel): - """Audio configuration for input and output. - - Parameters: - input: Configuration for input audio. - output: Configuration for output audio. - """ - - input: Optional[AudioInput] = None - output: Optional[AudioOutput] = None - - -class SessionProperties(BaseModel): - """Configuration properties for an OpenAI Realtime session. - - Parameters: - type: The type of session, always "realtime". - object: Object type identifier, always "realtime.session". - id: Unique identifier for the session. - model: The Realtime model used for this session. - output_modalities: The set of modalities the model can respond with. - instructions: System instructions for the assistant. - audio: Configuration for input and output audio. - tools: Available function tools for the assistant. - tool_choice: Tool usage strategy ("auto", "none", or "required"). - max_output_tokens: Maximum tokens in response or "inf" for unlimited. - tracing: Configuration options for tracing. - prompt: Reference to a prompt template and its variables. - expires_at: Session expiration timestamp. - include: Additional fields to include in server outputs. - """ - - type: Optional[Literal["realtime"]] = "realtime" - object: Optional[Literal["realtime.session"]] = None - id: Optional[str] = None - model: Optional[str] = None - output_modalities: Optional[List[Literal["text", "audio"]]] = None - instructions: Optional[str] = None - audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None - tool_choice: Optional[Literal["auto", "none", "required"]] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - tracing: Optional[Union[Literal["auto"], Dict]] = None - prompt: Optional[Dict] = None - expires_at: Optional[int] = None - include: Optional[List[str]] = None - - -# -# context -# - - -class ItemContent(BaseModel): - """Content within a conversation item. - - Parameters: - type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). - text: Text content for text-based items. - audio: Base64-encoded audio data for audio items. - transcript: Transcribed text for audio items. - """ - - type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] - text: Optional[str] = None - audio: Optional[str] = None # base64-encoded audio - transcript: Optional[str] = None - - -class ConversationItem(BaseModel): - """A conversation item in the realtime session. - - Parameters: - id: Unique identifier for the item, auto-generated if not provided. - object: Object type identifier for the realtime API. - type: Item type (message, function_call, or function_call_output). - status: Current status of the item. - role: Speaker role for message items (user, assistant, or system). - content: Content list for message items. - call_id: Function call identifier for function_call items. - name: Function name for function_call items. - arguments: Function arguments as JSON string for function_call items. - output: Function output as JSON string for function_call_output items. - """ - - id: str = Field(default_factory=lambda: str(uuid.uuid4().hex)) - object: Optional[Literal["realtime.item"]] = None - type: Literal["message", "function_call", "function_call_output"] - status: Optional[Literal["completed", "in_progress", "incomplete"]] = None - # role and content are present for message items - role: Optional[Literal["user", "assistant", "system"]] = None - content: Optional[List[ItemContent]] = None - # these four fields are present for function_call items - call_id: Optional[str] = None - name: Optional[str] = None - arguments: Optional[str] = None - output: Optional[str] = None - - -class RealtimeConversation(BaseModel): - """A realtime conversation session. - - Parameters: - id: Unique identifier for the conversation. - object: Object type identifier, always "realtime.conversation". - """ - - id: str - object: Literal["realtime.conversation"] - - -class ResponseProperties(BaseModel): - """Properties for configuring assistant responses. - - Parameters: - output_modalities: Output modalities for the response. Must be either ["text"] or ["audio"]. Defaults to ["audio"]. - instructions: Specific instructions for this response. - audio: Audio configuration for this response. - tools: Available tools for this response. - tool_choice: Tool usage strategy for this response. - temperature: Sampling temperature for this response. - max_output_tokens: Maximum tokens for this response. - """ - - output_modalities: Optional[List[Literal["text", "audio"]]] = ["audio"] - instructions: Optional[str] = None - audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None - tool_choice: Optional[Literal["auto", "none", "required"]] = None - temperature: Optional[float] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - - -# -# error class -# -class RealtimeError(BaseModel): - """Error information from the realtime API. - - Parameters: - type: Error type identifier. - code: Specific error code. - message: Human-readable error message. - param: Parameter name that caused the error, if applicable. - event_id: Event ID associated with the error, if applicable. - """ - - type: str - code: Optional[str] = "" - message: str - param: Optional[str] = None - event_id: Optional[str] = None - - -# -# client events -# - - -class ClientEvent(BaseModel): - """Base class for client events sent to the realtime API. - - Parameters: - event_id: Unique identifier for the event, auto-generated if not provided. - """ - - event_id: str = Field(default_factory=lambda: str(uuid.uuid4())) - - -class SessionUpdateEvent(ClientEvent): - """Event to update session properties. - - Parameters: - type: Event type, always "session.update". - session: Updated session properties. - """ - - type: Literal["session.update"] = "session.update" - session: SessionProperties - - def model_dump(self, *args, **kwargs) -> Dict[str, Any]: - """Serialize the event to a dictionary. - - Handles special serialization for turn_detection where False becomes null. - - Args: - *args: Positional arguments passed to parent model_dump. - **kwargs: Keyword arguments passed to parent model_dump. - - Returns: - Dictionary representation of the event. - """ - dump = super().model_dump(*args, **kwargs) - - # Handle turn_detection in audio.input so that False becomes null - if "audio" in dump["session"] and dump["session"]["audio"]: - if "input" in dump["session"]["audio"] and dump["session"]["audio"]["input"]: - if "turn_detection" in dump["session"]["audio"]["input"]: - if dump["session"]["audio"]["input"]["turn_detection"] is False: - dump["session"]["audio"]["input"]["turn_detection"] = None - - return dump - - -class InputAudioBufferAppendEvent(ClientEvent): - """Event to append audio data to the input buffer. - - Parameters: - type: Event type, always "input_audio_buffer.append". - audio: Base64-encoded audio data to append. - """ - - type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" - audio: str # base64-encoded audio - - -class InputAudioBufferCommitEvent(ClientEvent): - """Event to commit the current input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.commit". - """ - - type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" - - -class InputAudioBufferClearEvent(ClientEvent): - """Event to clear the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.clear". - """ - - type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" - - -class ConversationItemCreateEvent(ClientEvent): - """Event to create a new conversation item. - - Parameters: - type: Event type, always "conversation.item.create". - previous_item_id: ID of the item to insert after, if any. - item: The conversation item to create. - """ - - type: Literal["conversation.item.create"] = "conversation.item.create" - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemTruncateEvent(ClientEvent): - """Event to truncate a conversation item's audio content. - - Parameters: - type: Event type, always "conversation.item.truncate". - item_id: ID of the item to truncate. - content_index: Index of the content to truncate within the item. - audio_end_ms: End time in milliseconds for the truncated audio. - """ - - type: Literal["conversation.item.truncate"] = "conversation.item.truncate" - item_id: str - content_index: int - audio_end_ms: int - - -class ConversationItemDeleteEvent(ClientEvent): - """Event to delete a conversation item. - - Parameters: - type: Event type, always "conversation.item.delete". - item_id: ID of the item to delete. - """ - - type: Literal["conversation.item.delete"] = "conversation.item.delete" - item_id: str - - -class ConversationItemRetrieveEvent(ClientEvent): - """Event to retrieve a conversation item by ID. - - Parameters: - type: Event type, always "conversation.item.retrieve". - item_id: ID of the item to retrieve. - """ - - type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" - item_id: str - - -class ResponseCreateEvent(ClientEvent): - """Event to create a new assistant response. - - Parameters: - type: Event type, always "response.create". - response: Optional response configuration properties. - """ - - type: Literal["response.create"] = "response.create" - response: Optional[ResponseProperties] = None - - -class ResponseCancelEvent(ClientEvent): - """Event to cancel the current assistant response. - - Parameters: - type: Event type, always "response.cancel". - """ - - type: Literal["response.cancel"] = "response.cancel" - - -# -# server events -# - - -class ServerEvent(BaseModel): - """Base class for server events received from the realtime API. - - Parameters: - event_id: Unique identifier for the event. - type: Type of the server event. - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - - event_id: str - type: str - - -class SessionCreatedEvent(ServerEvent): - """Event indicating a session has been created. - - Parameters: - type: Event type, always "session.created". - session: The created session properties. - """ - - type: Literal["session.created"] - session: SessionProperties - - -class SessionUpdatedEvent(ServerEvent): - """Event indicating a session has been updated. - - Parameters: - type: Event type, always "session.updated". - session: The updated session properties. - """ - - type: Literal["session.updated"] - session: SessionProperties - - -class ConversationCreated(ServerEvent): - """Event indicating a conversation has been created. - - Parameters: - type: Event type, always "conversation.created". - conversation: The created conversation. - """ - - type: Literal["conversation.created"] - conversation: RealtimeConversation - - -class ConversationItemAdded(ServerEvent): - """Event indicating a conversation item has been added. - - Parameters: - type: Event type, always "conversation.item.added". - previous_item_id: ID of the previous item, if any. - item: The added conversation item. - """ - - type: Literal["conversation.item.added"] - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemDone(ServerEvent): - """Event indicating a conversation item is done processing. - - Parameters: - type: Event type, always "conversation.item.done". - previous_item_id: ID of the previous item, if any. - item: The completed conversation item. - """ - - type: Literal["conversation.item.done"] - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemInputAudioTranscriptionDelta(ServerEvent): - """Event containing incremental input audio transcription. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.delta". - item_id: ID of the conversation item being transcribed. - content_index: Index of the content within the item. - delta: Incremental transcription text. - """ - - type: Literal["conversation.item.input_audio_transcription.delta"] - item_id: str - content_index: int - delta: str - - -class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): - """Event indicating input audio transcription is complete. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.completed". - item_id: ID of the conversation item that was transcribed. - content_index: Index of the content within the item. - transcript: Complete transcription text. - """ - - type: Literal["conversation.item.input_audio_transcription.completed"] - item_id: str - content_index: int - transcript: str - - -class ConversationItemInputAudioTranscriptionFailed(ServerEvent): - """Event indicating input audio transcription failed. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.failed". - item_id: ID of the conversation item that failed transcription. - content_index: Index of the content within the item. - error: Error details for the transcription failure. - """ - - type: Literal["conversation.item.input_audio_transcription.failed"] - item_id: str - content_index: int - error: RealtimeError - - -class ConversationItemTruncated(ServerEvent): - """Event indicating a conversation item has been truncated. - - Parameters: - type: Event type, always "conversation.item.truncated". - item_id: ID of the truncated conversation item. - content_index: Index of the content within the item. - audio_end_ms: End time in milliseconds for the truncated audio. - """ - - type: Literal["conversation.item.truncated"] - item_id: str - content_index: int - audio_end_ms: int - - -class ConversationItemDeleted(ServerEvent): - """Event indicating a conversation item has been deleted. - - Parameters: - type: Event type, always "conversation.item.deleted". - item_id: ID of the deleted conversation item. - """ - - type: Literal["conversation.item.deleted"] - item_id: str - - -class ConversationItemRetrieved(ServerEvent): - """Event containing a retrieved conversation item. - - Parameters: - type: Event type, always "conversation.item.retrieved". - item: The retrieved conversation item. - """ - - type: Literal["conversation.item.retrieved"] - item: ConversationItem - - -class ResponseCreated(ServerEvent): - """Event indicating an assistant response has been created. - - Parameters: - type: Event type, always "response.created". - response: The created response object. - """ - - type: Literal["response.created"] - response: "Response" - - -class ResponseDone(ServerEvent): - """Event indicating an assistant response is complete. - - Parameters: - type: Event type, always "response.done". - response: The completed response object. - """ - - type: Literal["response.done"] - response: "Response" - - -class ResponseOutputItemAdded(ServerEvent): - """Event indicating an output item has been added to a response. - - Parameters: - type: Event type, always "response.output_item.added". - response_id: ID of the response. - output_index: Index of the output item. - item: The added conversation item. - """ - - type: Literal["response.output_item.added"] - response_id: str - output_index: int - item: ConversationItem - - -class ResponseOutputItemDone(ServerEvent): - """Event indicating an output item is complete. - - Parameters: - type: Event type, always "response.output_item.done". - response_id: ID of the response. - output_index: Index of the output item. - item: The completed conversation item. - """ - - type: Literal["response.output_item.done"] - response_id: str - output_index: int - item: ConversationItem - - -class ResponseContentPartAdded(ServerEvent): - """Event indicating a content part has been added to a response. - - Parameters: - type: Event type, always "response.content_part.added". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - part: The added content part. - """ - - type: Literal["response.content_part.added"] - response_id: str - item_id: str - output_index: int - content_index: int - part: ItemContent - - -class ResponseContentPartDone(ServerEvent): - """Event indicating a content part is complete. - - Parameters: - type: Event type, always "response.content_part.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - part: The completed content part. - """ - - type: Literal["response.content_part.done"] - response_id: str - item_id: str - output_index: int - content_index: int - part: ItemContent - - -class ResponseTextDelta(ServerEvent): - """Event containing incremental text from a response. - - Parameters: - type: Event type, always "response.output_text.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Incremental text content. - """ - - type: Literal["response.output_text.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str - - -class ResponseTextDone(ServerEvent): - """Event indicating text content is complete. - - Parameters: - type: Event type, always "response.output_text.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - text: Complete text content. - """ - - type: Literal["response.output_text.done"] - response_id: str - item_id: str - output_index: int - content_index: int - text: str - - -class ResponseAudioTranscriptDelta(ServerEvent): - """Event containing incremental audio transcript from a response. - - Parameters: - type: Event type, always "response.output_audio_transcript.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Incremental transcript text. - """ - - type: Literal["response.output_audio_transcript.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str - - -class ResponseAudioTranscriptDone(ServerEvent): - """Event indicating audio transcript is complete. - - Parameters: - type: Event type, always "response.output_audio_transcript.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - transcript: Complete transcript text. - """ - - type: Literal["response.output_audio_transcript.done"] - response_id: str - item_id: str - output_index: int - content_index: int - transcript: str - - -class ResponseAudioDelta(ServerEvent): - """Event containing incremental audio data from a response. - - Parameters: - type: Event type, always "response.output_audio.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Base64-encoded incremental audio data. - """ - - type: Literal["response.output_audio.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str # base64-encoded audio - - -class ResponseAudioDone(ServerEvent): - """Event indicating audio content is complete. - - Parameters: - type: Event type, always "response.output_audio.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - """ - - type: Literal["response.output_audio.done"] - response_id: str - item_id: str - output_index: int - content_index: int - - -class ResponseFunctionCallArgumentsDelta(ServerEvent): - """Event containing incremental function call arguments. - - Parameters: - type: Event type, always "response.function_call_arguments.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - call_id: ID of the function call. - delta: Incremental function arguments as JSON. - """ - - type: Literal["response.function_call_arguments.delta"] - response_id: str - item_id: str - output_index: int - call_id: str - delta: str - - -class ResponseFunctionCallArgumentsDone(ServerEvent): - """Event indicating function call arguments are complete. - - Parameters: - type: Event type, always "response.function_call_arguments.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - call_id: ID of the function call. - arguments: Complete function arguments as JSON string. - """ - - type: Literal["response.function_call_arguments.done"] - response_id: str - item_id: str - output_index: int - call_id: str - arguments: str - - -class InputAudioBufferSpeechStarted(ServerEvent): - """Event indicating speech has started in the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.speech_started". - audio_start_ms: Start time of speech in milliseconds. - item_id: ID of the associated conversation item. - """ - - type: Literal["input_audio_buffer.speech_started"] - audio_start_ms: int - item_id: str - - -class InputAudioBufferSpeechStopped(ServerEvent): - """Event indicating speech has stopped in the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.speech_stopped". - audio_end_ms: End time of speech in milliseconds. - item_id: ID of the associated conversation item. - """ - - type: Literal["input_audio_buffer.speech_stopped"] - audio_end_ms: int - item_id: str - - -class InputAudioBufferCommitted(ServerEvent): - """Event indicating the input audio buffer has been committed. - - Parameters: - type: Event type, always "input_audio_buffer.committed". - previous_item_id: ID of the previous item, if any. - item_id: ID of the committed conversation item. - """ - - type: Literal["input_audio_buffer.committed"] - previous_item_id: Optional[str] = None - item_id: str - - -class InputAudioBufferCleared(ServerEvent): - """Event indicating the input audio buffer has been cleared. - - Parameters: - type: Event type, always "input_audio_buffer.cleared". - """ - - type: Literal["input_audio_buffer.cleared"] - - -class ErrorEvent(ServerEvent): - """Event indicating an error occurred. - - Parameters: - type: Event type, always "error". - error: Error details. - """ - - type: Literal["error"] - error: RealtimeError - - -class RateLimitsUpdated(ServerEvent): - """Event indicating rate limits have been updated. - - Parameters: - type: Event type, always "rate_limits.updated". - rate_limits: List of rate limit information. - """ - - type: Literal["rate_limits.updated"] - rate_limits: List[Dict[str, Any]] - - -class CachedTokensDetails(BaseModel): - """Details about cached tokens. - - Parameters: - text_tokens: Number of cached text tokens. - audio_tokens: Number of cached audio tokens. - """ - - text_tokens: Optional[int] = 0 - audio_tokens: Optional[int] = 0 - - -class TokenDetails(BaseModel): - """Detailed token usage information. - - Parameters: - cached_tokens: Number of cached tokens used. Defaults to 0. - text_tokens: Number of text tokens used. Defaults to 0. - audio_tokens: Number of audio tokens used. Defaults to 0. - cached_tokens_details: Detailed breakdown of cached tokens. - image_tokens: Number of image tokens used (for input only). - """ - - cached_tokens: Optional[int] = 0 - text_tokens: Optional[int] = 0 - audio_tokens: Optional[int] = 0 - cached_tokens_details: Optional[CachedTokensDetails] = None - image_tokens: Optional[int] = 0 - - class Config: - """Pydantic configuration for TokenDetails.""" - - extra = "allow" - - -class Usage(BaseModel): - """Token usage statistics for a response. - - Parameters: - total_tokens: Total number of tokens used. - input_tokens: Number of input tokens used. - output_tokens: Number of output tokens used. - input_token_details: Detailed breakdown of input token usage. - output_token_details: Detailed breakdown of output token usage. - """ - - total_tokens: int - input_tokens: int - output_tokens: int - input_token_details: TokenDetails - output_token_details: TokenDetails - - -class Response(BaseModel): - """A complete assistant response. - - Parameters: - id: Unique identifier for the response. - object: Object type, always "realtime.response". - status: Current status of the response. - status_details: Additional status information. - output: List of conversation items in the response. - conversation_id: Which conversation the response is added to. - output_modalities: The set of modalities the model used to respond. - max_output_tokens: Maximum number of output tokens used. - audio: Audio configuration for the response. - usage: Token usage statistics for the response. - voice: The voice the model used to respond. - temperature: Sampling temperature used for the response. - output_audio_format: The format of output audio. - """ - - id: str - object: Literal["realtime.response"] - status: Literal["completed", "in_progress", "incomplete", "cancelled", "failed"] - status_details: Any - output: List[ConversationItem] - output_modalities: Optional[List[Literal["text", "audio"]]] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - audio: Optional[AudioConfiguration] = None - usage: Optional[Usage] = None - voice: Optional[str] = None - temperature: Optional[float] = None - output_audio_format: Optional[str] = None - - -_server_event_types = { - "error": ErrorEvent, - "session.created": SessionCreatedEvent, - "session.updated": SessionUpdatedEvent, - "conversation.created": ConversationCreated, - "input_audio_buffer.committed": InputAudioBufferCommitted, - "input_audio_buffer.cleared": InputAudioBufferCleared, - "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, - "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, - "conversation.item.added": ConversationItemAdded, - "conversation.item.done": ConversationItemDone, - "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, - "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, - "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, - "conversation.item.truncated": ConversationItemTruncated, - "conversation.item.deleted": ConversationItemDeleted, - "conversation.item.retrieved": ConversationItemRetrieved, - "response.created": ResponseCreated, - "response.done": ResponseDone, - "response.output_item.added": ResponseOutputItemAdded, - "response.output_item.done": ResponseOutputItemDone, - "response.content_part.added": ResponseContentPartAdded, - "response.content_part.done": ResponseContentPartDone, - "response.output_text.delta": ResponseTextDelta, - "response.output_text.done": ResponseTextDone, - "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta, - "response.output_audio_transcript.done": ResponseAudioTranscriptDone, - "response.output_audio.delta": ResponseAudioDelta, - "response.output_audio.done": ResponseAudioDone, - "response.function_call_arguments.delta": ResponseFunctionCallArgumentsDelta, - "response.function_call_arguments.done": ResponseFunctionCallArgumentsDone, - "rate_limits.updated": RateLimitsUpdated, -} - - -def parse_server_event(str): - """Parse a server event from JSON string. - - Args: - str: JSON string containing the server event. - - Returns: - Parsed server event object of the appropriate type. - - Raises: - Exception: If the event type is unimplemented or parsing fails. - """ - try: - event = json.loads(str) - event_type = event["type"] - if event_type not in _server_event_types: - raise Exception(f"Unimplemented server event type: {event_type}") - return _server_event_types[event_type].model_validate(event) - except Exception as e: - raise Exception(f"{e} \n\n{str}") +import warnings + +from pipecat.services.openai.realtime.events import * + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.events are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.events instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/frames.py b/src/pipecat/services/openai_realtime/frames.py index 290e025f9..e7e4d7d9f 100644 --- a/src/pipecat/services/openai_realtime/frames.py +++ b/src/pipecat/services/openai_realtime/frames.py @@ -1,37 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Custom frame types for OpenAI Realtime API integration.""" -from dataclasses import dataclass -from typing import TYPE_CHECKING +import warnings -from pipecat.frames.frames import DataFrame, FunctionCallResultFrame +from pipecat.services.openai.realtime.frames import * -if TYPE_CHECKING: - from pipecat.services.openai_realtime.context import OpenAIRealtimeLLMContext - - -@dataclass -class RealtimeMessagesUpdateFrame(DataFrame): - """Frame indicating that the realtime context messages have been updated. - - Parameters: - context: The updated OpenAI realtime LLM context. - """ - - context: "OpenAIRealtimeLLMContext" - - -@dataclass -class RealtimeFunctionCallResultFrame(DataFrame): - """Frame containing function call results for the realtime service. - - Parameters: - result_frame: The function call result frame to send to the realtime API. - """ - - result_frame: FunctionCallResultFrame +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.frames are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.frames instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index d5b663c77..fa43a720c 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -14,7 +14,6 @@ from loguru import logger from pipecat.frames.frames import ( ErrorFrame, Frame, - TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) @@ -99,16 +98,15 @@ class PiperTTSService(TTSService): await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + CHUNK_SIZE = self.chunk_size - yield TTSStartedFrame() - async for chunk in response.content.iter_chunked(CHUNK_SIZE): - # remove wav header if present - if chunk.startswith(b"RIFF"): - chunk = chunk[44:] - if len(chunk) > 0: - await self.stop_ttfb_metrics() - yield TTSAudioRawFrame(chunk, self.sample_rate, 1) + async for frame in self._stream_audio_frames_from_iterator( + response.content.iter_chunked(CHUNK_SIZE), strip_wav_header=True + ): + await self.stop_ttfb_metrics() + yield frame except Exception as e: logger.error(f"Error in run_tts: {e}") yield ErrorFrame(error=str(e)) diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 9288ebd59..925480794 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -269,6 +269,8 @@ class PlayHTTTSService(InterruptibleTTSService): raise ValueError("WebSocket URL is not a string") self._websocket = await websocket_connect(self._websocket_url) + + await self._call_event_handler("on_connected") except ValueError as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -291,6 +293,7 @@ class PlayHTTTSService(InterruptibleTTSService): finally: self._request_id = None self._websocket = None + await self._call_event_handler("on_disconnected") async def _get_websocket_url(self): """Retrieve WebSocket URL from PlayHT API.""" diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 917716545..fa3fa447d 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -255,6 +255,8 @@ class RimeTTSService(AudioContextWordTTSService): url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websocket_connect(url, additional_headers=headers) + + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -272,6 +274,7 @@ class RimeTTSService(AudioContextWordTTSService): finally: self._context_id = None self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): """Get active websocket connection or raise exception.""" @@ -553,15 +556,13 @@ class RimeHttpTTSService(TTSService): CHUNK_SIZE = self.chunk_size - async for chunk in response.content.iter_chunked(CHUNK_SIZE): - if need_to_strip_wav_header and chunk.startswith(b"RIFF"): - chunk = chunk[44:] - need_to_strip_wav_header = False + async for frame in self._stream_audio_frames_from_iterator( + response.content.iter_chunked(CHUNK_SIZE), + strip_wav_header=need_to_strip_wav_header, + ): + await self.stop_ttfb_metrics() + yield frame - if len(chunk) > 0: - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) - yield frame except Exception as e: logger.exception(f"Error generating TTS: {e}") yield ErrorFrame(error=f"Rime TTS error: {str(e)}") diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index a9fedcc58..75e6de125 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -525,6 +525,7 @@ class SarvamTTSService(InterruptibleTTSService): logger.debug("Connected to Sarvam TTS Websocket") await self._send_config() + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -556,6 +557,10 @@ class SarvamTTSService(InterruptibleTTSService): await self._websocket.close() except Exception as e: logger.error(f"{self} error closing websocket: {e}") + finally: + self._started = False + self._websocket = None + await self._call_event_handler("on_disconnected") def _get_websocket(self): if self._websocket: diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 4028dd248..2c1db2a15 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -577,6 +577,7 @@ class SpeechmaticsSTTService(STTService): ), ) logger.debug(f"{self} Connected to Speechmatics STT service") + await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} Error connecting to Speechmatics: {e}") self._client = None @@ -595,6 +596,7 @@ class SpeechmaticsSTTService(STTService): logger.error(f"{self} Error closing Speechmatics client: {e}") finally: self._client = None + await self._call_event_handler("on_disconnected") def _process_config(self) -> None: """Create a formatted STT transcription config. diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index a02619e44..6fb96c571 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -35,6 +35,25 @@ class STTService(AIService): Provides common functionality for STT services including audio passthrough, muting, settings management, and audio processing. Subclasses must implement the run_stt method to provide actual speech recognition. + + Event handlers: + on_connected: Called when connected to the STT service. + on_connected: Called when disconnected from the STT service. + on_connection_error: Called when a connection to the STT service error occurs. + + Example:: + + @stt.event_handler("on_connected") + async def on_connected(stt: STTService): + logger.debug(f"STT connected") + + @stt.event_handler("on_disconnected") + async def on_disconnected(stt: STTService): + logger.debug(f"STT disconnected") + + @stt.event_handler("on_connection_error") + async def on_connection_error(stt: STTService, error: str): + logger.error(f"STT connection error: {error}") """ def __init__( @@ -62,6 +81,10 @@ class STTService(AIService): self._muted: bool = False self._user_id: str = "" + self._register_event_handler("on_connected") + self._register_event_handler("on_disconnected") + self._register_event_handler("on_connection_error") + @property def is_muted(self) -> bool: """Check if the STT service is currently muted. @@ -292,15 +315,6 @@ class WebsocketSTTService(STTService, WebsocketService): Combines STT functionality with websocket connectivity, providing automatic error handling and reconnection capabilities. - - Event handlers: - on_connection_error: Called when a websocket connection error occurs. - - Example:: - - @stt.event_handler("on_connection_error") - async def on_connection_error(stt: STTService, error: str): - logger.error(f"STT connection error: {error}") """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): @@ -312,7 +326,6 @@ class WebsocketSTTService(STTService, WebsocketService): """ STTService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) - self._register_event_handler("on_connection_error") async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 02b80b609..b356c7244 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -8,7 +8,17 @@ import asyncio from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import ( + Any, + AsyncGenerator, + AsyncIterator, + Dict, + List, + Mapping, + Optional, + Sequence, + Tuple, +) from loguru import logger @@ -49,6 +59,25 @@ class TTSService(AIService): Provides common functionality for TTS services including text aggregation, filtering, audio generation, and frame management. Supports configurable sentence aggregation, silence insertion, and frame processing control. + + Event handlers: + on_connected: Called when connected to the STT service. + on_connected: Called when disconnected from the STT service. + on_connection_error: Called when a connection to the STT service error occurs. + + Example:: + + @tts.event_handler("on_connected") + async def on_connected(tts: TTSService): + logger.debug(f"TTS connected") + + @tts.event_handler("on_disconnected") + async def on_disconnected(tts: TTSService): + logger.debug(f"TTS disconnected") + + @tts.event_handler("on_connection_error") + async def on_connection_error(stt: TTSService, error: str): + logger.error(f"TTS connection error: {error}") """ def __init__( @@ -133,6 +162,10 @@ class TTSService(AIService): self._processing_text: bool = False + self._register_event_handler("on_connected") + self._register_event_handler("on_disconnected") + self._register_event_handler("on_connection_error") + @property def sample_rate(self) -> int: """Get the current sample rate for audio output. @@ -374,6 +407,36 @@ class TTSService(AIService): ): await self._stop_frame_queue.put(frame) + async def _stream_audio_frames_from_iterator( + self, iterator: AsyncIterator[bytes], *, strip_wav_header: bool + ) -> AsyncGenerator[Frame, None]: + buffer = bytearray() + need_to_strip_wav_header = strip_wav_header + async for chunk in iterator: + if need_to_strip_wav_header and chunk.startswith(b"RIFF"): + chunk = chunk[44:] + need_to_strip_wav_header = False + + # Append to current buffer. + buffer.extend(chunk) + + # Round to nearest even number. + aligned_length = len(buffer) & ~1 # 111111111...11110 + if aligned_length > 0: + aligned_chunk = buffer[:aligned_length] + buffer = buffer[aligned_length:] # keep any leftover byte + + if len(aligned_chunk) > 0: + frame = TTSAudioRawFrame(bytes(aligned_chunk), self.sample_rate, 1) + yield frame + + if len(buffer) > 0: + # Make sure we don't need an extra padding byte. + if len(buffer) % 2 == 1: + buffer.extend(b"\x00") + frame = TTSAudioRawFrame(bytes(buffer), self.sample_rate, 1) + yield frame + async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): self._processing_text = False await self._text_aggregator.handle_interruption() @@ -586,7 +649,6 @@ class WebsocketTTSService(TTSService, WebsocketService): """ TTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) - self._register_event_handler("on_connection_error") async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) @@ -638,15 +700,6 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): """Base class for websocket-based TTS services that support word timestamps. Combines word timestamp functionality with websocket connectivity. - - Event handlers: - on_connection_error: Called when a websocket connection error occurs. - - Example:: - - @tts.event_handler("on_connection_error") - async def on_connection_error(tts: TTSService, error: str): - logger.error(f"TTS connection error: {error}") """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): @@ -658,7 +711,6 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): """ WordTTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) - self._register_event_handler("on_connection_error") async def _report_error(self, error: ErrorFrame): await self._call_event_handler("on_connection_error", error.error) diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 25cc56f68..0f247c6dc 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -232,6 +232,9 @@ class BaseInputTransport(FrameProcessor): """ # Cancel and wait for the audio input task to finish. await self._cancel_audio_task() + # Stop audio filter. + if self._params.audio_in_filter: + await self._params.audio_in_filter.stop() async def set_transport_ready(self, frame: StartFrame): """Called when the transport is ready to stream. diff --git a/uv.lock b/uv.lock index 8dee6245e..521a3d5cd 100644 --- a/uv.lock +++ b/uv.lock @@ -891,16 +891,16 @@ wheels = [ [[package]] name = "compressed-tensors" -version = "0.10.2" +version = "0.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "torch" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/86/d43d369abc81ec63ec7b8f6f27fc8b113ea0fd18a4116ae12063387b8b34/compressed_tensors-0.10.2.tar.gz", hash = "sha256:6de13ac535d7ffdd8890fad3d229444c33076170acaa8fab6bab8ecfa96c1d8f", size = 173459, upload-time = "2025-06-23T13:19:06.135Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/eb/2229523a539e8074b238c225d168f734f6f056ab4ea2278eefe752f4a6f3/compressed_tensors-0.10.1.tar.gz", hash = "sha256:f99ce620ddcf8a657eaa7995daf5faa8e988d4b4cadc595bf2c4ff9346c2c19a", size = 126778, upload-time = "2025-06-06T18:25:16.538Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/ac/56bb4b6b3150783119479e2f05e32ebfc39ca6ff8e6fcd45eb178743b39e/compressed_tensors-0.10.2-py3-none-any.whl", hash = "sha256:e1b4d9bc2006e3fd3a938e59085f318fdb280c5af64688a4792bf1bc263e579d", size = 169030, upload-time = "2025-06-23T13:19:03.487Z" }, + { url = "https://files.pythonhosted.org/packages/5b/07/e70a0b9efc24a32740396c404e7213c62b8aeb4a577ed5a3f191f8d7806b/compressed_tensors-0.10.1-py3-none-any.whl", hash = "sha256:b8890735522c119900e8d4192cced0b0f70a98440ae070448cb699165c404659", size = 116998, upload-time = "2025-06-06T18:25:14.54Z" }, ] [[package]] @@ -2750,7 +2750,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "0.3.77" +version = "0.3.79" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -2761,23 +2761,23 @@ dependencies = [ { name = "tenacity" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/cc/786184e5f6a921a2aa4d2ac51d3adf0cd037289f3becff39644bee9654ee/langchain_core-0.3.77.tar.gz", hash = "sha256:1d6f2ad6bb98dd806c6c66a822fa93808d821e9f0348b28af0814b3a149830e7", size = 580255, upload-time = "2025-10-01T14:34:37.368Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/99/f926495f467e0f43289f12e951655d267d1eddc1136c3cf4dd907794a9a7/langchain_core-0.3.79.tar.gz", hash = "sha256:024ba54a346dd9b13fb8b2342e0c83d0111e7f26fa01f545ada23ad772b55a60", size = 580895, upload-time = "2025-10-09T21:59:08.359Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/18/e7462ae0ce57caa9f6d5d975dca861e9a751e5ca253d60a809e0d833eac3/langchain_core-0.3.77-py3-none-any.whl", hash = "sha256:9966dfe3d8365847c5fb85f97dd20e3e21b1904ae87cfd9d362b7196fb516637", size = 449525, upload-time = "2025-10-01T14:34:35.672Z" }, + { url = "https://files.pythonhosted.org/packages/fc/71/46b0efaf3fc6ad2c2bd600aef500f1cb2b7038a4042f58905805630dd29d/langchain_core-0.3.79-py3-none-any.whl", hash = "sha256:92045bfda3e741f8018e1356f83be203ec601561c6a7becfefe85be5ddc58fdb", size = 449779, upload-time = "2025-10-09T21:59:06.493Z" }, ] [[package]] name = "langchain-openai" -version = "0.3.23" +version = "0.3.29" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "openai" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/f1/575120e829430f9bdcfc2c5c4121f04b1b5a143d96e572ff32399b787ef2/langchain_openai-0.3.23.tar.gz", hash = "sha256:73411c06e04bc145db7146a6fcf33dd0f1a85130499dcae988829a4441ddaa66", size = 647923, upload-time = "2025-06-13T14:24:31.388Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/56/2e2010d15118ac52760f92ebf6ce75b3508e7a1023107ea04233fd6263e0/langchain_openai-0.3.29.tar.gz", hash = "sha256:83a0455f8ce874aa1806131ca3b4db08e482be037b7457a9b3ca21a213d2ab47", size = 766499, upload-time = "2025-08-08T15:12:32.402Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/65/88060305d5d627841bc8da7e9fb31fb603e5b103b4e5ec5b4d1a7edfbc3b/langchain_openai-0.3.23-py3-none-any.whl", hash = "sha256:624794394482c0923823f0aac44979968d77fdcfa810e42d4b0abd8096199a40", size = 65392, upload-time = "2025-06-13T14:24:30.263Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f2/a6a73beec15e90605e6a24c4498a8592d79a72c8e81c18ed0f5e9b7308e9/langchain_openai-0.3.29-py3-none-any.whl", hash = "sha256:71ae6791b3e017ec892a8062f993edc882c6665fd8385aa66e9dc3bff8205996", size = 74316, upload-time = "2025-08-08T15:12:30.794Z" }, ] [[package]] @@ -3233,23 +3233,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/b4/b61eeb92c424947675492dec3a411bdbeae307dfd78162d65ab47e8c3b4f/mlx-0.29.2-cp313-cp313-manylinux_2_35_x86_64.whl", hash = "sha256:c3b9a9aee13f346d060966472954eebe99d9f1b295c9a237c9a000f1ef9adf2c", size = 648709, upload-time = "2025-09-26T22:26:03.452Z" }, ] -[[package]] -name = "mlx-lm" -version = "0.28.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "mlx" }, - { name = "numpy" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1c/d7/fdde445c7bd443a2ed23badda6064f1477c4051543922106f365e94082cd/mlx_lm-0.28.2.tar.gz", hash = "sha256:d28752635ed5c89ff2b41361916c928e6b16f765c07b2908044e1dcaf921ed9b", size = 209374, upload-time = "2025-10-02T14:23:57.497Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/1c/89e0f60d45e364de8507065f73aeb8d2fd810d6cb95a9a512880b09399d5/mlx_lm-0.28.2-py3-none-any.whl", hash = "sha256:1501529e625d0d648216f7bb543b8b449d5fd17bd598f635536dbc1fbde6d1d6", size = 284600, upload-time = "2025-10-02T14:23:56.395Z" }, -] - [[package]] name = "mlx-metal" version = "0.29.2" @@ -3875,7 +3858,7 @@ wheels = [ [[package]] name = "openai" -version = "1.74.0" +version = "1.97.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -3887,9 +3870,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/86/c605a6e84da0248f2cebfcd864b5a6076ecf78849245af5e11d2a5ec7977/openai-1.74.0.tar.gz", hash = "sha256:592c25b8747a7cad33a841958f5eb859a785caea9ee22b9e4f4a2ec062236526", size = 427571, upload-time = "2025-04-14T16:45:25.062Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/57/1c471f6b3efb879d26686d31582997615e969f3bb4458111c9705e56332e/openai-1.97.1.tar.gz", hash = "sha256:a744b27ae624e3d4135225da9b1c89c107a2a7e5bc4c93e5b7b5214772ce7a4e", size = 494267, upload-time = "2025-07-22T13:10:12.607Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/91/8c150f16a96367e14bd7d20e86e0bbbec3080e3eb593e63f21a7f013f8e4/openai-1.74.0-py3-none-any.whl", hash = "sha256:aff3e0f9fb209836382ec112778667027f4fd6ae38bdb2334bc9e173598b092a", size = 644790, upload-time = "2025-04-14T16:45:23.041Z" }, + { url = "https://files.pythonhosted.org/packages/ee/35/412a0e9c3f0d37c94ed764b8ac7adae2d834dbd20e69f6aca582118e0f55/openai-1.97.1-py3-none-any.whl", hash = "sha256:4e96bbdf672ec3d44968c9ea39d2c375891db1acc1794668d8149d5fa6000606", size = 764380, upload-time = "2025-07-22T13:10:10.689Z" }, ] [[package]] @@ -3928,7 +3911,7 @@ wheels = [ [[package]] name = "openpipe" -version = "4.50.0" +version = "5.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anthropic" }, @@ -3937,9 +3920,9 @@ dependencies = [ { name = "openai" }, { name = "python-dateutil" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/0b/5ac4afd2253e058463fe46b44ebdf9cf153af343b457f13e9e592943c16d/openpipe-4.50.0.tar.gz", hash = "sha256:a2b1bf7a30a8d4c2cf45b85c749839ea9811e36f9d03916df8ffa343d9193a0e", size = 98954, upload-time = "2025-04-15T18:13:36.935Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/34/b487bc0ff60d3ed634e6f7bc34b5138f04e6ae319cc6578001822df93901/openpipe-5.0.0.tar.gz", hash = "sha256:040acc526fece42ba505fcedd8cd584f42482c9bd01f16b2538c9ea9c82882f4", size = 98910, upload-time = "2025-07-31T01:36:29.482Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/39/04870a3157d4ad6e8b1671f584da3e064750ccd64aa08339c6fc6dbd3a1c/openpipe-4.50.0-py3-none-any.whl", hash = "sha256:2071c3edbba3e08ceb977ad8c12d407f4da86c0c3815447fa33674d918276e5e", size = 440892, upload-time = "2025-04-15T18:13:35.258Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/516010c25a32884a87e1f8303a292f3981fa382cc7570a9ed88fb28681d5/openpipe-5.0.0-py3-none-any.whl", hash = "sha256:c04af7afb4d9bcd52e1250757dd93d0e0ed19c9ff4b524f131dd94aadf4c1a9b", size = 439951, upload-time = "2025-07-31T01:36:28.003Z" }, ] [[package]] @@ -3955,6 +3938,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/48/28ed9e55dcf2f453128df738210a980e09f4e468a456fa3c763dbc8be70a/opentelemetry_api-1.37.0-py3-none-any.whl", hash = "sha256:accf2024d3e89faec14302213bc39550ec0f4095d1cf5ca688e1bfb1c8612f47", size = 65732, upload-time = "2025-09-11T10:28:41.826Z" }, ] +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/df/47fde1de15a3d5ad410e98710fac60cd3d509df5dc7ec1359b71d6bf7e70/opentelemetry_exporter_otlp-1.37.0.tar.gz", hash = "sha256:f85b1929dd0d750751cc9159376fb05aa88bb7a08b6cdbf84edb0054d93e9f26", size = 6145, upload-time = "2025-09-11T10:29:03.075Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/23/7e35e41111e3834d918e414eca41555d585e8860c9149507298bb3b9b061/opentelemetry_exporter_otlp-1.37.0-py3-none-any.whl", hash = "sha256:bd44592c6bc7fc3e5c0a9b60f2ee813c84c2800c449e59504ab93f356cc450fc", size = 7019, upload-time = "2025-09-11T10:28:44.094Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/6c/10018cbcc1e6fff23aac67d7fd977c3d692dbe5f9ef9bb4db5c1268726cc/opentelemetry_exporter_otlp_proto_common-1.37.0.tar.gz", hash = "sha256:c87a1bdd9f41fdc408d9cc9367bb53f8d2602829659f2b90be9f9d79d0bfe62c", size = 20430, upload-time = "2025-09-11T10:29:03.605Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/13/b4ef09837409a777f3c0af2a5b4ba9b7af34872bc43609dda0c209e4060d/opentelemetry_exporter_otlp_proto_common-1.37.0-py3-none-any.whl", hash = "sha256:53038428449c559b0c564b8d718df3314da387109c4d36bd1b94c9a641b0292e", size = 18359, upload-time = "2025-09-11T10:28:44.939Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/11/4ad0979d0bb13ae5a845214e97c8d42da43980034c30d6f72d8e0ebe580e/opentelemetry_exporter_otlp_proto_grpc-1.37.0.tar.gz", hash = "sha256:f55bcb9fc848ce05ad3dd954058bc7b126624d22c4d9e958da24d8537763bec5", size = 24465, upload-time = "2025-09-11T10:29:04.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/17/46630b74751031a658706bef23ac99cdc2953cd3b2d28ec90590a0766b3e/opentelemetry_exporter_otlp_proto_grpc-1.37.0-py3-none-any.whl", hash = "sha256:aee5104835bf7993b7ddaaf380b6467472abaedb1f1dbfcc54a52a7d781a3890", size = 19305, upload-time = "2025-09-11T10:28:45.776Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/e3/6e320aeb24f951449e73867e53c55542bebbaf24faeee7623ef677d66736/opentelemetry_exporter_otlp_proto_http-1.37.0.tar.gz", hash = "sha256:e52e8600f1720d6de298419a802108a8f5afa63c96809ff83becb03f874e44ac", size = 17281, upload-time = "2025-09-11T10:29:04.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/e9/70d74a664d83976556cec395d6bfedd9b85ec1498b778367d5f93e373397/opentelemetry_exporter_otlp_proto_http-1.37.0-py3-none-any.whl", hash = "sha256:54c42b39945a6cc9d9a2a33decb876eabb9547e0dcb49df090122773447f1aef", size = 19576, upload-time = "2025-09-11T10:28:46.726Z" }, +] + [[package]] name = "opentelemetry-instrumentation" version = "0.58b0" @@ -3984,6 +4028,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/54/add1076cb37980e617723a96e29c84006983e8ad6fc589dde7f69ddc57d4/opentelemetry_instrumentation_threading-0.58b0-py3-none-any.whl", hash = "sha256:eacc072881006aceb5b9b6831bcdce718c67ef6f31ac0b32bd6a23a94d979b4a", size = 9312, upload-time = "2025-09-11T11:41:58.603Z" }, ] +[[package]] +name = "opentelemetry-proto" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/ea/a75f36b463a36f3c5a10c0b5292c58b31dbdde74f6f905d3d0ab2313987b/opentelemetry_proto-1.37.0.tar.gz", hash = "sha256:30f5c494faf66f77faeaefa35ed4443c5edb3b0aa46dad073ed7210e1a789538", size = 46151, upload-time = "2025-09-11T10:29:11.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/25/f89ea66c59bd7687e218361826c969443c4fa15dfe89733f3bf1e2a9e971/opentelemetry_proto-1.37.0-py3-none-any.whl", hash = "sha256:8ed8c066ae8828bbf0c39229979bdf583a126981142378a9cbe9d6fd5701c6e2", size = 72534, upload-time = "2025-09-11T10:28:56.831Z" }, +] + [[package]] name = "opentelemetry-sdk" version = "1.37.0" @@ -4011,6 +4067,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/90/68152b7465f50285d3ce2481b3aec2f82822e3f52e5152eeeaf516bab841/opentelemetry_semantic_conventions-0.58b0-py3-none-any.whl", hash = "sha256:5564905ab1458b96684db1340232729fce3b5375a06e140e8904c78e4f815b28", size = 207954, upload-time = "2025-09-11T10:28:59.218Z" }, ] +[[package]] +name = "opentelemetry-semantic-conventions-ai" +version = "0.4.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/e6/40b59eda51ac47009fb47afcdf37c6938594a0bd7f3b9fadcbc6058248e3/opentelemetry_semantic_conventions_ai-0.4.13.tar.gz", hash = "sha256:94efa9fb4ffac18c45f54a3a338ffeb7eedb7e1bb4d147786e77202e159f0036", size = 5368, upload-time = "2025-08-22T10:14:17.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/b5/cf25da2218910f0d6cdf7f876a06bed118c4969eacaf60a887cbaef44f44/opentelemetry_semantic_conventions_ai-0.4.13-py3-none-any.whl", hash = "sha256:883a30a6bb5deaec0d646912b5f9f6dcbb9f6f72557b73d0f2560bf25d13e2d5", size = 6080, upload-time = "2025-08-22T10:14:16.477Z" }, +] + [[package]] name = "orjson" version = "3.11.3" @@ -4604,9 +4669,9 @@ requires-dist = [ { name = "nvidia-riva-client", marker = "extra == 'riva'", specifier = "~=2.21.1" }, { name = "onnxruntime", marker = "extra == 'local-smart-turn-v3'", specifier = ">=1.20.1,<2" }, { name = "onnxruntime", marker = "extra == 'silero'", specifier = ">=1.20.1,<2" }, - { name = "openai", specifier = ">=1.74.0,<=1.99.1" }, + { name = "openai", specifier = ">=1.74.0,<3" }, { name = "opencv-python", marker = "extra == 'webrtc'", specifier = ">=4.11.0.86,<5" }, - { name = "openpipe", marker = "extra == 'openpipe'", specifier = "~=4.50.0" }, + { name = "openpipe", marker = "extra == 'openpipe'", specifier = ">=4.50.0,<6" }, { name = "opentelemetry-api", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "opentelemetry-instrumentation", marker = "extra == 'tracing'", specifier = ">=0.54b0" }, { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, @@ -4987,172 +5052,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/8b/7f9a061c1cc2b230f9ac02a6003fcd14c85ce1828013aecbaf45aa988d20/PyAudio-0.2.14-cp313-cp313-win_amd64.whl", hash = "sha256:692d8c1446f52ed2662120bcd9ddcb5aa2b71f38bda31e58b19fb4672fffba69", size = 173655, upload-time = "2024-11-20T19:12:13.616Z" }, ] -[[package]] -name = "pybase64" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/14/43297a7b7f0c1bf0c00b596f754ee3ac946128c64d21047ccf9c9bbc5165/pybase64-1.4.2.tar.gz", hash = "sha256:46cdefd283ed9643315d952fe44de80dc9b9a811ce6e3ec97fd1827af97692d0", size = 137246, upload-time = "2025-07-27T13:08:57.808Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/6d/0a7159c24ed35c8b9190b148376ad9b96598354f94ede29df74861da9ec6/pybase64-1.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82b4593b480773b17698fef33c68bae0e1c474ba07663fad74249370c46b46c9", size = 38240, upload-time = "2025-07-27T13:02:17.876Z" }, - { url = "https://files.pythonhosted.org/packages/86/2e/dad4cd832a90a49d98867e824180585e7c928504987d37304bccae11a314/pybase64-1.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a126f29d29cb4a498db179135dbf955442a0de5b00f374523f5dcceb9074ff58", size = 31658, upload-time = "2025-07-27T13:02:20.823Z" }, - { url = "https://files.pythonhosted.org/packages/1d/d8/30ea35dc2c8c568be93e1379efcaa35092e37efa2ce7f1985ccc63babee7/pybase64-1.4.2-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:1eef93c29cc5567480d168f9cc1ebd3fc3107c65787aed2019a8ea68575a33e0", size = 65963, upload-time = "2025-07-27T13:02:22.376Z" }, - { url = "https://files.pythonhosted.org/packages/f6/da/1c22f2a21d6bb9ec2a214d15ae02d5b20a95335de218a0ecbf769c535a5c/pybase64-1.4.2-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:17b871a34aaeb0644145cb6bf28feb163f593abea11aec3dbcc34a006edfc828", size = 68887, upload-time = "2025-07-27T13:02:23.606Z" }, - { url = "https://files.pythonhosted.org/packages/ac/8d/e04d489ba99b444ce94b4d5b232365d00b0f0e8564275d7ba7434dcabe72/pybase64-1.4.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1f734e16293637a35d282ce594eb05a7a90ea3ae2bc84a3496a5df9e6b890725", size = 57503, upload-time = "2025-07-27T13:02:24.83Z" }, - { url = "https://files.pythonhosted.org/packages/7e/b8/5ec9c334f30cf898709a084d596bf4b47aec2e07870f07bac5cf39754eca/pybase64-1.4.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:22bd38db2d990d5545dde83511edeec366630d00679dbd945472315c09041dc6", size = 54517, upload-time = "2025-07-27T13:02:26.006Z" }, - { url = "https://files.pythonhosted.org/packages/b9/5a/6e4424ecca041e53aa7c14525f99edd43d0117c23c5d9cb14e931458a536/pybase64-1.4.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:dc65cee686dda72007b7541b2014f33ee282459c781b9b61305bd8b9cfadc8e1", size = 57167, upload-time = "2025-07-27T13:02:27.47Z" }, - { url = "https://files.pythonhosted.org/packages/5f/d0/13f1a9467cf565eecc21dce89fb0723458d8c563d2ccfb99b96e8318dfd5/pybase64-1.4.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1e79641c420a22e49c67c046895efad05bf5f8b1dbe0dd78b4af3ab3f2923fe2", size = 57718, upload-time = "2025-07-27T13:02:28.631Z" }, - { url = "https://files.pythonhosted.org/packages/3e/34/d80335c36ad9400b18b4f92e9f680cf7646102fe4919f7bce5786a2ccb7b/pybase64-1.4.2-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:12f5e7db522ef780a8b333dab5f7d750d270b23a1684bc2235ba50756c7ba428", size = 53021, upload-time = "2025-07-27T13:02:29.823Z" }, - { url = "https://files.pythonhosted.org/packages/68/57/504ff75f7c78df28be126fe6634083d28d7f84c17e04a74a7dcb50ab2377/pybase64-1.4.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a618b1e1a63e75dd40c2a397d875935ed0835464dc55cb1b91e8f880113d0444", size = 56306, upload-time = "2025-07-27T13:02:31.314Z" }, - { url = "https://files.pythonhosted.org/packages/bf/bc/2d21cda8b73c8c9f5cd3d7e6e26dd6dfc96491052112f282332a3d5bf1d9/pybase64-1.4.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:89b0a51702c7746fa914e75e680ad697b979cdead6b418603f56a6fc9de2f50f", size = 50101, upload-time = "2025-07-27T13:02:32.662Z" }, - { url = "https://files.pythonhosted.org/packages/88/6d/51942e7737bb0711ca3e55db53924fd7f07166d79da5508ab8f5fd5972a8/pybase64-1.4.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c5161b8b82f8ba5dbbc3f76e0270622a2c2fdb9ffaf092d8f774ad7ec468c027", size = 66555, upload-time = "2025-07-27T13:02:34.122Z" }, - { url = "https://files.pythonhosted.org/packages/b6/c8/c46024d196402e7be4d3fad85336863a34816c3436c51fcf9c7c0781bf11/pybase64-1.4.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:2168de920c9b1e57850e9ff681852923a953601f73cc96a0742a42236695c316", size = 55684, upload-time = "2025-07-27T13:02:35.427Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c5/953782c9d599ff5217ee87f19e317c494cd4840afcab4c48f99cb78ca201/pybase64-1.4.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7a1e3dc977562abe40ab43483223013be71b215a5d5f3c78a666e70a5076eeec", size = 52475, upload-time = "2025-07-27T13:02:36.634Z" }, - { url = "https://files.pythonhosted.org/packages/05/fb/57d36173631aab67ca4558cdbde1047fc67a09b77f9c53addd57c7e9fdd4/pybase64-1.4.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:4cf1e8a57449e48137ef4de00a005e24c3f1cffc0aafc488e36ceb5bb2cbb1da", size = 53943, upload-time = "2025-07-27T13:02:37.777Z" }, - { url = "https://files.pythonhosted.org/packages/75/73/23e5bb0bffac0cabe2d11d1c618f6ef73da9f430da03c5249931e3c49b63/pybase64-1.4.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d8e1a381ba124f26a93d5925efbf6e6c36287fc2c93d74958e8b677c30a53fc0", size = 68411, upload-time = "2025-07-27T13:02:39.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e7/0d5c99e5e61ff5e46949a0128b49fc2c47afc0d2b815333459b17aa9d467/pybase64-1.4.2-cp310-cp310-win32.whl", hash = "sha256:8fdd9c5b60ec9a1db854f5f96bba46b80a9520069282dc1d37ff433eb8248b1f", size = 33614, upload-time = "2025-07-27T13:02:40.478Z" }, - { url = "https://files.pythonhosted.org/packages/23/40/879b6de61d7c07a2cbf76b75e9739c4938c3a1f66ac03243f2ff7ec9fb6b/pybase64-1.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:37a6c73f14c6539c0ad1aebf0cce92138af25c99a6e7aee637d9f9fc634c8a40", size = 35790, upload-time = "2025-07-27T13:02:41.864Z" }, - { url = "https://files.pythonhosted.org/packages/d2/e2/75cec12880ce3f47a79a2b9a0cdc766dc0429a7ce967bb3ab3a4b55a7f6b/pybase64-1.4.2-cp310-cp310-win_arm64.whl", hash = "sha256:b3280d03b7b361622c469d005cc270d763d9e29d0a490c26addb4f82dfe71a79", size = 30900, upload-time = "2025-07-27T13:02:43.022Z" }, - { url = "https://files.pythonhosted.org/packages/da/fb/edaa56bbf04715efc3c36966cc0150e01d7a8336c3da182f850b7fd43d32/pybase64-1.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26284ef64f142067293347bcc9d501d2b5d44b92eab9d941cb10a085fb01c666", size = 38238, upload-time = "2025-07-27T13:02:44.224Z" }, - { url = "https://files.pythonhosted.org/packages/28/a4/ca1538e9adf08f5016b3543b0060c18aea9a6e805dd20712a197c509d90d/pybase64-1.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:52dd32fe5cbfd8af8f3f034a4a65ee61948c72e5c358bf69d59543fc0dbcf950", size = 31659, upload-time = "2025-07-27T13:02:45.445Z" }, - { url = "https://files.pythonhosted.org/packages/0b/8f/f9b49926a60848ba98350dd648227ec524fb78340b47a450c4dbaf24b1bb/pybase64-1.4.2-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:37f133e8c96427995480bb6d396d9d49e949a3e829591845bb6a5a7f215ca177", size = 68318, upload-time = "2025-07-27T13:02:46.644Z" }, - { url = "https://files.pythonhosted.org/packages/29/9b/6ed2dd2bc8007f33b8316d6366b0901acbdd5665b419c2893b3dd48708de/pybase64-1.4.2-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6ee3874b0abbdd4c903d3989682a3f016fd84188622879f6f95a5dc5718d7e5", size = 71357, upload-time = "2025-07-27T13:02:47.937Z" }, - { url = "https://files.pythonhosted.org/packages/fb/69/be9ac8127da8d8339db7129683bd2975cecb0bf40a82731e1a492577a177/pybase64-1.4.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c69f177b1e404b22b05802127d6979acf4cb57f953c7de9472410f9c3fdece7", size = 59817, upload-time = "2025-07-27T13:02:49.163Z" }, - { url = "https://files.pythonhosted.org/packages/f4/a2/e3e09e000b509609276ee28b71beb0b61462d4a43b3e0db0a44c8652880c/pybase64-1.4.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:80c817e88ef2ca3cc9a285fde267690a1cb821ce0da4848c921c16f0fec56fda", size = 56639, upload-time = "2025-07-27T13:02:50.384Z" }, - { url = "https://files.pythonhosted.org/packages/01/70/ad7eff88aa4f1be06db705812e1f01749606933bf8fe9df553bb04b703e6/pybase64-1.4.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7a4bb6e7e45bfdaea0f2aaf022fc9a013abe6e46ccea31914a77e10f44098688", size = 59368, upload-time = "2025-07-27T13:02:51.883Z" }, - { url = "https://files.pythonhosted.org/packages/9d/82/0cd1b4bcd2a4da7805cfa04587be783bf9583b34ac16cadc29cf119a4fa2/pybase64-1.4.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2710a80d41a2b41293cb0e5b84b5464f54aa3f28f7c43de88784d2d9702b8a1c", size = 59981, upload-time = "2025-07-27T13:02:53.16Z" }, - { url = "https://files.pythonhosted.org/packages/3c/4c/8029a03468307dfaf0f9694d31830487ee43af5f8a73407004907724e8ac/pybase64-1.4.2-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:aa6122c8a81f6597e1c1116511f03ed42cf377c2100fe7debaae7ca62521095a", size = 54908, upload-time = "2025-07-27T13:02:54.363Z" }, - { url = "https://files.pythonhosted.org/packages/a1/8b/70bd0fe659e242efd0f60895a8ce1fe88e3a4084fd1be368974c561138c9/pybase64-1.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7e22b02505d64db308e9feeb6cb52f1d554ede5983de0befa59ac2d2ffb6a5f", size = 58650, upload-time = "2025-07-27T13:02:55.905Z" }, - { url = "https://files.pythonhosted.org/packages/64/ca/9c1d23cbc4b9beac43386a32ad53903c816063cef3f14c10d7c3d6d49a23/pybase64-1.4.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:edfe4a3c8c4007f09591f49b46a89d287ef5e8cd6630339536fe98ff077263c2", size = 52323, upload-time = "2025-07-27T13:02:57.192Z" }, - { url = "https://files.pythonhosted.org/packages/aa/29/a6292e9047248c8616dc53131a49da6c97a61616f80e1e36c73d7ef895fe/pybase64-1.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b79b4a53dd117ffbd03e96953f2e6bd2827bfe11afeb717ea16d9b0893603077", size = 68979, upload-time = "2025-07-27T13:02:58.594Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e0/cfec7b948e170395d8e88066e01f50e71195db9837151db10c14965d6222/pybase64-1.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fd9afa7a61d89d170607faf22287290045757e782089f0357b8f801d228d52c3", size = 58037, upload-time = "2025-07-27T13:02:59.753Z" }, - { url = "https://files.pythonhosted.org/packages/74/7e/0ac1850198c9c35ef631174009cee576f4d8afff3bf493ce310582976ab4/pybase64-1.4.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5c17b092e4da677a595178d2db17a5d2fafe5c8e418d46c0c4e4cde5adb8cff3", size = 54416, upload-time = "2025-07-27T13:03:00.978Z" }, - { url = "https://files.pythonhosted.org/packages/1b/45/b0b037f27e86c50e62d927f0bc1bde8b798dd55ab39197b116702e508d05/pybase64-1.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:120799274cf55f3f5bb8489eaa85142f26170564baafa7cf3e85541c46b6ab13", size = 56257, upload-time = "2025-07-27T13:03:02.201Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0d/5034598aac56336d88fd5aaf6f34630330643b51d399336b8c788d798fc5/pybase64-1.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:522e4e712686acec2d25de9759dda0b0618cb9f6588523528bc74715c0245c7b", size = 70889, upload-time = "2025-07-27T13:03:03.437Z" }, - { url = "https://files.pythonhosted.org/packages/8a/3b/0645f21bb08ecf45635b624958b5f9e569069d31ecbf125dc7e0e5b83f60/pybase64-1.4.2-cp311-cp311-win32.whl", hash = "sha256:bfd828792982db8d787515535948c1e340f1819407c8832f94384c0ebeaf9d74", size = 33631, upload-time = "2025-07-27T13:03:05.194Z" }, - { url = "https://files.pythonhosted.org/packages/8f/08/24f8103c1f19e78761026cdd9f3b3be73239bc19cf5ab6fef0e8042d0bc6/pybase64-1.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:7a9e89d40dbf833af481d1d5f1a44d173c9c4b56a7c8dba98e39a78ee87cfc52", size = 35781, upload-time = "2025-07-27T13:03:06.779Z" }, - { url = "https://files.pythonhosted.org/packages/66/cd/832fb035a0ea7eb53d776a5cfa961849e22828f6dfdfcdb9eb43ba3c0166/pybase64-1.4.2-cp311-cp311-win_arm64.whl", hash = "sha256:ce5809fa90619b03eab1cd63fec142e6cf1d361731a9b9feacf27df76c833343", size = 30903, upload-time = "2025-07-27T13:03:07.903Z" }, - { url = "https://files.pythonhosted.org/packages/28/6d/11ede991e800797b9f5ebd528013b34eee5652df93de61ffb24503393fa5/pybase64-1.4.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2c75d1388855b5a1015b65096d7dbcc708e7de3245dcbedeb872ec05a09326", size = 38326, upload-time = "2025-07-27T13:03:09.065Z" }, - { url = "https://files.pythonhosted.org/packages/fe/84/87f1f565f42e2397e2aaa2477c86419f5173c3699881c42325c090982f0a/pybase64-1.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b621a972a01841368fdb9dedc55fd3c6e0c7217d0505ba3b1ebe95e7ef1b493", size = 31661, upload-time = "2025-07-27T13:03:10.295Z" }, - { url = "https://files.pythonhosted.org/packages/cb/2a/a24c810e7a61d2cc6f73fe9ee4872a03030887fa8654150901b15f376f65/pybase64-1.4.2-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f48c32ac6a16cbf57a5a96a073fef6ff7e3526f623cd49faa112b7f9980bafba", size = 68192, upload-time = "2025-07-27T13:03:11.467Z" }, - { url = "https://files.pythonhosted.org/packages/ee/87/d9baf98cbfc37b8657290ad4421f3a3c36aa0eafe4872c5859cfb52f3448/pybase64-1.4.2-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ace8b23093a6bb862477080d9059b784096ab2f97541e8bfc40d42f062875149", size = 71587, upload-time = "2025-07-27T13:03:12.719Z" }, - { url = "https://files.pythonhosted.org/packages/0b/89/3df043cc56ef3b91b7aa0c26ae822a2d7ec8da0b0fd7c309c879b0eb5988/pybase64-1.4.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1772c7532a7fb6301baea3dd3e010148dbf70cd1136a83c2f5f91bdc94822145", size = 59910, upload-time = "2025-07-27T13:03:14.266Z" }, - { url = "https://files.pythonhosted.org/packages/75/4f/6641e9edf37aeb4d4524dc7ba2168eff8d96c90e77f6283c2be3400ab380/pybase64-1.4.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:f86f7faddcba5cbfea475f8ab96567834c28bf09ca6c7c3d66ee445adac80d8f", size = 56701, upload-time = "2025-07-27T13:03:15.6Z" }, - { url = "https://files.pythonhosted.org/packages/2d/7f/20d8ac1046f12420a0954a45a13033e75f98aade36eecd00c64e3549b071/pybase64-1.4.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0b8c8e275b5294089f314814b4a50174ab90af79d6a4850f6ae11261ff6a7372", size = 59288, upload-time = "2025-07-27T13:03:16.823Z" }, - { url = "https://files.pythonhosted.org/packages/17/ea/9c0ca570e3e50b3c6c3442e280c83b321a0464c86a9db1f982a4ff531550/pybase64-1.4.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:864d85a0470c615807ae8b97d724d068b940a2d10ac13a5f1b9e75a3ce441758", size = 60267, upload-time = "2025-07-27T13:03:18.132Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ac/46894929d71ccedebbfb0284173b0fea96bc029cd262654ba8451a7035d6/pybase64-1.4.2-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:47254d97ed2d8351e30ecfdb9e2414547f66ba73f8a09f932c9378ff75cd10c5", size = 54801, upload-time = "2025-07-27T13:03:19.669Z" }, - { url = "https://files.pythonhosted.org/packages/6a/1e/02c95218ea964f0b2469717c2c69b48e63f4ca9f18af01a5b2a29e4c1216/pybase64-1.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:264b65ecc4f0ee73f3298ab83bbd8008f7f9578361b8df5b448f985d8c63e02a", size = 58599, upload-time = "2025-07-27T13:03:20.951Z" }, - { url = "https://files.pythonhosted.org/packages/15/45/ccc21004930789b8fb439d43e3212a6c260ccddb2bf450c39a20db093f33/pybase64-1.4.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fbcc2b30cd740c16c9699f596f22c7a9e643591311ae72b1e776f2d539e9dd9d", size = 52388, upload-time = "2025-07-27T13:03:23.064Z" }, - { url = "https://files.pythonhosted.org/packages/c4/45/22e46e549710c4c237d77785b6fb1bc4c44c288a5c44237ba9daf5c34b82/pybase64-1.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cda9f79c22d51ee4508f5a43b673565f1d26af4330c99f114e37e3186fdd3607", size = 68802, upload-time = "2025-07-27T13:03:24.673Z" }, - { url = "https://files.pythonhosted.org/packages/55/0c/232c6261b81296e5593549b36e6e7884a5da008776d12665923446322c36/pybase64-1.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0c91c6d2a7232e2a1cd10b3b75a8bb657defacd4295a1e5e80455df2dfc84d4f", size = 57841, upload-time = "2025-07-27T13:03:25.948Z" }, - { url = "https://files.pythonhosted.org/packages/20/8a/b35a615ae6f04550d696bb179c414538b3b477999435fdd4ad75b76139e4/pybase64-1.4.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a370dea7b1cee2a36a4d5445d4e09cc243816c5bc8def61f602db5a6f5438e52", size = 54320, upload-time = "2025-07-27T13:03:27.495Z" }, - { url = "https://files.pythonhosted.org/packages/d3/a9/8bd4f9bcc53689f1b457ecefed1eaa080e4949d65a62c31a38b7253d5226/pybase64-1.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9aa4de83f02e462a6f4e066811c71d6af31b52d7484de635582d0e3ec3d6cc3e", size = 56482, upload-time = "2025-07-27T13:03:28.942Z" }, - { url = "https://files.pythonhosted.org/packages/75/e5/4a7735b54a1191f61c3f5c2952212c85c2d6b06eb5fb3671c7603395f70c/pybase64-1.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83a1c2f9ed00fee8f064d548c8654a480741131f280e5750bb32475b7ec8ee38", size = 70959, upload-time = "2025-07-27T13:03:30.171Z" }, - { url = "https://files.pythonhosted.org/packages/d3/67/e2b6cb32c782e12304d467418e70da0212567f42bd4d3b5eb1fdf64920ad/pybase64-1.4.2-cp312-cp312-win32.whl", hash = "sha256:a6e5688b18d558e8c6b8701cc8560836c4bbeba61d33c836b4dba56b19423716", size = 33683, upload-time = "2025-07-27T13:03:31.775Z" }, - { url = "https://files.pythonhosted.org/packages/4f/bc/d5c277496063a09707486180f17abbdbdebbf2f5c4441b20b11d3cb7dc7c/pybase64-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:c995d21b8bd08aa179cd7dd4db0695c185486ecc72da1e8f6c37ec86cadb8182", size = 35817, upload-time = "2025-07-27T13:03:32.99Z" }, - { url = "https://files.pythonhosted.org/packages/e6/69/e4be18ae685acff0ae77f75d4586590f29d2cd187bf603290cf1d635cad4/pybase64-1.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:e254b9258c40509c2ea063a7784f6994988f3f26099d6e08704e3c15dfed9a55", size = 30900, upload-time = "2025-07-27T13:03:34.499Z" }, - { url = "https://files.pythonhosted.org/packages/f4/56/5337f27a8b8d2d6693f46f7b36bae47895e5820bfa259b0072574a4e1057/pybase64-1.4.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:0f331aa59549de21f690b6ccc79360ffed1155c3cfbc852eb5c097c0b8565a2b", size = 33888, upload-time = "2025-07-27T13:03:35.698Z" }, - { url = "https://files.pythonhosted.org/packages/4c/09/f3f4b11fc9beda7e8625e29fb0f549958fcbb34fea3914e1c1d95116e344/pybase64-1.4.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:9dad20bf1f3ed9e6fe566c4c9d07d9a6c04f5a280daebd2082ffb8620b0a880d", size = 40796, upload-time = "2025-07-27T13:03:36.927Z" }, - { url = "https://files.pythonhosted.org/packages/e3/ff/470768f0fe6de0aa302a8cb1bdf2f9f5cffc3f69e60466153be68bc953aa/pybase64-1.4.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:69d3f0445b0faeef7bb7f93bf8c18d850785e2a77f12835f49e524cc54af04e7", size = 30914, upload-time = "2025-07-27T13:03:38.475Z" }, - { url = "https://files.pythonhosted.org/packages/75/6b/d328736662665e0892409dc410353ebef175b1be5eb6bab1dad579efa6df/pybase64-1.4.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2372b257b1f4dd512f317fb27e77d313afd137334de64c87de8374027aacd88a", size = 31380, upload-time = "2025-07-27T13:03:39.7Z" }, - { url = "https://files.pythonhosted.org/packages/ca/96/7ff718f87c67f4147c181b73d0928897cefa17dc75d7abc6e37730d5908f/pybase64-1.4.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:fb794502b4b1ec91c4ca5d283ae71aef65e3de7721057bd9e2b3ec79f7a62d7d", size = 38230, upload-time = "2025-07-27T13:03:41.637Z" }, - { url = "https://files.pythonhosted.org/packages/4d/58/a3307b048d799ff596a3c7c574fcba66f9b6b8c899a3c00a698124ca7ad5/pybase64-1.4.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d5c532b03fd14a5040d6cf6571299a05616f925369c72ddf6fe2fb643eb36fed", size = 38319, upload-time = "2025-07-27T13:03:42.847Z" }, - { url = "https://files.pythonhosted.org/packages/08/a7/0bda06341b0a2c830d348c6e1c4d348caaae86c53dc9a046e943467a05e9/pybase64-1.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f699514dc1d5689ca9cf378139e0214051922732f9adec9404bc680a8bef7c0", size = 31655, upload-time = "2025-07-27T13:03:44.426Z" }, - { url = "https://files.pythonhosted.org/packages/87/df/e1d6e8479e0c5113c2c63c7b44886935ce839c2d99884c7304ca9e86547c/pybase64-1.4.2-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:cd3e8713cbd32c8c6aa935feaf15c7670e2b7e8bfe51c24dc556811ebd293a29", size = 68232, upload-time = "2025-07-27T13:03:45.729Z" }, - { url = "https://files.pythonhosted.org/packages/71/ab/db4dbdfccb9ca874d6ce34a0784761471885d96730de85cee3d300381529/pybase64-1.4.2-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d377d48acf53abf4b926c2a7a24a19deb092f366a04ffd856bf4b3aa330b025d", size = 71608, upload-time = "2025-07-27T13:03:47.01Z" }, - { url = "https://files.pythonhosted.org/packages/11/e9/508df958563951045d728bbfbd3be77465f9231cf805cb7ccaf6951fc9f1/pybase64-1.4.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d83c076e78d619b9e1dd674e2bf5fb9001aeb3e0b494b80a6c8f6d4120e38cd9", size = 59912, upload-time = "2025-07-27T13:03:48.277Z" }, - { url = "https://files.pythonhosted.org/packages/f2/58/7f2cef1ceccc682088958448d56727369de83fa6b29148478f4d2acd107a/pybase64-1.4.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:ab9cdb6a8176a5cb967f53e6ad60e40c83caaa1ae31c5e1b29e5c8f507f17538", size = 56413, upload-time = "2025-07-27T13:03:49.908Z" }, - { url = "https://files.pythonhosted.org/packages/08/7c/7e0af5c5728fa7e2eb082d88eca7c6bd17429be819d58518e74919d42e66/pybase64-1.4.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:adf0c103ad559dbfb9fe69edfd26a15c65d9c991a5ab0a25b04770f9eb0b9484", size = 59311, upload-time = "2025-07-27T13:03:51.238Z" }, - { url = "https://files.pythonhosted.org/packages/03/8b/09825d0f37e45b9a3f546e5f990b6cf2dd838e54ea74122c2464646e0c77/pybase64-1.4.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:0d03ef2f253d97ce0685d3624bf5e552d716b86cacb8a6c971333ba4b827e1fc", size = 60282, upload-time = "2025-07-27T13:03:52.56Z" }, - { url = "https://files.pythonhosted.org/packages/9c/3f/3711d2413f969bfd5b9cc19bc6b24abae361b7673ff37bcb90c43e199316/pybase64-1.4.2-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:e565abf906efee76ae4be1aef5df4aed0fda1639bc0d7732a3dafef76cb6fc35", size = 54845, upload-time = "2025-07-27T13:03:54.167Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3c/4c7ce1ae4d828c2bb56d144322f81bffbaaac8597d35407c3d7cbb0ff98f/pybase64-1.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3c6a5f15fd03f232fc6f295cce3684f7bb08da6c6d5b12cc771f81c9f125cc6", size = 58615, upload-time = "2025-07-27T13:03:55.494Z" }, - { url = "https://files.pythonhosted.org/packages/f5/8f/c2fc03bf4ed038358620065c75968a30184d5d3512d09d3ef9cc3bd48592/pybase64-1.4.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:bad9e3db16f448728138737bbd1af9dc2398efd593a8bdd73748cc02cd33f9c6", size = 52434, upload-time = "2025-07-27T13:03:56.808Z" }, - { url = "https://files.pythonhosted.org/packages/e2/0a/757d6df0a60327c893cfae903e15419914dd792092dc8cc5c9523d40bc9b/pybase64-1.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2683ef271328365c31afee0ed8fa29356fb8fb7c10606794656aa9ffb95e92be", size = 68824, upload-time = "2025-07-27T13:03:58.735Z" }, - { url = "https://files.pythonhosted.org/packages/a0/14/84abe2ed8c29014239be1cfab45dfebe5a5ca779b177b8b6f779bd8b69da/pybase64-1.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:265b20089cd470079114c09bb74b101b3bfc3c94ad6b4231706cf9eff877d570", size = 57898, upload-time = "2025-07-27T13:04:00.379Z" }, - { url = "https://files.pythonhosted.org/packages/7e/c6/d193031f90c864f7b59fa6d1d1b5af41f0f5db35439988a8b9f2d1b32a13/pybase64-1.4.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e53173badead10ef8b839aa5506eecf0067c7b75ad16d9bf39bc7144631f8e67", size = 54319, upload-time = "2025-07-27T13:04:01.742Z" }, - { url = "https://files.pythonhosted.org/packages/cb/37/ec0c7a610ff8f994ee6e0c5d5d66b6b6310388b96ebb347b03ae39870fdf/pybase64-1.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5823b8dcf74da7da0f761ed60c961e8928a6524e520411ad05fe7f9f47d55b40", size = 56472, upload-time = "2025-07-27T13:04:03.089Z" }, - { url = "https://files.pythonhosted.org/packages/c4/5a/e585b74f85cedd261d271e4c2ef333c5cfce7e80750771808f56fee66b98/pybase64-1.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1237f66c54357d325390da60aa5e21c6918fbcd1bf527acb9c1f4188c62cb7d5", size = 70966, upload-time = "2025-07-27T13:04:04.361Z" }, - { url = "https://files.pythonhosted.org/packages/ad/20/1b2fdd98b4ba36008419668c813025758214c543e362c66c49214ecd1127/pybase64-1.4.2-cp313-cp313-win32.whl", hash = "sha256:b0b851eb4f801d16040047f6889cca5e9dfa102b3e33f68934d12511245cef86", size = 33681, upload-time = "2025-07-27T13:04:06.126Z" }, - { url = "https://files.pythonhosted.org/packages/ff/64/3df4067d169c047054889f34b5a946cbe3785bca43404b93c962a5461a41/pybase64-1.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:19541c6e26d17d9522c02680fe242206ae05df659c82a657aabadf209cd4c6c7", size = 35822, upload-time = "2025-07-27T13:04:07.752Z" }, - { url = "https://files.pythonhosted.org/packages/d1/fd/db505188adf812e60ee923f196f9deddd8a1895b2b29b37f5db94afc3b1c/pybase64-1.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:77a191863d576c0a5dd81f8a568a5ca15597cc980ae809dce62c717c8d42d8aa", size = 30899, upload-time = "2025-07-27T13:04:09.062Z" }, - { url = "https://files.pythonhosted.org/packages/d9/27/5f5fecd206ec1e06e1608a380af18dcb76a6ab08ade6597a3251502dcdb2/pybase64-1.4.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2e194bbabe3fdf9e47ba9f3e157394efe0849eb226df76432126239b3f44992c", size = 38677, upload-time = "2025-07-27T13:04:10.334Z" }, - { url = "https://files.pythonhosted.org/packages/bf/0f/abe4b5a28529ef5f74e8348fa6a9ef27d7d75fbd98103d7664cf485b7d8f/pybase64-1.4.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:39aef1dadf4a004f11dd09e703abaf6528a87c8dbd39c448bb8aebdc0a08c1be", size = 32066, upload-time = "2025-07-27T13:04:11.641Z" }, - { url = "https://files.pythonhosted.org/packages/ac/7e/ea0ce6a7155cada5526017ec588b6d6185adea4bf9331565272f4ef583c2/pybase64-1.4.2-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:91cb920c7143e36ec8217031282c8651da3b2206d70343f068fac0e7f073b7f9", size = 72300, upload-time = "2025-07-27T13:04:12.969Z" }, - { url = "https://files.pythonhosted.org/packages/45/2d/e64c7a056c9ec48dfe130d1295e47a8c2b19c3984488fc08e5eaa1e86c88/pybase64-1.4.2-cp313-cp313t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6958631143fb9e71f9842000da042ec2f6686506b6706e2dfda29e97925f6aa0", size = 75520, upload-time = "2025-07-27T13:04:14.374Z" }, - { url = "https://files.pythonhosted.org/packages/43/e0/e5f93b2e1cb0751a22713c4baa6c6eaf5f307385e369180486c8316ed21e/pybase64-1.4.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dc35f14141ef3f1ac70d963950a278a2593af66fe5a1c7a208e185ca6278fa25", size = 65384, upload-time = "2025-07-27T13:04:16.204Z" }, - { url = "https://files.pythonhosted.org/packages/ff/23/8c645a1113ad88a1c6a3d0e825e93ef8b74ad3175148767853a0a4d7626e/pybase64-1.4.2-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5d949d2d677859c3a8507e1b21432a039d2b995e0bd3fe307052b6ded80f207a", size = 60471, upload-time = "2025-07-27T13:04:17.947Z" }, - { url = "https://files.pythonhosted.org/packages/8b/81/edd0f7d8b0526b91730a0dd4ce6b4c8be2136cd69d424afe36235d2d2a06/pybase64-1.4.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:09caacdd3e15fe7253a67781edd10a6a918befab0052a2a3c215fe5d1f150269", size = 63945, upload-time = "2025-07-27T13:04:19.383Z" }, - { url = "https://files.pythonhosted.org/packages/a5/a5/edc224cd821fd65100b7af7c7e16b8f699916f8c0226c9c97bbae5a75e71/pybase64-1.4.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:e44b0e793b23f28ea0f15a9754bd0c960102a2ac4bccb8fafdedbd4cc4d235c0", size = 64858, upload-time = "2025-07-27T13:04:20.807Z" }, - { url = "https://files.pythonhosted.org/packages/11/3b/92853f968f1af7e42b7e54d21bdd319097b367e7dffa2ca20787361df74c/pybase64-1.4.2-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:849f274d0bcb90fc6f642c39274082724d108e41b15f3a17864282bd41fc71d5", size = 58557, upload-time = "2025-07-27T13:04:22.229Z" }, - { url = "https://files.pythonhosted.org/packages/76/09/0ec6bd2b2303b0ea5c6da7535edc9a608092075ef8c0cdd96e3e726cd687/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:528dba7ef1357bd7ce1aea143084501f47f5dd0fff7937d3906a68565aa59cfe", size = 63624, upload-time = "2025-07-27T13:04:23.952Z" }, - { url = "https://files.pythonhosted.org/packages/73/6e/52cb1ced2a517a3118b2e739e9417432049013ac7afa15d790103059e8e4/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:1da54be743d9a68671700cfe56c3ab8c26e8f2f5cc34eface905c55bc3a9af94", size = 56174, upload-time = "2025-07-27T13:04:25.419Z" }, - { url = "https://files.pythonhosted.org/packages/5b/9d/820fe79347467e48af985fe46180e1dd28e698ade7317bebd66de8a143f5/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:9b07c0406c3eaa7014499b0aacafb21a6d1146cfaa85d56f0aa02e6d542ee8f3", size = 72640, upload-time = "2025-07-27T13:04:26.824Z" }, - { url = "https://files.pythonhosted.org/packages/53/58/e863e10d08361e694935c815b73faad7e1ab03f99ae154d86c4e2f331896/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:312f2aa4cf5d199a97fbcaee75d2e59ebbaafcd091993eb373b43683498cdacb", size = 62453, upload-time = "2025-07-27T13:04:28.562Z" }, - { url = "https://files.pythonhosted.org/packages/95/f0/c392c4ac8ccb7a34b28377c21faa2395313e3c676d76c382642e19a20703/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad59362fc267bf15498a318c9e076686e4beeb0dfe09b457fabbc2b32468b97a", size = 58103, upload-time = "2025-07-27T13:04:29.996Z" }, - { url = "https://files.pythonhosted.org/packages/32/30/00ab21316e7df8f526aa3e3dc06f74de6711d51c65b020575d0105a025b2/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:01593bd064e7dcd6c86d04e94e44acfe364049500c20ac68ca1e708fbb2ca970", size = 60779, upload-time = "2025-07-27T13:04:31.549Z" }, - { url = "https://files.pythonhosted.org/packages/a6/65/114ca81839b1805ce4a2b7d58bc16e95634734a2059991f6382fc71caf3e/pybase64-1.4.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5b81547ad8ea271c79fdf10da89a1e9313cb15edcba2a17adf8871735e9c02a0", size = 74684, upload-time = "2025-07-27T13:04:32.976Z" }, - { url = "https://files.pythonhosted.org/packages/54/8f/aa9d445b9bb693b8f6bb1456bd6d8576d79b7a63bf6c69af3a539235b15f/pybase64-1.4.2-cp313-cp313t-win32.whl", hash = "sha256:7edbe70b5654545a37e6e6b02de738303b1bbdfcde67f6cfec374cfb5cc4099e", size = 33961, upload-time = "2025-07-27T13:04:34.806Z" }, - { url = "https://files.pythonhosted.org/packages/0e/e5/da37cfb173c646fd4fc7c6aae2bc41d40de2ee49529854af8f4e6f498b45/pybase64-1.4.2-cp313-cp313t-win_amd64.whl", hash = "sha256:385690addf87c25d6366fab5d8ff512eed8a7ecb18da9e8152af1c789162f208", size = 36199, upload-time = "2025-07-27T13:04:36.223Z" }, - { url = "https://files.pythonhosted.org/packages/66/3e/1eb68fb7d00f2cec8bd9838e2a30d183d6724ae06e745fd6e65216f170ff/pybase64-1.4.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c2070d0aa88580f57fe15ca88b09f162e604d19282915a95a3795b5d3c1c05b5", size = 31221, upload-time = "2025-07-27T13:04:37.704Z" }, - { url = "https://files.pythonhosted.org/packages/99/bf/00a87d951473ce96c8c08af22b6983e681bfabdb78dd2dcf7ee58eac0932/pybase64-1.4.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4157ad277a32cf4f02a975dffc62a3c67d73dfa4609b2c1978ef47e722b18b8e", size = 30924, upload-time = "2025-07-27T13:04:39.189Z" }, - { url = "https://files.pythonhosted.org/packages/ae/43/dee58c9d60e60e6fb32dc6da722d84592e22f13c277297eb4ce6baf99a99/pybase64-1.4.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:e113267dc349cf624eb4f4fbf53fd77835e1aa048ac6877399af426aab435757", size = 31390, upload-time = "2025-07-27T13:04:40.995Z" }, - { url = "https://files.pythonhosted.org/packages/e1/11/b28906fc2e330b8b1ab4bc845a7bef808b8506734e90ed79c6062b095112/pybase64-1.4.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:cea5aaf218fd9c5c23afacfe86fd4464dfedc1a0316dd3b5b4075b068cc67df0", size = 38212, upload-time = "2025-07-27T13:04:42.729Z" }, - { url = "https://files.pythonhosted.org/packages/24/9e/868d1e104413d14b19feaf934fc7fad4ef5b18946385f8bb79684af40f24/pybase64-1.4.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:41213497abbd770435c7a9c8123fb02b93709ac4cf60155cd5aefc5f3042b600", size = 38303, upload-time = "2025-07-27T13:04:44.095Z" }, - { url = "https://files.pythonhosted.org/packages/a3/73/f7eac96ca505df0600280d6bfc671a9e2e2f947c2b04b12a70e36412f7eb/pybase64-1.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8b522df7ee00f2ac1993ccd5e1f6608ae7482de3907668c2ff96a83ef213925", size = 31669, upload-time = "2025-07-27T13:04:45.845Z" }, - { url = "https://files.pythonhosted.org/packages/c6/43/8e18bea4fd455100112d6a73a83702843f067ef9b9272485b6bdfd9ed2f0/pybase64-1.4.2-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:06725022e540c5b098b978a0418ca979773e2cbdbb76f10bd97536f2ad1c5b49", size = 68452, upload-time = "2025-07-27T13:04:47.788Z" }, - { url = "https://files.pythonhosted.org/packages/e4/2e/851eb51284b97354ee5dfa1309624ab90920696e91a33cd85b13d20cc5c1/pybase64-1.4.2-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a3e54dcf0d0305ec88473c9d0009f698cabf86f88a8a10090efeff2879c421bb", size = 71674, upload-time = "2025-07-27T13:04:49.294Z" }, - { url = "https://files.pythonhosted.org/packages/57/0d/5cf1e5dc64aec8db43e8dee4e4046856d639a72bcb0fb3e716be42ced5f1/pybase64-1.4.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67675cee727a60dc91173d2790206f01aa3c7b3fbccfa84fd5c1e3d883fe6caa", size = 60027, upload-time = "2025-07-27T13:04:50.769Z" }, - { url = "https://files.pythonhosted.org/packages/a4/8e/3479266bc0e65f6cc48b3938d4a83bff045330649869d950a378f2ddece0/pybase64-1.4.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:753da25d4fd20be7bda2746f545935773beea12d5cb5ec56ec2d2960796477b1", size = 56461, upload-time = "2025-07-27T13:04:52.37Z" }, - { url = "https://files.pythonhosted.org/packages/20/b6/f2b6cf59106dd78bae8717302be5b814cec33293504ad409a2eb752ad60c/pybase64-1.4.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a78c768ce4ca550885246d14babdb8923e0f4a848dfaaeb63c38fc99e7ea4052", size = 59446, upload-time = "2025-07-27T13:04:53.967Z" }, - { url = "https://files.pythonhosted.org/packages/16/70/3417797dfccdfdd0a54e4ad17c15b0624f0fc2d6a362210f229f5c4e8fd0/pybase64-1.4.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:51b17f36d890c92f0618fb1c8db2ccc25e6ed07afa505bab616396fc9b0b0492", size = 60350, upload-time = "2025-07-27T13:04:55.881Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c6/6e4269dd98d150ae95d321b311a345eae0f7fd459d97901b4a586d7513bb/pybase64-1.4.2-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:f92218d667049ab4f65d54fa043a88ffdb2f07fff1f868789ef705a5221de7ec", size = 54989, upload-time = "2025-07-27T13:04:57.436Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e8/18c1b0c255f964fafd0412b0d5a163aad588aeccb8f84b9bf9c8611d80f6/pybase64-1.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3547b3d1499919a06491b3f879a19fbe206af2bd1a424ecbb4e601eb2bd11fea", size = 58724, upload-time = "2025-07-27T13:04:59.406Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ad/ddfbd2125fc20b94865fb232b2e9105376fa16eee492e4b7786d42a86cbf/pybase64-1.4.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:958af7b0e09ddeb13e8c2330767c47b556b1ade19c35370f6451d139cde9f2a9", size = 52285, upload-time = "2025-07-27T13:05:01.198Z" }, - { url = "https://files.pythonhosted.org/packages/b6/4c/b9d4ec9224add33c84b925a03d1a53cd4106efb449ea8e0ae7795fed7bf7/pybase64-1.4.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4facc57f6671e2229a385a97a618273e7be36a9ea0a9d1c1b9347f14d19ceba8", size = 69036, upload-time = "2025-07-27T13:05:03.109Z" }, - { url = "https://files.pythonhosted.org/packages/92/38/7b96794da77bed3d9b4fea40f14ae563648fba83a696e7602fabe60c0eb7/pybase64-1.4.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a32fc57d05d73a7c9b0ca95e9e265e21cf734195dc6873829a890058c35f5cfd", size = 57938, upload-time = "2025-07-27T13:05:04.744Z" }, - { url = "https://files.pythonhosted.org/packages/eb/c5/ae8bbce3c322d1b074e79f51f5df95961fe90cb8748df66c6bc97616e974/pybase64-1.4.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3dc853243c81ce89cc7318e6946f860df28ddb7cd2a0648b981652d9ad09ee5a", size = 54474, upload-time = "2025-07-27T13:05:06.662Z" }, - { url = "https://files.pythonhosted.org/packages/15/9a/c09887c4bb1b43c03fc352e2671ef20c6686c6942a99106a45270ee5b840/pybase64-1.4.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0e6d863a86b3e7bc6ac9bd659bebda4501b9da842521111b0b0e54eb51295df5", size = 56533, upload-time = "2025-07-27T13:05:08.368Z" }, - { url = "https://files.pythonhosted.org/packages/4f/0f/d5114d63d35d085639606a880cb06e2322841cd4b213adfc14d545c1186f/pybase64-1.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6579475140ff2067903725d8aca47f5747bcb211597a1edd60b58f6d90ada2bd", size = 71030, upload-time = "2025-07-27T13:05:10.3Z" }, - { url = "https://files.pythonhosted.org/packages/40/0e/fe6f1ed22ea52eb99f490a8441815ba21de288f4351aeef4968d71d20d2d/pybase64-1.4.2-cp314-cp314-win32.whl", hash = "sha256:373897f728d7b4f241a1f803ac732c27b6945d26d86b2741ad9b75c802e4e378", size = 34174, upload-time = "2025-07-27T13:05:12.254Z" }, - { url = "https://files.pythonhosted.org/packages/71/46/0e15bea52ffc63e8ae7935e945accbaf635e0aefa26d3e31fdf9bc9dcd01/pybase64-1.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:1afe3361344617d298c1d08bc657ef56d0f702d6b72cb65d968b2771017935aa", size = 36308, upload-time = "2025-07-27T13:05:13.898Z" }, - { url = "https://files.pythonhosted.org/packages/4f/dc/55849fee2577bda77c1e078da04cc9237e8e474a8c8308deb702a26f2511/pybase64-1.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:f131c9360babe522f3d90f34da3f827cba80318125cf18d66f2ee27e3730e8c4", size = 31341, upload-time = "2025-07-27T13:05:15.553Z" }, - { url = "https://files.pythonhosted.org/packages/39/44/c69d088e28b25e70ac742b6789cde038473815b2a69345c4bae82d5e244d/pybase64-1.4.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2583ac304131c1bd6e3120b0179333610f18816000db77c0a2dd6da1364722a8", size = 38678, upload-time = "2025-07-27T13:05:17.544Z" }, - { url = "https://files.pythonhosted.org/packages/00/93/2860ec067497b9cbb06242f96d44caebbd9eed32174e4eb8c1ffef760f94/pybase64-1.4.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:75a8116be4ea4cdd30a5c4f1a6f3b038e0d457eb03c8a2685d8ce2aa00ef8f92", size = 32066, upload-time = "2025-07-27T13:05:19.18Z" }, - { url = "https://files.pythonhosted.org/packages/d3/55/1e96249a38759332e8a01b31c370d88c60ceaf44692eb6ba4f0f451ee496/pybase64-1.4.2-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:217ea776a098d7c08668e5526b9764f5048bbfd28cac86834217ddfe76a4e3c4", size = 72465, upload-time = "2025-07-27T13:05:20.866Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ab/0f468605b899f3e35dbb7423fba3ff98aeed1ec16abb02428468494a58f4/pybase64-1.4.2-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4ec14683e343c95b14248cdfdfa78c052582be7a3865fd570aa7cffa5ab5cf37", size = 75693, upload-time = "2025-07-27T13:05:22.896Z" }, - { url = "https://files.pythonhosted.org/packages/91/d1/9980a0159b699e2489baba05b71b7c953b29249118ba06fdbb3e9ea1b9b5/pybase64-1.4.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:480ecf21e1e956c5a10d3cf7b3b7e75bce3f9328cf08c101e4aab1925d879f34", size = 65577, upload-time = "2025-07-27T13:05:25Z" }, - { url = "https://files.pythonhosted.org/packages/16/86/b27e7b95f9863d245c0179a7245582eda3d262669d8f822777364d8fd7d5/pybase64-1.4.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:1fe1ebdc55e9447142e2f6658944aadfb5a4fbf03dbd509be34182585515ecc1", size = 60662, upload-time = "2025-07-27T13:05:27.138Z" }, - { url = "https://files.pythonhosted.org/packages/28/87/a7f0dde0abc26bfbee761f1d3558eb4b139f33ddd9fe1f6825ffa7daa22d/pybase64-1.4.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c793a2b06753accdaf5e1a8bbe5d800aab2406919e5008174f989a1ca0081411", size = 64179, upload-time = "2025-07-27T13:05:28.996Z" }, - { url = "https://files.pythonhosted.org/packages/1e/88/5d6fa1c60e1363b4cac4c396978f39e9df4689e75225d7d9c0a5998e3a14/pybase64-1.4.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6acae6e1d1f7ebe40165f08076c7a73692b2bf9046fefe673f350536e007f556", size = 64968, upload-time = "2025-07-27T13:05:30.818Z" }, - { url = "https://files.pythonhosted.org/packages/20/6e/2ed585af5b2211040445d9849326dd2445320c9316268794f5453cfbaf30/pybase64-1.4.2-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:88b91cd0949358aadcea75f8de5afbcf3c8c5fb9ec82325bd24285b7119cf56e", size = 58738, upload-time = "2025-07-27T13:05:32.629Z" }, - { url = "https://files.pythonhosted.org/packages/ce/94/e2960b56322eabb3fbf303fc5a72e6444594c1b90035f3975c6fe666db5c/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:53316587e1b1f47a11a5ff068d3cbd4a3911c291f2aec14882734973684871b2", size = 63802, upload-time = "2025-07-27T13:05:34.687Z" }, - { url = "https://files.pythonhosted.org/packages/95/47/312139d764c223f534f751528ce3802887c279125eac64f71cd3b4e05abc/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:caa7f20f43d00602cf9043b5ba758d54f5c41707d3709b2a5fac17361579c53c", size = 56341, upload-time = "2025-07-27T13:05:36.554Z" }, - { url = "https://files.pythonhosted.org/packages/3f/d7/aec9a6ed53b128dac32f8768b646ca5730c88eef80934054d7fa7d02f3ef/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:2d93817e24fdd79c534ed97705df855af6f1d2535ceb8dfa80da9de75482a8d7", size = 72838, upload-time = "2025-07-27T13:05:38.459Z" }, - { url = "https://files.pythonhosted.org/packages/e3/a8/6ccc54c5f1f7c3450ad7c56da10c0f131d85ebe069ea6952b5b42f2e92d9/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:63cd769b51474d8d08f7f2ce73b30380d9b4078ec92ea6b348ea20ed1e1af88a", size = 62633, upload-time = "2025-07-27T13:05:40.624Z" }, - { url = "https://files.pythonhosted.org/packages/34/22/2b9d89f8ff6f2a01d6d6a88664b20a4817049cfc3f2c62caca040706660c/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cd07e6a9993c392ec8eb03912a43c6a6b21b2deb79ee0d606700fe276e9a576f", size = 58282, upload-time = "2025-07-27T13:05:42.565Z" }, - { url = "https://files.pythonhosted.org/packages/b2/14/dbf6266177532a6a11804ac080ebffcee272f491b92820c39886ee20f201/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6a8944e8194adff4668350504bc6b7dbde2dab9244c88d99c491657d145b5af5", size = 60948, upload-time = "2025-07-27T13:05:44.48Z" }, - { url = "https://files.pythonhosted.org/packages/fd/7a/b2ae9046a66dd5746cd72836a41386517b1680bea5ce02f2b4f1c9ebc688/pybase64-1.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:04ab398ec4b6a212af57f6a21a6336d5a1d754ff4ccb215951366ab9080481b2", size = 74854, upload-time = "2025-07-27T13:05:46.416Z" }, - { url = "https://files.pythonhosted.org/packages/ef/7e/9856f6d6c38a7b730e001123d2d9fa816b8b1a45f0cdee1d509d5947b047/pybase64-1.4.2-cp314-cp314t-win32.whl", hash = "sha256:3b9201ecdcb1c3e23be4caebd6393a4e6615bd0722528f5413b58e22e3792dd3", size = 34490, upload-time = "2025-07-27T13:05:48.304Z" }, - { url = "https://files.pythonhosted.org/packages/c7/38/8523a9dc1ec8704dedbe5ccc95192ae9a7585f7eec85cc62946fe3cacd32/pybase64-1.4.2-cp314-cp314t-win_amd64.whl", hash = "sha256:36e9b0cad8197136d73904ef5a71d843381d063fd528c5ab203fc4990264f682", size = 36680, upload-time = "2025-07-27T13:05:50.264Z" }, - { url = "https://files.pythonhosted.org/packages/3c/52/5600104ef7b85f89fb8ec54f73504ead3f6f0294027e08d281f3cafb5c1a/pybase64-1.4.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f25140496b02db0e7401567cd869fb13b4c8118bf5c2428592ec339987146d8b", size = 31600, upload-time = "2025-07-27T13:05:52.24Z" }, - { url = "https://files.pythonhosted.org/packages/32/34/b67371f4fcedd5e2def29b1cf92a4311a72f590c04850f370c75297b48ce/pybase64-1.4.2-graalpy311-graalpy242_311_native-macosx_10_9_x86_64.whl", hash = "sha256:b4eed40a5f1627ee65613a6ac834a33f8ba24066656f569c852f98eb16f6ab5d", size = 38667, upload-time = "2025-07-27T13:07:25.315Z" }, - { url = "https://files.pythonhosted.org/packages/aa/3e/e57fe09ed1c7e740d21c37023c5f7c8963b4c36380f41d10261cc76f93b4/pybase64-1.4.2-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:57885fa521e9add235af4db13e9e048d3a2934cd27d7c5efac1925e1b4d6538d", size = 32094, upload-time = "2025-07-27T13:07:28.235Z" }, - { url = "https://files.pythonhosted.org/packages/51/34/f40d3262c3953814b9bcdcf858436bd5bc1133a698be4bcc7ed2a8c0730d/pybase64-1.4.2-graalpy311-graalpy242_311_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eef9255d926c64e2fca021d3aee98023bacb98e1518e5986d6aab04102411b04", size = 43212, upload-time = "2025-07-27T13:07:31.327Z" }, - { url = "https://files.pythonhosted.org/packages/8c/2a/5e05d25718cb8ffd68bd46553ddfd2b660893d937feda1716b8a3b21fb38/pybase64-1.4.2-graalpy311-graalpy242_311_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89614ea2d2329b6708746c540e0f14d692125df99fb1203ff0de948d9e68dfc9", size = 35789, upload-time = "2025-07-27T13:07:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/d5/9d/f56c3ee6e94faaae2896ecaf666428330cb24096abf7d2427371bb2b403a/pybase64-1.4.2-graalpy311-graalpy242_311_native-win_amd64.whl", hash = "sha256:e401cecd2d7ddcd558768b2140fd4430746be4d17fb14c99eec9e40789df136d", size = 35861, upload-time = "2025-07-27T13:07:37.099Z" }, - { url = "https://files.pythonhosted.org/packages/fb/04/bfe2bd0d76385750f3541724b4abfe4ea111b3cc01ff7e83f410054adc30/pybase64-1.4.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b29c93414ba965777643a9d98443f08f76ac04519ad717aa859113695372a07", size = 38226, upload-time = "2025-07-27T13:07:40.121Z" }, - { url = "https://files.pythonhosted.org/packages/22/13/c717855760b78ded1a9d308984c7e3e99fcf79c6cac5a231ed8c1238218f/pybase64-1.4.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5e0c3353c0bf099c5c3f8f750202c486abee8f23a566b49e9e7b1222fbf5f259", size = 31524, upload-time = "2025-07-27T13:07:43.946Z" }, - { url = "https://files.pythonhosted.org/packages/cf/da/2b7e69abfc62abe4d54b10d1e09ec78021a6b9b2d7e6e7b632243a19433e/pybase64-1.4.2-pp310-pypy310_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4f98c5c6152d3c01d933fcde04322cd9ddcf65b5346034aac69a04c1a7cbb012", size = 40667, upload-time = "2025-07-27T13:07:46.715Z" }, - { url = "https://files.pythonhosted.org/packages/f1/11/ba738655fb3ba85c7a0605eddd2709fef606e654840c72ee5c5ff7ab29bf/pybase64-1.4.2-pp310-pypy310_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9096a4977b7aff7ef250f759fb6a4b6b7b6199d99c84070c7fc862dd3b208b34", size = 41290, upload-time = "2025-07-27T13:07:49.534Z" }, - { url = "https://files.pythonhosted.org/packages/5d/38/2d5502fcaf712297b95c1b6ca924656dd7d17501fd7f9c9e0b3bbf8892ef/pybase64-1.4.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49d8597e2872966399410502310b1e2a5b7e8d8ba96766ee1fe242e00bd80775", size = 35438, upload-time = "2025-07-27T13:07:52.327Z" }, - { url = "https://files.pythonhosted.org/packages/b6/db/e03b8b6daa60a3fbef21741403e0cf18b2aff3beebdf6e3596bb9bab16c7/pybase64-1.4.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2ef16366565389a287df82659e055e88bdb6c36e46a3394950903e0a9cb2e5bf", size = 36121, upload-time = "2025-07-27T13:07:55.54Z" }, - { url = "https://files.pythonhosted.org/packages/0e/bf/5ebaa2d9ddb5fc506633bc8b820fc27e64da964937fb30929c0367c47d00/pybase64-1.4.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0a5393be20b0705870f5a8969749af84d734c077de80dd7e9f5424a247afa85e", size = 38162, upload-time = "2025-07-27T13:07:58.364Z" }, - { url = "https://files.pythonhosted.org/packages/25/41/795c5fd6e5571bb675bf9add8a048166dddf8951c2a903fea8557743886b/pybase64-1.4.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:448f0259a2f1a17eb086f70fe2ad9b556edba1fc5bc4e62ce6966179368ee9f8", size = 31452, upload-time = "2025-07-27T13:08:01.259Z" }, - { url = "https://files.pythonhosted.org/packages/aa/dd/c819003b59b2832256b72ad23cbeadbd95d083ef0318d07149a58b7a88af/pybase64-1.4.2-pp311-pypy311_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:1159e70cba8e76c3d8f334bd1f8fd52a1bb7384f4c3533831b23ab2df84a6ef3", size = 40668, upload-time = "2025-07-27T13:08:04.176Z" }, - { url = "https://files.pythonhosted.org/packages/0e/c5/38c6aba28678c4a4db49312a6b8171b93a0ffe9f21362cf4c0f325caa850/pybase64-1.4.2-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7d943bc5dad8388971494554b97f22ae06a46cc7779ad0de3d4bfdf7d0bbea30", size = 41281, upload-time = "2025-07-27T13:08:07.395Z" }, - { url = "https://files.pythonhosted.org/packages/e5/23/5927bd9e59714e4e8cefd1d21ccd7216048bb1c6c3e7104b1b200afdc63d/pybase64-1.4.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b99182c561d86422c5de4265fd1f8f172fb38efaed9d72c71fb31e279a7f94", size = 35433, upload-time = "2025-07-27T13:08:10.551Z" }, - { url = "https://files.pythonhosted.org/packages/01/0f/fab7ed5bf4926523c3b39f7621cea3e0da43f539fbc2270e042f1afccb79/pybase64-1.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bb082c1114f046e59fcbc4f2be13edc93b36d7b54b58605820605be948f8fdf6", size = 36131, upload-time = "2025-07-27T13:08:13.777Z" }, -] - [[package]] name = "pycairo" version = "1.28.0" @@ -7570,7 +7469,7 @@ wheels = [ [[package]] name = "vllm" -version = "0.9.2" +version = "0.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -7594,6 +7493,10 @@ dependencies = [ { name = "numpy" }, { name = "openai" }, { name = "opencv-python-headless" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions-ai" }, { name = "outlines" }, { name = "partial-json-parser" }, { name = "pillow" }, @@ -7602,7 +7505,6 @@ dependencies = [ { name = "protobuf" }, { name = "psutil" }, { name = "py-cpuinfo" }, - { name = "pybase64" }, { name = "pydantic" }, { name = "python-json-logger" }, { name = "pyyaml" }, @@ -7625,11 +7527,11 @@ dependencies = [ { name = "typing-extensions" }, { name = "watchfiles" }, { name = "xformers", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/35/89/2fbf95d398b5751b44c7256bd80e57c589142f1bfcc15f5dc76438b8853a/vllm-0.9.2.tar.gz", hash = "sha256:6b0d855ea8ba18d76364c9b82ea94bfcaa9c9e724055438b5733e4716ed104e1", size = 8997087, upload-time = "2025-07-08T04:49:01.722Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c5/5b/5f42b41d045c01821be62162fc6b1cfb14db1674027c7b623adb3a66dccf/vllm-0.9.1.tar.gz", hash = "sha256:c5ad11603f49a1fad05c88debabb8b839780403ce1b51751ec4da4e8a838082c", size = 8670972, upload-time = "2025-06-10T21:46:12.114Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/72/c14ff1acac64294f45782769b9c8144a1c3e8d4f2228d4648197511b015a/vllm-0.9.2-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:f3c5da29a286f4933b480a5b4749fab226564f35c96928eeef547f88d385cd34", size = 383350132, upload-time = "2025-07-08T04:48:54.133Z" }, + { url = "https://files.pythonhosted.org/packages/b5/56/ffcf6215a571cf9aa58ded06a9640bff21b4918e27344677cd33290ab9da/vllm-0.9.1-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:28b99e8df39c7aaeda04f7e5353b18564a1a9d1c579691945523fc4777a1a8c8", size = 394637693, upload-time = "2025-06-10T21:46:01.784Z" }, ] [[package]] @@ -7939,7 +7841,6 @@ name = "xgrammar" version = "0.1.19" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, { name = "ninja" }, { name = "pydantic" }, { name = "sentencepiece" },