From 607e3040d47698e058150fbdfd74247c58e381af Mon Sep 17 00:00:00 2001
From: LucasStringPay <160168817+LucasStringPay@users.noreply.github.com>
Date: Thu, 2 Oct 2025 15:16:11 -0700
Subject: [PATCH 01/51] Ignore None 'completion_tokens' or similar
Similar as https://github.com/pipecat-ai/pipecat/commit/144ea36c81e0698163493e945a33d5b5b4ebd406 , reported in https://github.com/pipecat-ai/pipecat/issues/2207
---
src/pipecat/services/google/llm_openai.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/pipecat/services/google/llm_openai.py b/src/pipecat/services/google/llm_openai.py
index 81d124cb7..fde62154e 100644
--- a/src/pipecat/services/google/llm_openai.py
+++ b/src/pipecat/services/google/llm_openai.py
@@ -94,9 +94,9 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService):
async for chunk in chunk_stream:
if chunk.usage:
tokens = LLMTokenUsage(
- prompt_tokens=chunk.usage.prompt_tokens,
- completion_tokens=chunk.usage.completion_tokens,
- total_tokens=chunk.usage.total_tokens,
+ prompt_tokens=chunk.usage.prompt_tokens or 0,
+ completion_tokens=chunk.usage.completion_tokens or 0,
+ total_tokens=chunk.usage.total_tokens or 0,
)
await self.start_llm_usage_metrics(tokens)
From 728361a6a7d83f4074e62581ae86ee91b4c00e18 Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Mon, 6 Oct 2025 10:17:50 -0400
Subject: [PATCH 02/51] Add `GeminiVertexMultimodalLiveLLMService`
---
CHANGELOG.md | 3 +
.../26h-gemini-multimodal-live-vertex.py | 134 +++++++++++++
.../services/gemini_multimodal_live/gemini.py | 42 +++-
.../services/gemini_multimodal_live/vertex.py | 187 ++++++++++++++++++
4 files changed, 360 insertions(+), 6 deletions(-)
create mode 100644 examples/foundational/26h-gemini-multimodal-live-vertex.py
create mode 100644 src/pipecat/services/gemini_multimodal_live/vertex.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 10cd8738f..6999bd37c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- Added `GeminiVertexMultimodalLiveLLMService`, for accessing Gemini Live via
+ Google Vertex AI.
+
- Added some new configuration options to `GeminiMultimodalLiveLLMService`:
- `thinking`
diff --git a/examples/foundational/26h-gemini-multimodal-live-vertex.py b/examples/foundational/26h-gemini-multimodal-live-vertex.py
new file mode 100644
index 000000000..581d2524a
--- /dev/null
+++ b/examples/foundational/26h-gemini-multimodal-live-vertex.py
@@ -0,0 +1,134 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import LLMMessagesAppendFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.runner.types import RunnerArguments
+from pipecat.runner.utils import create_transport
+from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_multimodal_live.vertex import GeminiVertexMultimodalLiveLLMService
+from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.transports.daily.transport import DailyParams
+from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
+
+# Load environment variables
+load_dotenv(override=True)
+
+
+# We store functions so objects (e.g. SileroVADAnalyzer) don't get
+# instantiated. The function will be called when the desired transport gets
+# selected.
+transport_params = {
+ "daily": lambda: DailyParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ # set stop_secs to something roughly similar to the internal setting
+ # of the Multimodal Live api, just to align events.
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
+ ),
+ "twilio": lambda: FastAPIWebsocketParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ # set stop_secs to something roughly similar to the internal setting
+ # of the Multimodal Live api, just to align events.
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
+ ),
+ "webrtc": lambda: TransportParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ # set stop_secs to something roughly similar to the internal setting
+ # of the Multimodal Live api, just to align events.
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
+ ),
+}
+
+
+async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
+ logger.info(f"Starting bot")
+
+ # Create the Gemini Vertex Multimodal Live LLM service
+ system_instruction = f"""
+ You are a helpful AI assistant.
+ Your goal is to demonstrate your capabilities in a helpful and engaging way.
+ Your output will be converted to audio so don't include special characters in your answers.
+ Respond to what the user said in a creative and helpful way.
+ """
+
+ llm = GeminiVertexMultimodalLiveLLMService(
+ credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"),
+ project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
+ location=os.getenv("GOOGLE_CLOUD_LOCATION"),
+ system_instruction=system_instruction,
+ voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
+ )
+
+ # Build the pipeline
+ pipeline = Pipeline(
+ [
+ transport.input(),
+ llm,
+ transport.output(),
+ ]
+ )
+
+ # Configure the pipeline task
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ enable_metrics=True,
+ enable_usage_metrics=True,
+ ),
+ idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
+ )
+
+ # Handle client connection event
+ @transport.event_handler("on_client_connected")
+ async def on_client_connected(transport, client):
+ logger.info(f"Client connected")
+ # Kick off the conversation.
+ await task.queue_frames(
+ [
+ LLMMessagesAppendFrame(
+ messages=[
+ {
+ "role": "user",
+ "content": f"Greet the user and introduce yourself.",
+ }
+ ]
+ )
+ ]
+ )
+
+ # Handle client disconnection events
+ @transport.event_handler("on_client_disconnected")
+ async def on_client_disconnected(transport, client):
+ logger.info(f"Client disconnected")
+ await task.cancel()
+
+ # Run the pipeline
+ runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
+ await runner.run(task)
+
+
+async def bot(runner_args: RunnerArguments):
+ """Main bot entry point compatible with Pipecat Cloud."""
+ transport = await create_transport(runner_args, transport_params)
+ await run_bot(transport, runner_args)
+
+
+if __name__ == "__main__":
+ from pipecat.runner.run import main
+
+ main()
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index 65b5eb781..0de76fbf1 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -601,7 +601,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused
self._context = None
- self._create_client(api_key, http_options)
+ self._api_key = api_key
+ self._http_options = http_options
self._session: AsyncSession = None
self._connection_task = None
@@ -649,8 +650,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
"extra": params.extra if isinstance(params.extra, dict) else {},
}
- # Initialize the File API client
- self.file_api = GeminiFileAPI(api_key=api_key, base_url=file_api_base_url)
+ self._file_api_base_url = file_api_base_url
+ self._file_api: Optional[GeminiFileAPI] = None
# Grounding metadata tracking
self._search_result_buffer = ""
@@ -662,8 +663,23 @@ class GeminiMultimodalLiveLLMService(LLMService):
# Bookkeeping for ending gracefully (i.e. after the bot is finished)
self._end_frame_pending_bot_turn_finished: Optional[EndFrame] = None
- def _create_client(self, api_key: str, http_options: Optional[HttpOptions] = None):
- self._client = Client(api_key=api_key, http_options=http_options)
+ # Initialize the API client. Subclasses can override this if needed.
+ self.create_client()
+
+ def create_client(self):
+ """Create the Gemini API client instance. Subclasses can override this."""
+ self._client = Client(api_key=self._api_key, http_options=self._http_options)
+
+ @property
+ def file_api(self) -> GeminiFileAPI:
+ """Get the Gemini File API client instance. Subclasses can override this.
+
+ Returns:
+ The Gemini File API client.
+ """
+ if not self._file_api:
+ self._file_api = GeminiFileAPI(api_key=self._api_key, base_url=self._file_api_base_url)
+ return self._file_api
def can_generate_metrics(self) -> bool:
"""Check if the service can generate usage metrics.
@@ -1282,7 +1298,21 @@ class GeminiMultimodalLiveLLMService(LLMService):
inline_data = part.inline_data
if not inline_data:
return
- if inline_data.mime_type != f"audio/pcm;rate={self._sample_rate}":
+
+ # Check if mime type matches expected format
+ expected_mime_type = f"audio/pcm;rate={self._sample_rate}"
+ if inline_data.mime_type == expected_mime_type:
+ # Perfect match, continue processing
+ pass
+ elif inline_data.mime_type == "audio/pcm":
+ # Sample rate not provided in mime type, assume default
+ if not hasattr(self, "_sample_rate_warning_logged"):
+ logger.warning(
+ f"Sample rate not provided in mime type '{inline_data.mime_type}', assuming rate of {self._sample_rate}"
+ )
+ self._sample_rate_warning_logged = True
+ else:
+ # Unrecognized format
logger.warning(f"Unrecognized server_content format {inline_data.mime_type}")
return
diff --git a/src/pipecat/services/gemini_multimodal_live/vertex.py b/src/pipecat/services/gemini_multimodal_live/vertex.py
new file mode 100644
index 000000000..e07eb474f
--- /dev/null
+++ b/src/pipecat/services/gemini_multimodal_live/vertex.py
@@ -0,0 +1,187 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Google Vertex AI Gemini Multimodal Live service.
+
+This module provides integration with Google's Gemini Multimodal Live model via
+Vertex AI, supporting both text and audio modalities with voice transcription,
+streaming responses, and tool usage.
+"""
+
+import json
+import os
+from typing import List, Optional, Union
+
+from loguru import logger
+
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.services.gemini_multimodal_live.gemini import (
+ GeminiMultimodalLiveLLMService,
+ HttpOptions,
+ InputParams,
+)
+
+try:
+ from google.auth import default
+ from google.auth.exceptions import GoogleAuthError
+ from google.auth.transport.requests import Request
+ from google.genai import Client
+ from google.oauth2 import service_account
+
+except ModuleNotFoundError as e:
+ logger.error(f"Exception: {e}")
+ logger.error(
+ "In order to use Google Vertex AI, you need to `pip install pipecat-ai[google]`. Also, set up your Google credentials properly."
+ )
+ raise Exception(f"Missing module: {e}")
+
+
+class GeminiVertexMultimodalLiveLLMService(GeminiMultimodalLiveLLMService):
+ """Google Vertex AI Gemini Multimodal Live service.
+
+ Provides access to Google's Gemini Multimodal Live model via Vertex AI,
+ supporting both text and audio modalities. It handles voice transcription,
+ streaming audio responses, and tool usage.
+ """
+
+ def __init__(
+ self,
+ *,
+ credentials: Optional[str] = None,
+ credentials_path: Optional[str] = None,
+ location: str = "us-east4",
+ project_id: str,
+ model="google/gemini-2.0-flash-live-preview-04-09",
+ voice_id: str = "Charon",
+ start_audio_paused: bool = False,
+ start_video_paused: bool = False,
+ system_instruction: Optional[str] = None,
+ tools: Optional[Union[List[dict], ToolsSchema]] = None,
+ params: Optional[InputParams] = None,
+ inference_on_context_initialization: bool = True,
+ file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files",
+ http_options: Optional[HttpOptions] = None,
+ **kwargs,
+ ):
+ """Initialize the Google Vertex AI Gemini Multimodal Live service.
+
+ Args:
+ credentials: JSON string of service account credentials.
+ credentials_path: Path to the service account JSON file.
+ location: GCP region for Vertex AI endpoint (e.g., "us-east4").
+ project_id: Google Cloud project ID.
+ model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-preview-04-09".
+ voice_id: TTS voice identifier. Defaults to "Charon".
+ start_audio_paused: Whether to start with audio input paused. Defaults to False.
+ start_video_paused: Whether to start with video input paused. Defaults to False.
+ system_instruction: System prompt for the model. Defaults to None.
+ tools: Tools/functions available to the model. Defaults to None.
+ params: Configuration parameters for the model along with Vertex AI
+ location and project ID.
+ inference_on_context_initialization: Whether to generate a response when context
+ is first set. Defaults to True.
+ file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint.
+ http_options: HTTP options for the client.
+ **kwargs: Additional arguments passed to parent GeminiMultimodalLiveLLMService.
+ """
+ # Check if user incorrectly passed api_key, which is used by parent
+ # class but not here.
+ if "api_key" in kwargs:
+ logger.error(
+ "GeminiVertexMultimodalLiveLLMService does not accept 'api_key' parameter. "
+ "Use 'credentials' or 'credentials_path' instead for Vertex AI authentication."
+ )
+ raise ValueError(
+ "Invalid parameter 'api_key'. Use 'credentials' or 'credentials_path' for Vertex AI authentication."
+ )
+
+ # These need to be set before calling super().__init__() because
+ # super().__init__() invokes create_client(), which needs these.
+ self._credentials = self._get_credentials(credentials, credentials_path)
+ self._project_id = project_id
+ self._location = location
+
+ # Call parent constructor with the obtained API key
+ super().__init__(
+ # api_key is required by parent class, but actually not used with
+ # Vertex
+ api_key="dummy",
+ model=model,
+ voice_id=voice_id,
+ start_audio_paused=start_audio_paused,
+ start_video_paused=start_video_paused,
+ system_instruction=system_instruction,
+ tools=tools,
+ params=params,
+ inference_on_context_initialization=inference_on_context_initialization,
+ file_api_base_url=file_api_base_url,
+ http_options=http_options,
+ **kwargs,
+ )
+
+ def create_client(self):
+ """Create the Gemini client instance."""
+ self._client = Client(
+ vertexai=True,
+ credentials=self._credentials,
+ project=self._project_id,
+ location=self._location,
+ )
+
+ @property
+ def file_api(self):
+ """Gemini File API is not supported with Vertex AI."""
+ raise NotImplementedError(
+ "When using Vertex AI, the recommended approach is to use Google Cloud Storage for file handling. The Gemini File API is not directly supported in this context."
+ )
+
+ @staticmethod
+ def _get_credentials(credentials: Optional[str], credentials_path: Optional[str]) -> str:
+ """Retrieve Credentials using Google service account credentials JSON.
+
+ Supports multiple authentication methods:
+ 1. Direct JSON credentials string
+ 2. Path to service account JSON file
+ 3. Default application credentials (ADC)
+
+ Args:
+ credentials: JSON string of service account credentials.
+ credentials_path: Path to the service account JSON file.
+
+ Returns:
+ OAuth token for API authentication.
+
+ Raises:
+ ValueError: If no valid credentials are provided or found.
+ """
+ creds: Optional[service_account.Credentials] = None
+
+ if credentials:
+ # Parse and load credentials from JSON string
+ creds = service_account.Credentials.from_service_account_info(
+ json.loads(credentials),
+ scopes=["https://www.googleapis.com/auth/cloud-platform"],
+ )
+ elif credentials_path:
+ # Load credentials from JSON file
+ creds = service_account.Credentials.from_service_account_file(
+ credentials_path,
+ scopes=["https://www.googleapis.com/auth/cloud-platform"],
+ )
+ else:
+ try:
+ creds, project_id = default(
+ scopes=["https://www.googleapis.com/auth/cloud-platform"]
+ )
+ except GoogleAuthError:
+ pass
+
+ if not creds:
+ raise ValueError("No valid credentials provided.")
+
+ creds.refresh(Request()) # Ensure token is up-to-date, lifetime is 1 hour.
+
+ return creds
From a14fb20d15e976b49e41fe09e27326db5515e58e Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Mon, 6 Oct 2025 15:10:23 -0400
Subject: [PATCH 03/51] Fix Gemini Live w/Vertex AI not being able to handle an
empty list provided for "function_declarations"
---
src/pipecat/adapters/services/gemini_adapter.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py
index 63a86e6d2..abe33a8ec 100644
--- a/src/pipecat/adapters/services/gemini_adapter.py
+++ b/src/pipecat/adapters/services/gemini_adapter.py
@@ -87,9 +87,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
Includes both converted standard tools and any custom Gemini-specific tools.
"""
functions_schema = tools_schema.standard_tools
- formatted_standard_tools = [
- {"function_declarations": [func.to_default_dict() for func in functions_schema]}
- ]
+ formatted_standard_tools = (
+ [{"function_declarations": [func.to_default_dict() for func in functions_schema]}]
+ if functions_schema
+ else []
+ )
custom_gemini_tools = []
if tools_schema.custom_tools:
custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, [])
From 0b6dd98000e2426c7a7bd8539f2b31d65288d6b0 Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Mon, 6 Oct 2025 15:13:05 -0400
Subject: [PATCH 04/51] Make a note in our examples that there's an issue with
Gemini Live + Vertex around using "google_search" alongside other tools
---
.../26b-gemini-multimodal-live-function-calling.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
index f14713a5c..a5024d4a6 100644
--- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py
+++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
@@ -122,6 +122,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
required=["location"],
)
search_tool = {"google_search": {}}
+ # KNOWN ISSUE: If using GeminiVertexMultimodalLiveLLMService, it appears
+ # you cannot use the "google_search" tool alongside other tools.
+ # See https://github.com/googleapis/python-genai/issues/941.
tools = ToolsSchema(
standard_tools=[weather_function, restaurant_function],
custom_tools={AdapterType.GEMINI: [search_tool]},
From 2699f0c2a61fee9ffb8e60766f90a75caee5ff8d Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Mon, 6 Oct 2025 15:39:08 -0400
Subject: [PATCH 05/51] Fix tool calls when using Gemini Live + Vertex AI
---
src/pipecat/services/gemini_multimodal_live/gemini.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index 0de76fbf1..8ab836173 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -16,6 +16,7 @@ import io
import json
import random
import time
+import uuid
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Optional, Union
@@ -1345,7 +1346,11 @@ class GeminiMultimodalLiveLLMService(LLMService):
function_calls_llm = [
FunctionCallFromLLM(
context=self._context,
- tool_call_id=f.id,
+ tool_call_id=(
+ # NOTE: when using Vertex AI we don't get server-provided
+ # tool call IDs here
+ f.id or str(uuid.uuid4())
+ ),
function_name=f.name,
arguments=f.args,
)
From 99f008e9279b9565c21fc9585d4aaabf0aa7991b Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Mon, 6 Oct 2025 15:54:56 -0400
Subject: [PATCH 06/51] Make a note in our examples that there's an issue with
Gemini Live + Vertex around specifying a modality other than AUDIO
---
examples/foundational/26d-gemini-multimodal-live-text.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py
index 667887b03..dd3e0d65b 100644
--- a/examples/foundational/26d-gemini-multimodal-live-text.py
+++ b/examples/foundational/26d-gemini-multimodal-live-text.py
@@ -80,6 +80,8 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
+ # KNOWN ISSUE: If using GeminiVertexMultimodalLiveLLMService, it appears
+ # you cannot specify a modality other than AUDIO.
llm = GeminiMultimodalLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=SYSTEM_INSTRUCTION,
From f2d90639845b90d25c9f8218697a32936c4e856e Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Thu, 9 Oct 2025 09:40:47 -0400
Subject: [PATCH 07/51] Renames: remove "multimodal" from Gemini Live types
---
.../foundational/26-gemini-multimodal-live.py | 4 +-
...6a-gemini-multimodal-live-transcription.py | 4 +-
...gemini-multimodal-live-function-calling.py | 4 +-
.../26c-gemini-multimodal-live-video.py | 4 +-
.../26d-gemini-multimodal-live-text.py | 10 +-
.../26e-gemini-multimodal-google-search.py | 4 +-
.../26f-gemini-multimodal-live-files-api.py | 6 +-
...emini-multimodal-live-groundingMetadata.py | 4 +-
.../26h-gemini-multimodal-live-vertex.py | 6 +-
...26i-gemini-multimodal-live-graceful-end.py | 4 +-
examples/foundational/46-video-processing.py | 4 +-
src/pipecat/services/gemini_live/__init__.py | 3 +
src/pipecat/services/gemini_live/file_api.py | 189 ++
src/pipecat/services/gemini_live/gemini.py | 1582 ++++++++++++++++
.../vertex.py | 24 +-
.../services/gemini_multimodal_live/events.py | 17 +-
.../gemini_multimodal_live/file_api.py | 194 +-
.../services/gemini_multimodal_live/gemini.py | 1617 +----------------
18 files changed, 1887 insertions(+), 1793 deletions(-)
create mode 100644 src/pipecat/services/gemini_live/__init__.py
create mode 100644 src/pipecat/services/gemini_live/file_api.py
create mode 100644 src/pipecat/services/gemini_live/gemini.py
rename src/pipecat/services/{gemini_multimodal_live => gemini_live}/vertex.py (89%)
diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py
index b446a9b8c..f063c247f 100644
--- a/examples/foundational/26-gemini-multimodal-live.py
+++ b/examples/foundational/26-gemini-multimodal-live.py
@@ -17,7 +17,7 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -65,7 +65,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
Respond to what the user said in a creative and helpful way.
"""
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=system_instruction,
voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py
index ad3cd06ee..3c0eb19ce 100644
--- a/examples/foundational/26a-gemini-multimodal-live-transcription.py
+++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py
@@ -20,7 +20,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.transcript_processor import TranscriptProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -65,7 +65,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
# system_instruction="Talk like a pirate."
diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
index a5024d4a6..a86822a20 100644
--- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py
+++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
@@ -22,7 +22,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -130,7 +130,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
custom_tools={AdapterType.GEMINI: [search_tool]},
)
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=system_instruction,
tools=tools,
diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py
index a28eaaacf..7eac0bc0b 100644
--- a/examples/foundational/26c-gemini-multimodal-live-video.py
+++ b/examples/foundational/26c-gemini-multimodal-live-video.py
@@ -24,7 +24,7 @@ from pipecat.runner.utils import (
maybe_capture_participant_camera,
maybe_capture_participant_screen,
)
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -58,7 +58,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
# system_instruction="Talk like a pirate."
diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py
index dd3e0d65b..625ca17d0 100644
--- a/examples/foundational/26d-gemini-multimodal-live-text.py
+++ b/examples/foundational/26d-gemini-multimodal-live-text.py
@@ -20,9 +20,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.cartesia.tts import CartesiaTTSService
-from pipecat.services.gemini_multimodal_live.gemini import (
- GeminiMultimodalLiveLLMService,
- GeminiMultimodalModalities,
+from pipecat.services.gemini_live.gemini import (
+ GeminiLiveLLMService,
+ GeminiModalities,
InputParams,
)
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -82,11 +82,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
# KNOWN ISSUE: If using GeminiVertexMultimodalLiveLLMService, it appears
# you cannot specify a modality other than AUDIO.
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=SYSTEM_INSTRUCTION,
tools=[{"google_search": {}}, {"code_execution": {}}],
- params=InputParams(modalities=GeminiMultimodalModalities.TEXT),
+ params=InputParams(modalities=GeminiModalities.TEXT),
)
# Optionally, you can set the response modalities via a function
diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-multimodal-google-search.py
index 31fad54fe..1bcf02db9 100644
--- a/examples/foundational/26e-gemini-multimodal-google-search.py
+++ b/examples/foundational/26e-gemini-multimodal-google-search.py
@@ -19,7 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
# Initialize the Gemini Multimodal Live model
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
system_instruction=system_instruction,
diff --git a/examples/foundational/26f-gemini-multimodal-live-files-api.py b/examples/foundational/26f-gemini-multimodal-live-files-api.py
index b01c21803..7ce4a483d 100644
--- a/examples/foundational/26f-gemini-multimodal-live-files-api.py
+++ b/examples/foundational/26f-gemini-multimodal-live-files-api.py
@@ -19,8 +19,8 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import (
- GeminiMultimodalLiveLLMService,
+from pipecat.services.gemini_live.gemini import (
+ GeminiLiveLLMService,
)
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -110,7 +110,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
"""
# Initialize Gemini service with File API support
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=system_instruction,
voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck
diff --git a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py b/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
index 6cfeed51b..9b8b25315 100644
--- a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
+++ b/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
@@ -14,7 +14,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.services.google.frames import LLMSearchResponseFrame
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -105,7 +105,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
custom_tools={AdapterType.GEMINI: [{"google_search": {}}, {"code_execution": {}}]},
)
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=SYSTEM_INSTRUCTION,
voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck
diff --git a/examples/foundational/26h-gemini-multimodal-live-vertex.py b/examples/foundational/26h-gemini-multimodal-live-vertex.py
index 581d2524a..4210dc0e2 100644
--- a/examples/foundational/26h-gemini-multimodal-live-vertex.py
+++ b/examples/foundational/26h-gemini-multimodal-live-vertex.py
@@ -17,8 +17,8 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
-from pipecat.services.gemini_multimodal_live.vertex import GeminiVertexMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.gemini_live.vertex import GeminiLiveVertexLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
Respond to what the user said in a creative and helpful way.
"""
- llm = GeminiVertexMultimodalLiveLLMService(
+ llm = GeminiLiveVertexLLMService(
credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"),
project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
location=os.getenv("GOOGLE_CLOUD_LOCATION"),
diff --git a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py b/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
index e178997b8..9451d0ef0 100644
--- a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
+++ b/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
@@ -24,7 +24,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -144,7 +144,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
custom_tools={AdapterType.GEMINI: [search_tool]},
)
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=system_instruction,
tools=tools,
diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py
index bfb718ff2..7f7363538 100644
--- a/examples/foundational/46-video-processing.py
+++ b/examples/foundational/46-video-processing.py
@@ -20,7 +20,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService
+from pipecat.services.gemini_live import GeminiLiveLLMService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.daily.transport import DailyParams, DailyTransport
@@ -94,7 +94,7 @@ Respond to what the user said in a creative and helpful way. Keep your responses
async def run_bot(pipecat_transport):
- llm = GeminiMultimodalLiveLLMService(
+ llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
transcribe_user_audio=True,
diff --git a/src/pipecat/services/gemini_live/__init__.py b/src/pipecat/services/gemini_live/__init__.py
new file mode 100644
index 000000000..6a2d33bd8
--- /dev/null
+++ b/src/pipecat/services/gemini_live/__init__.py
@@ -0,0 +1,3 @@
+from .file_api import GeminiFileAPI
+from .gemini import GeminiLiveLLMService
+from .vertex import GeminiLiveVertexLLMService
diff --git a/src/pipecat/services/gemini_live/file_api.py b/src/pipecat/services/gemini_live/file_api.py
new file mode 100644
index 000000000..5ae7fdbb7
--- /dev/null
+++ b/src/pipecat/services/gemini_live/file_api.py
@@ -0,0 +1,189 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Gemini File API client for uploading and managing files.
+
+This module provides a client for Google's Gemini File API, enabling file
+uploads, metadata retrieval, listing, and deletion. Files uploaded through
+this API can be referenced in Gemini generative model calls.
+"""
+
+import mimetypes
+from typing import Any, Dict, Optional
+
+import aiohttp
+from loguru import logger
+
+
+class GeminiFileAPI:
+ """Client for the Gemini File API.
+
+ This class provides methods for uploading, fetching, listing, and deleting files
+ through Google's Gemini File API.
+
+ Files uploaded through this API remain available for 48 hours and can be referenced
+ in calls to the Gemini generative models. Maximum file size is 2GB, with total
+ project storage limited to 20GB.
+ """
+
+ def __init__(
+ self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files"
+ ):
+ """Initialize the Gemini File API client.
+
+ Args:
+ api_key: Google AI API key
+ base_url: Base URL for the Gemini File API (default is the v1beta endpoint)
+ """
+ self._api_key = api_key
+ self._base_url = base_url
+ # Upload URL uses the /upload/ path
+ self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files"
+
+ async def upload_file(
+ self, file_path: str, display_name: Optional[str] = None
+ ) -> Dict[str, Any]:
+ """Upload a file to the Gemini File API using the correct resumable upload protocol.
+
+ Args:
+ file_path: Path to the file to upload
+ display_name: Optional display name for the file
+
+ Returns:
+ File metadata including uri, name, and display_name
+ """
+ logger.info(f"Uploading file: {file_path}")
+
+ async with aiohttp.ClientSession() as session:
+ # Determine the file's MIME type
+ mime_type, _ = mimetypes.guess_type(file_path)
+ if not mime_type:
+ mime_type = "application/octet-stream"
+
+ # Read the file
+ with open(file_path, "rb") as f:
+ file_data = f.read()
+
+ # Create the metadata payload
+ metadata = {}
+ if display_name:
+ metadata = {"file": {"display_name": display_name}}
+
+ # Step 1: Initial resumable request to get upload URL
+ headers = {
+ "X-Goog-Upload-Protocol": "resumable",
+ "X-Goog-Upload-Command": "start",
+ "X-Goog-Upload-Header-Content-Length": str(len(file_data)),
+ "X-Goog-Upload-Header-Content-Type": mime_type,
+ "Content-Type": "application/json",
+ }
+
+ logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}")
+ async with session.post(
+ f"{self.upload_base_url}?key={self._api_key}", headers=headers, json=metadata
+ ) as response:
+ if response.status != 200:
+ error_text = await response.text()
+ logger.error(f"Error initiating file upload: {error_text}")
+ raise Exception(f"Failed to initiate upload: {response.status} - {error_text}")
+
+ # Get the upload URL from the response header
+ upload_url = response.headers.get("X-Goog-Upload-URL")
+ if not upload_url:
+ logger.error(f"Response headers: {dict(response.headers)}")
+ raise Exception("No upload URL in response headers")
+
+ logger.debug(f"Got upload URL: {upload_url}")
+
+ # Step 2: Upload the actual file data
+ upload_headers = {
+ "Content-Length": str(len(file_data)),
+ "X-Goog-Upload-Offset": "0",
+ "X-Goog-Upload-Command": "upload, finalize",
+ }
+
+ logger.debug(f"Step 2: Uploading file data to {upload_url}")
+ async with session.post(upload_url, headers=upload_headers, data=file_data) as response:
+ if response.status != 200:
+ error_text = await response.text()
+ logger.error(f"Error uploading file data: {error_text}")
+ raise Exception(f"Failed to upload file: {response.status} - {error_text}")
+
+ file_info = await response.json()
+ logger.info(f"File uploaded successfully: {file_info.get('file', {}).get('name')}")
+ return file_info
+
+ async def get_file(self, name: str) -> Dict[str, Any]:
+ """Get metadata for a file.
+
+ Args:
+ name: File name (or full path)
+
+ Returns:
+ File metadata
+ """
+ # Extract just the name part if a full path is provided
+ if "/" in name:
+ name = name.split("/")[-1]
+
+ async with aiohttp.ClientSession() as session:
+ async with session.get(f"{self._base_url}/{name}?key={self._api_key}") as response:
+ if response.status != 200:
+ error_text = await response.text()
+ logger.error(f"Error getting file metadata: {error_text}")
+ raise Exception(f"Failed to get file metadata: {response.status}")
+
+ file_info = await response.json()
+ return file_info
+
+ async def list_files(
+ self, page_size: int = 10, page_token: Optional[str] = None
+ ) -> Dict[str, Any]:
+ """List uploaded files.
+
+ Args:
+ page_size: Number of files to return per page
+ page_token: Token for pagination
+
+ Returns:
+ List of files and next page token if available
+ """
+ params = {"key": self._api_key, "pageSize": page_size}
+
+ if page_token:
+ params["pageToken"] = page_token
+
+ async with aiohttp.ClientSession() as session:
+ async with session.get(self._base_url, params=params) as response:
+ if response.status != 200:
+ error_text = await response.text()
+ logger.error(f"Error listing files: {error_text}")
+ raise Exception(f"Failed to list files: {response.status}")
+
+ result = await response.json()
+ return result
+
+ async def delete_file(self, name: str) -> bool:
+ """Delete a file.
+
+ Args:
+ name: File name (or full path)
+
+ Returns:
+ True if deleted successfully
+ """
+ # Extract just the name part if a full path is provided
+ if "/" in name:
+ name = name.split("/")[-1]
+
+ async with aiohttp.ClientSession() as session:
+ async with session.delete(f"{self._base_url}/{name}?key={self._api_key}") as response:
+ if response.status != 200:
+ error_text = await response.text()
+ logger.error(f"Error deleting file: {error_text}")
+ raise Exception(f"Failed to delete file: {response.status}")
+
+ return True
diff --git a/src/pipecat/services/gemini_live/gemini.py b/src/pipecat/services/gemini_live/gemini.py
new file mode 100644
index 000000000..4b5f05209
--- /dev/null
+++ b/src/pipecat/services/gemini_live/gemini.py
@@ -0,0 +1,1582 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Google Gemini Live API service implementation.
+
+This module provides real-time conversational AI capabilities using Google's
+Gemini Live API, supporting both text and audio modalities with
+voice transcription, streaming responses, and tool usage.
+"""
+
+import base64
+import io
+import json
+import random
+import time
+import uuid
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from loguru import logger
+from PIL import Image
+from pydantic import BaseModel, Field
+
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
+from pipecat.frames.frames import (
+ BotStartedSpeakingFrame,
+ BotStoppedSpeakingFrame,
+ CancelFrame,
+ EndFrame,
+ ErrorFrame,
+ Frame,
+ InputAudioRawFrame,
+ InputImageRawFrame,
+ InputTextRawFrame,
+ InterruptionFrame,
+ LLMContextFrame,
+ LLMFullResponseEndFrame,
+ LLMFullResponseStartFrame,
+ LLMMessagesAppendFrame,
+ LLMSetToolsFrame,
+ LLMTextFrame,
+ LLMUpdateSettingsFrame,
+ StartFrame,
+ TranscriptionFrame,
+ TTSAudioRawFrame,
+ TTSStartedFrame,
+ TTSStoppedFrame,
+ TTSTextFrame,
+ UserImageRawFrame,
+ UserStartedSpeakingFrame,
+ UserStoppedSpeakingFrame,
+)
+from pipecat.metrics.metrics import LLMTokenUsage
+from pipecat.processors.aggregators.llm_response import (
+ LLMAssistantAggregatorParams,
+ LLMUserAggregatorParams,
+)
+from pipecat.processors.aggregators.openai_llm_context import (
+ OpenAILLMContext,
+ OpenAILLMContextFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult
+from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
+from pipecat.services.openai.llm import (
+ OpenAIAssistantContextAggregator,
+ OpenAIUserContextAggregator,
+)
+from pipecat.transcriptions.language import Language
+from pipecat.utils.string import match_endofsentence
+from pipecat.utils.time import time_now_iso8601
+from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt
+
+from .file_api import GeminiFileAPI
+
+try:
+ from google.genai import Client
+ from google.genai.live import AsyncSession
+ from google.genai.types import (
+ AudioTranscriptionConfig,
+ AutomaticActivityDetection,
+ Blob,
+ Content,
+ ContextWindowCompressionConfig,
+ EndSensitivity,
+ FileData,
+ FunctionResponse,
+ GenerationConfig,
+ GroundingMetadata,
+ HttpOptions,
+ LiveConnectConfig,
+ LiveServerMessage,
+ MediaResolution,
+ Modality,
+ Part,
+ ProactivityConfig,
+ RealtimeInputConfig,
+ SessionResumptionConfig,
+ SlidingWindow,
+ SpeechConfig,
+ StartSensitivity,
+ ThinkingConfig,
+ VoiceConfig,
+ )
+except ModuleNotFoundError as e:
+ logger.error(f"Exception: {e}")
+ logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
+ raise Exception(f"Missing module: {e}")
+
+
+# Connection management constants
+MAX_CONSECUTIVE_FAILURES = 3
+CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds
+
+
+def language_to_gemini_language(language: Language) -> Optional[str]:
+ """Maps a Language enum value to a Gemini Live supported language code.
+
+ Source:
+ https://ai.google.dev/api/generate-content#MediaResolution
+
+ Args:
+ language: The language enum value to convert.
+
+ Returns:
+ The Gemini language code string, or None if the language is not supported.
+ """
+ language_map = {
+ # Arabic
+ Language.AR: "ar-XA",
+ # Bengali
+ Language.BN_IN: "bn-IN",
+ # Chinese (Mandarin)
+ Language.CMN: "cmn-CN",
+ Language.CMN_CN: "cmn-CN",
+ Language.ZH: "cmn-CN", # Map general Chinese to Mandarin for Gemini
+ Language.ZH_CN: "cmn-CN", # Map Simplified Chinese to Mandarin for Gemini
+ # German
+ Language.DE: "de-DE",
+ Language.DE_DE: "de-DE",
+ # English
+ Language.EN: "en-US", # Default to US English (though not explicitly listed in supported codes)
+ Language.EN_US: "en-US",
+ Language.EN_AU: "en-AU",
+ Language.EN_GB: "en-GB",
+ Language.EN_IN: "en-IN",
+ # Spanish
+ Language.ES: "es-ES", # Default to Spain Spanish
+ Language.ES_ES: "es-ES",
+ Language.ES_US: "es-US",
+ # French
+ Language.FR: "fr-FR", # Default to France French
+ Language.FR_FR: "fr-FR",
+ Language.FR_CA: "fr-CA",
+ # Gujarati
+ Language.GU: "gu-IN",
+ Language.GU_IN: "gu-IN",
+ # Hindi
+ Language.HI: "hi-IN",
+ Language.HI_IN: "hi-IN",
+ # Indonesian
+ Language.ID: "id-ID",
+ Language.ID_ID: "id-ID",
+ # Italian
+ Language.IT: "it-IT",
+ Language.IT_IT: "it-IT",
+ # Japanese
+ Language.JA: "ja-JP",
+ Language.JA_JP: "ja-JP",
+ # Kannada
+ Language.KN: "kn-IN",
+ Language.KN_IN: "kn-IN",
+ # Korean
+ Language.KO: "ko-KR",
+ Language.KO_KR: "ko-KR",
+ # Malayalam
+ Language.ML: "ml-IN",
+ Language.ML_IN: "ml-IN",
+ # Marathi
+ Language.MR: "mr-IN",
+ Language.MR_IN: "mr-IN",
+ # Dutch
+ Language.NL: "nl-NL",
+ Language.NL_NL: "nl-NL",
+ # Polish
+ Language.PL: "pl-PL",
+ Language.PL_PL: "pl-PL",
+ # Portuguese (Brazil)
+ Language.PT_BR: "pt-BR",
+ # Russian
+ Language.RU: "ru-RU",
+ Language.RU_RU: "ru-RU",
+ # Tamil
+ Language.TA: "ta-IN",
+ Language.TA_IN: "ta-IN",
+ # Telugu
+ Language.TE: "te-IN",
+ Language.TE_IN: "te-IN",
+ # Thai
+ Language.TH: "th-TH",
+ Language.TH_TH: "th-TH",
+ # Turkish
+ Language.TR: "tr-TR",
+ Language.TR_TR: "tr-TR",
+ # Vietnamese
+ Language.VI: "vi-VN",
+ Language.VI_VN: "vi-VN",
+ }
+ return language_map.get(language)
+
+
+class GeminiLiveContext(OpenAILLMContext):
+ """Extended OpenAI context for Gemini Live API.
+
+ Provides Gemini-specific context management including system instruction
+ extraction and message format conversion for the Live API.
+ """
+
+ @staticmethod
+ def upgrade(obj: OpenAILLMContext) -> "GeminiLiveContext":
+ """Upgrade an OpenAI context to Gemini context.
+
+ Args:
+ obj: The OpenAI context to upgrade.
+
+ Returns:
+ The upgraded Gemini context instance.
+ """
+ if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiLiveContext):
+ logger.debug(f"Upgrading to Gemini Live Context: {obj}")
+ obj.__class__ = GeminiLiveContext
+ obj._restructure_from_openai_messages()
+ return obj
+
+ def _restructure_from_openai_messages(self):
+ pass
+
+ def extract_system_instructions(self):
+ """Extract system instructions from context messages.
+
+ Returns:
+ Combined system instruction text from all system messages.
+ """
+ system_instruction = ""
+ for item in self.messages:
+ if item.get("role") == "system":
+ content = item.get("content", "")
+ if content:
+ if system_instruction and not system_instruction.endswith("\n"):
+ system_instruction += "\n"
+ system_instruction += str(content)
+ return system_instruction
+
+ def add_file_reference(self, file_uri: str, mime_type: str, text: Optional[str] = None):
+ """Add a file reference to the context.
+
+ This adds a user message with a file reference that will be sent during context initialization.
+
+ Args:
+ file_uri: URI of the uploaded file
+ mime_type: MIME type of the file
+ text: Optional text prompt to accompany the file
+ """
+ # Create parts list with file reference
+ parts = []
+ if text:
+ parts.append({"type": "text", "text": text})
+
+ # Add file reference part
+ parts.append(
+ {"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}}
+ )
+
+ # Add to messages
+ message = {"role": "user", "content": parts}
+ self.messages.append(message)
+ logger.info(f"Added file reference to context: {file_uri}")
+
+ def get_messages_for_initializing_history(self) -> List[Content]:
+ """Get messages formatted for Gemini history initialization.
+
+ Returns:
+ List of messages in Gemini format for conversation history.
+ """
+ messages: List[Content] = []
+ for item in self.messages:
+ role = item.get("role")
+
+ if role == "system":
+ continue
+
+ elif role == "assistant":
+ role = "model"
+
+ content = item.get("content")
+ parts: List[Part] = []
+ if isinstance(content, str):
+ parts = [Part(text=content)]
+ elif isinstance(content, list):
+ for part in content:
+ if part.get("type") == "text":
+ parts.append(Part(text=part.get("text")))
+ elif part.get("type") == "file_data":
+ file_data = part.get("file_data", {})
+ parts.append(
+ Part(
+ file_data=FileData(
+ mime_type=file_data.get("mime_type"),
+ file_uri=file_data.get("file_uri"),
+ )
+ )
+ )
+ else:
+ logger.warning(f"Unsupported content type: {str(part)[:80]}")
+ else:
+ logger.warning(f"Unsupported content type: {str(content)[:80]}")
+ messages.append(Content(role=role, parts=parts))
+ return messages
+
+
+class GeminiLiveUserContextAggregator(OpenAIUserContextAggregator):
+ """User context aggregator for Gemini Live.
+
+ Extends OpenAI user aggregator to handle Gemini-specific message passing
+ while maintaining compatibility with the standard aggregation pipeline.
+ """
+
+ async def process_frame(self, frame, direction):
+ """Process incoming frames for user context aggregation.
+
+ Args:
+ frame: The frame to process.
+ direction: The frame processing direction.
+ """
+ await super().process_frame(frame, direction)
+ # kind of a hack just to pass the LLMMessagesAppendFrame through, but it's fine for now
+ if isinstance(frame, LLMMessagesAppendFrame):
+ await self.push_frame(frame, direction)
+
+
+class GeminiLiveAssistantContextAggregator(OpenAIAssistantContextAggregator):
+ """Assistant context aggregator for Gemini Live.
+
+ Handles assistant response aggregation while filtering out LLMTextFrames
+ to prevent duplicate context entries, as Gemini Live pushes both
+ LLMTextFrames and TTSTextFrames.
+ """
+
+ async def process_frame(self, frame: Frame, direction: FrameDirection):
+ """Process incoming frames for assistant context aggregation.
+
+ Args:
+ frame: The frame to process.
+ direction: The frame processing direction.
+ """
+ # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output,
+ # but the GeminiLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We
+ # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames
+ # are process. This ensures that the context gets only one set of messages.
+ if not isinstance(frame, LLMTextFrame):
+ await super().process_frame(frame, direction)
+
+ async def handle_user_image_frame(self, frame: UserImageRawFrame):
+ """Handle user image frames.
+
+ Args:
+ frame: The user image frame to handle.
+ """
+ # We don't want to store any images in the context. Revisit this later
+ # when the API evolves.
+ pass
+
+
+@dataclass
+class GeminiLiveContextAggregatorPair:
+ """Pair of user and assistant context aggregators for Gemini Live.
+
+ Parameters:
+ _user: The user context aggregator instance.
+ _assistant: The assistant context aggregator instance.
+ """
+
+ _user: GeminiLiveUserContextAggregator
+ _assistant: GeminiLiveAssistantContextAggregator
+
+ def user(self) -> GeminiLiveUserContextAggregator:
+ """Get the user context aggregator.
+
+ Returns:
+ The user context aggregator instance.
+ """
+ return self._user
+
+ def assistant(self) -> GeminiLiveAssistantContextAggregator:
+ """Get the assistant context aggregator.
+
+ Returns:
+ The assistant context aggregator instance.
+ """
+ return self._assistant
+
+
+class GeminiModalities(Enum):
+ """Supported modalities for Gemini Live.
+
+ Parameters:
+ TEXT: Text responses.
+ AUDIO: Audio responses.
+ """
+
+ TEXT = "TEXT"
+ AUDIO = "AUDIO"
+
+
+class GeminiMediaResolution(str, Enum):
+ """Media resolution options for Gemini Live.
+
+ Parameters:
+ UNSPECIFIED: Use default resolution setting.
+ LOW: Low resolution with 64 tokens.
+ MEDIUM: Medium resolution with 256 tokens.
+ HIGH: High resolution with zoomed reframing and 256 tokens.
+ """
+
+ UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default
+ LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens
+ MEDIUM = "MEDIA_RESOLUTION_MEDIUM" # 256 tokens
+ HIGH = "MEDIA_RESOLUTION_HIGH" # Zoomed reframing with 256 tokens
+
+
+class GeminiVADParams(BaseModel):
+ """Voice Activity Detection parameters for Gemini Live.
+
+ Parameters:
+ disabled: Whether to disable VAD. Defaults to None.
+ start_sensitivity: Sensitivity for speech start detection. Defaults to None.
+ end_sensitivity: Sensitivity for speech end detection. Defaults to None.
+ prefix_padding_ms: Prefix padding in milliseconds. Defaults to None.
+ silence_duration_ms: Silence duration threshold in milliseconds. Defaults to None.
+ """
+
+ disabled: Optional[bool] = Field(default=None)
+ start_sensitivity: Optional[StartSensitivity] = Field(default=None)
+ end_sensitivity: Optional[EndSensitivity] = Field(default=None)
+ prefix_padding_ms: Optional[int] = Field(default=None)
+ silence_duration_ms: Optional[int] = Field(default=None)
+
+
+class ContextWindowCompressionParams(BaseModel):
+ """Parameters for context window compression in Gemini Live.
+
+ Parameters:
+ enabled: Whether compression is enabled. Defaults to False.
+ trigger_tokens: Token count to trigger compression. None uses 80% of context window.
+ """
+
+ enabled: bool = Field(default=False)
+ trigger_tokens: Optional[int] = Field(
+ default=None
+ ) # None = use default (80% of context window)
+
+
+class InputParams(BaseModel):
+ """Input parameters for Gemini Live generation.
+
+ Parameters:
+ frequency_penalty: Frequency penalty for generation (0.0-2.0). Defaults to None.
+ max_tokens: Maximum tokens to generate. Must be >= 1. Defaults to 4096.
+ presence_penalty: Presence penalty for generation (0.0-2.0). Defaults to None.
+ temperature: Sampling temperature (0.0-2.0). Defaults to None.
+ top_k: Top-k sampling parameter. Must be >= 0. Defaults to None.
+ top_p: Top-p sampling parameter (0.0-1.0). Defaults to None.
+ modalities: Response modalities. Defaults to AUDIO.
+ language: Language for generation. Defaults to EN_US.
+ media_resolution: Media resolution setting. Defaults to UNSPECIFIED.
+ vad: Voice activity detection parameters. Defaults to None.
+ context_window_compression: Context compression settings. Defaults to None.
+ thinking: Thinking settings. Defaults to None.
+ Note that these settings may require specifying a model that
+ supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025".
+ enable_affective_dialog: Enable affective dialog, which allows Gemini
+ to adapt to expression and tone. Defaults to None.
+ Note that these settings may require specifying a model that
+ supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025".
+ Also note that this setting may require specifying an API version that
+ supports it, e.g. HttpOptions(api_version="v1alpha").
+ proactivity: Proactivity settings, which allows Gemini to proactively
+ decide how to behave, such as whether to avoid responding to
+ content that is not relevant. Defaults to None.
+ Note that these settings may require specifying a model that
+ supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025".
+ Also note that this setting may require specifying an API version that
+ supports it, e.g. HttpOptions(api_version="v1alpha").
+ extra: Additional parameters. Defaults to empty dict.
+ """
+
+ frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0)
+ max_tokens: Optional[int] = Field(default=4096, ge=1)
+ presence_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0)
+ temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
+ top_k: Optional[int] = Field(default=None, ge=0)
+ top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+ modalities: Optional[GeminiModalities] = Field(default=GeminiModalities.AUDIO)
+ language: Optional[Language] = Field(default=Language.EN_US)
+ media_resolution: Optional[GeminiMediaResolution] = Field(
+ default=GeminiMediaResolution.UNSPECIFIED
+ )
+ vad: Optional[GeminiVADParams] = Field(default=None)
+ context_window_compression: Optional[ContextWindowCompressionParams] = Field(default=None)
+ thinking: Optional[ThinkingConfig] = Field(default=None)
+ enable_affective_dialog: Optional[bool] = Field(default=None)
+ proactivity: Optional[ProactivityConfig] = Field(default=None)
+ extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
+
+
+class GeminiLiveLLMService(LLMService):
+ """Provides access to Google's Gemini Live API.
+
+ This service enables real-time conversations with Gemini, supporting both
+ text and audio modalities. It handles voice transcription, streaming audio
+ responses, and tool usage.
+ """
+
+ # Overriding the default adapter to use the Gemini one.
+ adapter_class = GeminiLLMAdapter
+
+ def __init__(
+ self,
+ *,
+ api_key: str,
+ base_url: Optional[str] = None,
+ model="models/gemini-2.0-flash-live-001",
+ voice_id: str = "Charon",
+ start_audio_paused: bool = False,
+ start_video_paused: bool = False,
+ system_instruction: Optional[str] = None,
+ tools: Optional[Union[List[dict], ToolsSchema]] = None,
+ params: Optional[InputParams] = None,
+ inference_on_context_initialization: bool = True,
+ file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files",
+ http_options: Optional[HttpOptions] = None,
+ **kwargs,
+ ):
+ """Initialize the Gemini Live LLM service.
+
+ Args:
+ api_key: Google AI API key for authentication.
+ base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint.
+
+ .. deprecated:: 0.0.90
+ This parameter is deprecated and no longer has any effect.
+ Please use `http_options` to customize requests made by the
+ API client.
+
+ model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001".
+ voice_id: TTS voice identifier. Defaults to "Charon".
+ start_audio_paused: Whether to start with audio input paused. Defaults to False.
+ start_video_paused: Whether to start with video input paused. Defaults to False.
+ system_instruction: System prompt for the model. Defaults to None.
+ tools: Tools/functions available to the model. Defaults to None.
+ params: Configuration parameters for the model. Defaults to InputParams().
+ inference_on_context_initialization: Whether to generate a response when context
+ is first set. Defaults to True.
+ file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint.
+ http_options: HTTP options for the client.
+ **kwargs: Additional arguments passed to parent LLMService.
+ """
+ # Check for deprecated parameter usage
+ if base_url is not None:
+ import warnings
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Parameter 'base_url' is deprecated and no longer has any effect. Please use 'http_options' to customize requests made by the API client.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ super().__init__(base_url=base_url, **kwargs)
+
+ params = params or InputParams()
+
+ self._last_sent_time = 0
+ self._base_url = base_url
+ self.set_model_name(model)
+ self._voice_id = voice_id
+ self._language_code = params.language
+
+ self._system_instruction = system_instruction
+ self._tools = tools
+ self._inference_on_context_initialization = inference_on_context_initialization
+ self._needs_turn_complete_message = False
+
+ self._audio_input_paused = start_audio_paused
+ self._video_input_paused = start_video_paused
+ self._context = None
+ self._api_key = api_key
+ self._http_options = http_options
+ self._session: AsyncSession = None
+ self._connection_task = None
+
+ self._disconnecting = False
+ self._run_llm_when_session_ready = False
+
+ self._user_is_speaking = False
+ self._bot_is_speaking = False
+ self._user_audio_buffer = bytearray()
+ self._user_transcription_buffer = ""
+ self._last_transcription_sent = ""
+ self._bot_audio_buffer = bytearray()
+ self._bot_text_buffer = ""
+ self._llm_output_buffer = ""
+
+ self._sample_rate = 24000
+
+ self._language = params.language
+ self._language_code = (
+ language_to_gemini_language(params.language) if params.language else "en-US"
+ )
+ self._vad_params = params.vad
+
+ # Reconnection tracking
+ self._consecutive_failures = 0
+ self._connection_start_time = None
+
+ self._settings = {
+ "frequency_penalty": params.frequency_penalty,
+ "max_tokens": params.max_tokens,
+ "presence_penalty": params.presence_penalty,
+ "temperature": params.temperature,
+ "top_k": params.top_k,
+ "top_p": params.top_p,
+ "modalities": params.modalities,
+ "language": self._language_code,
+ "media_resolution": params.media_resolution,
+ "vad": params.vad,
+ "context_window_compression": params.context_window_compression.model_dump()
+ if params.context_window_compression
+ else {},
+ "thinking": params.thinking or {},
+ "enable_affective_dialog": params.enable_affective_dialog or False,
+ "proactivity": params.proactivity or {},
+ "extra": params.extra if isinstance(params.extra, dict) else {},
+ }
+
+ self._file_api_base_url = file_api_base_url
+ self._file_api: Optional[GeminiFileAPI] = None
+
+ # Grounding metadata tracking
+ self._search_result_buffer = ""
+ self._accumulated_grounding_metadata = None
+
+ # Session resumption
+ self._session_resumption_handle: Optional[str] = None
+
+ # Bookkeeping for ending gracefully (i.e. after the bot is finished)
+ self._end_frame_pending_bot_turn_finished: Optional[EndFrame] = None
+
+ # Initialize the API client. Subclasses can override this if needed.
+ self.create_client()
+
+ def create_client(self):
+ """Create the Gemini API client instance. Subclasses can override this."""
+ self._client = Client(api_key=self._api_key, http_options=self._http_options)
+
+ @property
+ def file_api(self) -> GeminiFileAPI:
+ """Get the Gemini File API client instance. Subclasses can override this.
+
+ Returns:
+ The Gemini File API client.
+ """
+ if not self._file_api:
+ self._file_api = GeminiFileAPI(api_key=self._api_key, base_url=self._file_api_base_url)
+ return self._file_api
+
+ def can_generate_metrics(self) -> bool:
+ """Check if the service can generate usage metrics.
+
+ Returns:
+ True as Gemini Live supports token usage metrics.
+ """
+ return True
+
+ def needs_mcp_alternate_schema(self) -> bool:
+ """Check if this LLM service requires alternate MCP schema.
+
+ Google/Gemini has stricter JSON schema validation and requires
+ certain properties to be removed or modified for compatibility.
+
+ Returns:
+ True for Google/Gemini services.
+ """
+ return True
+
+ def set_audio_input_paused(self, paused: bool):
+ """Set the audio input pause state.
+
+ Args:
+ paused: Whether to pause audio input.
+ """
+ self._audio_input_paused = paused
+
+ def set_video_input_paused(self, paused: bool):
+ """Set the video input pause state.
+
+ Args:
+ paused: Whether to pause video input.
+ """
+ self._video_input_paused = paused
+
+ def set_model_modalities(self, modalities: GeminiModalities):
+ """Set the model response modalities.
+
+ Args:
+ modalities: The modalities to use for responses.
+ """
+ self._settings["modalities"] = modalities
+
+ def set_language(self, language: Language):
+ """Set the language for generation.
+
+ Args:
+ language: The language to use for generation.
+ """
+ self._language = language
+ self._language_code = language_to_gemini_language(language) or "en-US"
+ self._settings["language"] = self._language_code
+ logger.info(f"Set Gemini language to: {self._language_code}")
+
+ async def set_context(self, context: OpenAILLMContext):
+ """Set the context explicitly from outside the pipeline.
+
+ This is useful when initializing a conversation because in server-side VAD mode we might not have a
+ way to trigger the pipeline. This sends the history to the server. The `inference_on_context_initialization`
+ flag controls whether to set the turnComplete flag when we do this. Without that flag, the model will
+ not respond. This is often what we want when setting the context at the beginning of a conversation.
+
+ Args:
+ context: The OpenAI LLM context to set.
+ """
+ if self._context:
+ logger.error("Context already set. Can only set up Gemini Live context once.")
+ return
+ self._context = GeminiLiveContext.upgrade(context)
+ await self._create_initial_response()
+
+ #
+ # standard AIService frame handling
+ #
+
+ async def start(self, frame: StartFrame):
+ """Start the service and establish connection.
+
+ Args:
+ frame: The start frame.
+ """
+ await super().start(frame)
+ await self._connect()
+
+ async def stop(self, frame: EndFrame):
+ """Stop the service and close connections.
+
+ Args:
+ frame: The end frame.
+ """
+ await super().stop(frame)
+ await self._disconnect()
+
+ async def cancel(self, frame: CancelFrame):
+ """Cancel the service and close connections.
+
+ Args:
+ frame: The cancel frame.
+ """
+ await super().cancel(frame)
+ await self._disconnect()
+
+ #
+ # speech and interruption handling
+ #
+
+ async def _handle_interruption(self):
+ await self._set_bot_is_speaking(False)
+ await self.push_frame(TTSStoppedFrame())
+ await self.push_frame(LLMFullResponseEndFrame())
+
+ async def _handle_user_started_speaking(self, frame):
+ self._user_is_speaking = True
+ pass
+
+ async def _handle_user_stopped_speaking(self, frame):
+ self._user_is_speaking = False
+ self._user_audio_buffer = bytearray()
+ await self.start_ttfb_metrics()
+ if self._needs_turn_complete_message:
+ self._needs_turn_complete_message = False
+ # NOTE: without this, the model ignores the context it's been
+ # seeded with before the user started speaking
+ await self._session.send_client_content(turn_complete=True)
+
+ #
+ # frame processing
+ #
+ # StartFrame, StopFrame, CancelFrame implemented in base class
+ #
+
+ async def process_frame(self, frame: Frame, direction: FrameDirection):
+ """Process incoming frames for the Gemini Live service.
+
+ Args:
+ frame: The frame to process.
+ direction: The frame processing direction.
+ """
+ # Defer EndFrame handling until after the bot turn is finished
+ if isinstance(frame, EndFrame):
+ if self._bot_is_speaking:
+ logger.debug("Deferring handling EndFrame until bot turn is finished")
+ self._end_frame_pending_bot_turn_finished = frame
+ return
+
+ await super().process_frame(frame, direction)
+
+ if isinstance(frame, TranscriptionFrame):
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, OpenAILLMContextFrame):
+ context: GeminiLiveContext = GeminiLiveContext.upgrade(frame.context)
+ # For now, we'll only trigger inference here when either:
+ # 1. We have not seen a context frame before
+ # 2. The last message is a tool call result
+ if not self._context:
+ self._context = context
+ if frame.context.tools:
+ self._tools = frame.context.tools
+ await self._create_initial_response()
+ elif context.messages and context.messages[-1].get("role") == "tool":
+ # Support just one tool call per context frame for now
+ tool_result_message = context.messages[-1]
+ await self._tool_result(tool_result_message)
+ elif isinstance(frame, LLMContextFrame):
+ raise NotImplementedError("Universal LLMContext is not yet supported for Gemini Live.")
+ elif isinstance(frame, InputTextRawFrame):
+ await self._send_user_text(frame.text)
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, InputAudioRawFrame):
+ await self._send_user_audio(frame)
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, InputImageRawFrame):
+ await self._send_user_video(frame)
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, InterruptionFrame):
+ await self._handle_interruption()
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, UserStartedSpeakingFrame):
+ await self._handle_user_started_speaking(frame)
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, UserStoppedSpeakingFrame):
+ await self._handle_user_stopped_speaking(frame)
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, BotStartedSpeakingFrame):
+ # Ignore this frame. Use the serverContent API message instead
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, BotStoppedSpeakingFrame):
+ # ignore this frame. Use the serverContent.turnComplete API message
+ await self.push_frame(frame, direction)
+ elif isinstance(frame, LLMMessagesAppendFrame):
+ # NOTE: handling LLMMessagesAppendFrame here in the LLMService is
+ # unusual - typically this would be handled in the user context
+ # aggregator. Leaving this handling here so that user code that
+ # uses this frame *without* a user context aggregator still works
+ # (we have an example that does just that, actually).
+ await self._create_single_response(frame.messages)
+ elif isinstance(frame, LLMUpdateSettingsFrame):
+ await self._update_settings(frame.settings)
+ elif isinstance(frame, LLMSetToolsFrame):
+ await self._update_settings()
+ else:
+ await self.push_frame(frame, direction)
+
+ async def _set_bot_is_speaking(self, speaking: bool):
+ if self._bot_is_speaking == speaking:
+ return
+
+ self._bot_is_speaking = speaking
+
+ if not self._bot_is_speaking and self._end_frame_pending_bot_turn_finished:
+ await self.queue_frame(self._end_frame_pending_bot_turn_finished)
+ self._end_frame_pending_bot_turn_finished = None
+
+ async def _connect(self, session_resumption_handle: Optional[str] = None):
+ """Establish client connection to Gemini Live API."""
+ if self._session:
+ # Here we assume that if we have a client, we are connected. We
+ # handle disconnections in the send/recv code paths.
+ return
+
+ if session_resumption_handle:
+ logger.info(
+ f"Connecting to Gemini service with session_resumption_handle: {session_resumption_handle}"
+ )
+ else:
+ logger.info("Connecting to Gemini service")
+ try:
+ # Assemble basic configuration
+ config = LiveConnectConfig(
+ generation_config=GenerationConfig(
+ frequency_penalty=self._settings["frequency_penalty"],
+ max_output_tokens=self._settings["max_tokens"],
+ presence_penalty=self._settings["presence_penalty"],
+ temperature=self._settings["temperature"],
+ top_k=self._settings["top_k"],
+ top_p=self._settings["top_p"],
+ response_modalities=[Modality(self._settings["modalities"].value)],
+ speech_config=SpeechConfig(
+ voice_config=VoiceConfig(
+ prebuilt_voice_config={"voice_name": self._voice_id}
+ ),
+ language_code=self._settings["language"],
+ ),
+ media_resolution=MediaResolution(self._settings["media_resolution"].value),
+ ),
+ input_audio_transcription=AudioTranscriptionConfig(),
+ output_audio_transcription=AudioTranscriptionConfig(),
+ session_resumption=SessionResumptionConfig(handle=session_resumption_handle),
+ )
+
+ # Add context window compression to configuration, if enabled
+ if self._settings.get("context_window_compression", {}).get("enabled", False):
+ compression_config = ContextWindowCompressionConfig()
+
+ # Add sliding window (always true if compression is enabled)
+ compression_config.sliding_window = SlidingWindow()
+
+ # Add trigger_tokens if specified
+ trigger_tokens = self._settings.get("context_window_compression", {}).get(
+ "trigger_tokens"
+ )
+ if trigger_tokens is not None:
+ compression_config.trigger_tokens = trigger_tokens
+
+ config.context_window_compression = compression_config
+
+ # Add thinking configuration to configuration, if provided
+ if self._settings.get("thinking"):
+ config.thinking_config = self._settings["thinking"]
+
+ # Add affective dialog setting, if provided
+ if self._settings.get("enable_affective_dialog", False):
+ config.enable_affective_dialog = self._settings["enable_affective_dialog"]
+
+ # Add proactivity configuration to configuration, if provided
+ if self._settings.get("proactivity"):
+ config.proactivity = self._settings["proactivity"]
+
+ # Add VAD configuration to configuration, if provided
+ if self._settings.get("vad"):
+ vad_config = AutomaticActivityDetection()
+ vad_params = self._settings["vad"]
+ has_vad_settings = False
+
+ # Only add parameters that are explicitly set
+ if vad_params.disabled is not None:
+ vad_config.disabled = vad_params.disabled
+ has_vad_settings = True
+
+ if vad_params.start_sensitivity:
+ vad_config.start_of_speech_sensitivity = vad_params.start_sensitivity
+ has_vad_settings = True
+
+ if vad_params.end_sensitivity:
+ vad_config.end_of_speech_sensitivity = vad_params.end_sensitivity
+ has_vad_settings = True
+
+ if vad_params.prefix_padding_ms is not None:
+ vad_config.prefix_padding_ms = vad_params.prefix_padding_ms
+ has_vad_settings = True
+
+ if vad_params.silence_duration_ms is not None:
+ vad_config.silence_duration_ms = vad_params.silence_duration_ms
+ has_vad_settings = True
+
+ # Only add automatic_activity_detection if we have VAD settings
+ if has_vad_settings:
+ config.realtime_input_config = RealtimeInputConfig(
+ automatic_activity_detection=vad_config
+ )
+
+ # Add system instruction to configuration, if provided
+ system_instruction = self._system_instruction or ""
+ if self._context and hasattr(self._context, "extract_system_instructions"):
+ system_instruction += "\n" + self._context.extract_system_instructions()
+ if system_instruction:
+ logger.debug(f"Setting system instruction: {system_instruction}")
+ config.system_instruction = system_instruction
+
+ # Add tools to configuration, if provided
+ if self._tools:
+ logger.debug(f"Setting tools: {self._tools}")
+ config.tools = self.get_llm_adapter().from_standard_tools(self._tools)
+
+ # Start the connection
+ self._connection_task = self.create_task(self._connection_task_handler(config=config))
+
+ except Exception as e:
+ await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}", fatal=True))
+
+ async def _connection_task_handler(self, config: LiveConnectConfig):
+ async with self._client.aio.live.connect(model=self._model_name, config=config) as session:
+ logger.info("Connected to Gemini service")
+
+ # Mark connection start time
+ self._connection_start_time = time.time()
+
+ await self._handle_session_ready(session)
+
+ while True:
+ try:
+ turn = self._session.receive()
+ async for message in turn:
+ # Reset failure counter if connection has been stable
+ self._check_and_reset_failure_counter()
+
+ if message.server_content and message.server_content.model_turn:
+ await self._handle_msg_model_turn(message)
+ elif (
+ message.server_content
+ and message.server_content.turn_complete
+ and message.usage_metadata
+ ):
+ await self._handle_msg_turn_complete(message)
+ await self._handle_msg_usage_metadata(message)
+ elif message.server_content and message.server_content.input_transcription:
+ await self._handle_msg_input_transcription(message)
+ elif message.server_content and message.server_content.output_transcription:
+ await self._handle_msg_output_transcription(message)
+ elif message.server_content and message.server_content.grounding_metadata:
+ await self._handle_msg_grounding_metadata(message)
+ elif message.tool_call:
+ await self._handle_msg_tool_call(message)
+ elif message.session_resumption_update:
+ self._handle_msg_resumption_update(message)
+ except Exception as e:
+ if not self._disconnecting:
+ should_reconnect = await self._handle_connection_error(e)
+ if should_reconnect:
+ await self._reconnect()
+ return # Exit this connection handler, _reconnect will start a new one
+ break
+
+ def _check_and_reset_failure_counter(self):
+ """Check if connection has been stable long enough to reset the failure counter.
+
+ If the connection has been active for longer than the established threshold
+ and there are accumulated failures, reset the counter to 0.
+ """
+ if (
+ self._connection_start_time
+ and self._consecutive_failures > 0
+ and time.time() - self._connection_start_time >= CONNECTION_ESTABLISHED_THRESHOLD
+ ):
+ logger.info(
+ f"Connection stable for {CONNECTION_ESTABLISHED_THRESHOLD}s, "
+ f"resetting failure counter from {self._consecutive_failures} to 0"
+ )
+ self._consecutive_failures = 0
+
+ async def _handle_connection_error(self, error: Exception) -> bool:
+ """Handle a connection error and determine if reconnection should be attempted.
+
+ Args:
+ error: The exception that caused the connection error.
+
+ Returns:
+ True if reconnection should be attempted, False if a fatal error should be pushed.
+ """
+ self._consecutive_failures += 1
+ logger.warning(
+ f"Connection error (failure {self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES}): {error}"
+ )
+
+ if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES:
+ logger.error(
+ f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, "
+ "treating as fatal error"
+ )
+ await self.push_error(
+ ErrorFrame(error=f"{self} Error in receive loop: {error}", fatal=True)
+ )
+ return False
+ else:
+ logger.info(
+ f"Attempting reconnection ({self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES})"
+ )
+ return True
+
+ async def _reconnect(self):
+ """Reconnect to Gemini Live API."""
+ await self._disconnect()
+ await self._connect(session_resumption_handle=self._session_resumption_handle)
+
+ async def _disconnect(self):
+ """Disconnect from Gemini Live API and clean up resources."""
+ logger.info("Disconnecting from Gemini service")
+ try:
+ self._disconnecting = True
+ await self.stop_all_metrics()
+ if self._connection_task:
+ await self.cancel_task(self._connection_task, timeout=1.0)
+ self._connection_task = None
+ if self._session:
+ await self._session.close()
+ self._session = None
+ self._disconnecting = False
+ except Exception as e:
+ logger.error(f"{self} error disconnecting: {e}")
+
+ async def _send_user_audio(self, frame):
+ """Send user audio frame to Gemini Live API."""
+ if self._audio_input_paused or self._disconnecting or not self._session:
+ return
+
+ # Send all audio to Gemini
+ try:
+ await self._session.send_realtime_input(
+ audio=Blob(data=frame.audio, mime_type=f"audio/pcm;rate={frame.sample_rate}")
+ )
+ except Exception as e:
+ await self._handle_send_error(e)
+
+ # Manage a buffer of audio to use for transcription
+ audio = frame.audio
+ if self._user_is_speaking:
+ self._user_audio_buffer.extend(audio)
+ else:
+ # Keep 1/2 second of audio in the buffer even when not speaking.
+ self._user_audio_buffer.extend(audio)
+ length = int((frame.sample_rate * frame.num_channels * 2) * 0.5)
+ self._user_audio_buffer = self._user_audio_buffer[-length:]
+
+ async def _send_user_text(self, text: str):
+ """Send user text via Gemini Live API's realtime input stream.
+
+ This method sends text through the realtimeInput stream (via TextInputMessage)
+ rather than the clientContent stream. This ensures text input is synchronized
+ with audio and video inputs, preventing temporal misalignment that can occur
+ when different modalities are processed through separate API pathways.
+
+ For realtimeInput, turn completion is automatically inferred by the API based
+ on user activity, so no explicit turnComplete signal is needed.
+
+ Args:
+ text: The text to send as user input.
+ """
+ if self._disconnecting or not self._session:
+ return
+
+ try:
+ await self._session.send_realtime_input(text=text)
+ except Exception as e:
+ await self._handle_send_error(e)
+
+ async def _send_user_video(self, frame):
+ """Send user video frame to Gemini Live API."""
+ if self._video_input_paused or self._disconnecting or not self._session:
+ return
+
+ now = time.time()
+ if now - self._last_sent_time < 1:
+ return # Ignore if less than 1 second has passed
+
+ self._last_sent_time = now # Update last sent time
+ logger.debug(f"Sending video frame to Gemini: {frame}")
+
+ buffer = io.BytesIO()
+ Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
+ data = base64.b64encode(buffer.getvalue()).decode("utf-8")
+
+ try:
+ await self._session.send_realtime_input(video=Blob(data=data, mime_type="image/jpeg"))
+ except Exception as e:
+ await self._handle_send_error(e)
+
+ async def _create_initial_response(self):
+ """Create initial response based on context history."""
+ if self._disconnecting:
+ return
+
+ if not self._session:
+ self._run_llm_when_session_ready = True
+ return
+
+ messages = self._context.get_messages_for_initializing_history()
+ if not messages:
+ return
+
+ logger.debug(f"Creating initial response: {messages}")
+
+ await self.start_ttfb_metrics()
+
+ try:
+ await self._session.send_client_content(
+ turns=messages, turn_complete=self._inference_on_context_initialization
+ )
+ except Exception as e:
+ await self._handle_send_error(e)
+
+ # If we're generating a response right away upon initializing
+ # conversation history, set a flag saying that we need a turn complete
+ # message when the user stops speaking.
+ if not self._inference_on_context_initialization:
+ self._needs_turn_complete_message = True
+
+ async def _create_single_response(self, messages_list):
+ """Create a single response from a list of messages."""
+ if self._disconnecting or not self._session:
+ return
+
+ # Create a throwaway context just for the purpose of getting messages
+ # in the right format
+ context = GeminiLiveContext.upgrade(OpenAILLMContext(messages=messages_list))
+ messages = context.get_messages_for_initializing_history()
+
+ if not messages:
+ return
+
+ logger.debug(f"Creating response: {messages}")
+
+ await self.start_ttfb_metrics()
+
+ try:
+ await self._session.send_client_content(turns=messages, turn_complete=True)
+ except Exception as e:
+ await self._handle_send_error(e)
+
+ @traced_gemini_live(operation="llm_tool_result")
+ async def _tool_result(self, tool_result_message):
+ """Send tool result back to the API."""
+ if self._disconnecting or not self._session:
+ return
+
+ # For now we're shoving the name into the tool_call_id field, so this
+ # will work until we revisit that.
+ id = tool_result_message.get("tool_call_id")
+ name = tool_result_message.get("tool_call_name")
+ result = json.loads(tool_result_message.get("content") or "")
+ response = FunctionResponse(name=name, id=id, response=result)
+
+ try:
+ await self._session.send_tool_response(function_responses=response)
+ except Exception as e:
+ await self._handle_send_error(e)
+
+ @traced_gemini_live(operation="llm_setup")
+ async def _handle_session_ready(self, session: AsyncSession):
+ """Handle the session being ready."""
+ self._session = session
+ # If we were just waititng for the session to be ready to run the LLM,
+ # do that now.
+ if self._run_llm_when_session_ready:
+ self._run_llm_when_session_ready = False
+ await self._create_initial_response()
+
+ async def _handle_msg_model_turn(self, msg: LiveServerMessage):
+ """Handle the model turn message."""
+ part = msg.server_content.model_turn.parts[0]
+ if not part:
+ return
+
+ await self.stop_ttfb_metrics()
+
+ # part.text is added when `modalities` is set to TEXT; otherwise, it's None
+ text = part.text
+ if text:
+ if not self._bot_text_buffer:
+ await self.push_frame(LLMFullResponseStartFrame())
+
+ self._bot_text_buffer += text
+ self._search_result_buffer += text # Also accumulate for grounding
+ await self.push_frame(LLMTextFrame(text=text))
+
+ # Check for grounding metadata in server content
+ if msg.server_content and msg.server_content.grounding_metadata:
+ self._accumulated_grounding_metadata = msg.server_content.grounding_metadata
+
+ inline_data = part.inline_data
+ if not inline_data:
+ return
+
+ # Check if mime type matches expected format
+ expected_mime_type = f"audio/pcm;rate={self._sample_rate}"
+ if inline_data.mime_type == expected_mime_type:
+ # Perfect match, continue processing
+ pass
+ elif inline_data.mime_type == "audio/pcm":
+ # Sample rate not provided in mime type, assume default
+ if not hasattr(self, "_sample_rate_warning_logged"):
+ logger.warning(
+ f"Sample rate not provided in mime type '{inline_data.mime_type}', assuming rate of {self._sample_rate}"
+ )
+ self._sample_rate_warning_logged = True
+ else:
+ # Unrecognized format
+ logger.warning(f"Unrecognized server_content format {inline_data.mime_type}")
+ return
+
+ audio = inline_data.data
+ if not audio:
+ return
+
+ if not self._bot_is_speaking:
+ await self._set_bot_is_speaking(True)
+ await self.push_frame(TTSStartedFrame())
+ await self.push_frame(LLMFullResponseStartFrame())
+
+ self._bot_audio_buffer.extend(audio)
+ frame = TTSAudioRawFrame(
+ audio=audio,
+ sample_rate=self._sample_rate,
+ num_channels=1,
+ )
+ await self.push_frame(frame)
+
+ @traced_gemini_live(operation="llm_tool_call")
+ async def _handle_msg_tool_call(self, message: LiveServerMessage):
+ """Handle tool call messages."""
+ function_calls = message.tool_call.function_calls
+ if not function_calls:
+ return
+ if not self._context:
+ logger.error("Function calls are not supported without a context object.")
+
+ function_calls_llm = [
+ FunctionCallFromLLM(
+ context=self._context,
+ tool_call_id=(
+ # NOTE: when using Vertex AI we don't get server-provided
+ # tool call IDs here
+ f.id or str(uuid.uuid4())
+ ),
+ function_name=f.name,
+ arguments=f.args,
+ )
+ for f in function_calls
+ ]
+
+ await self.run_function_calls(function_calls_llm)
+
+ @traced_gemini_live(operation="llm_response")
+ async def _handle_msg_turn_complete(self, message: LiveServerMessage):
+ """Handle the turn complete message."""
+ await self._set_bot_is_speaking(False)
+ text = self._bot_text_buffer
+
+ # Trace the complete LLM response (this will be handled by the decorator)
+ # The decorator will extract the output text and usage metadata from the message
+
+ self._bot_text_buffer = ""
+ self._llm_output_buffer = ""
+
+ # Process grounding metadata if we have accumulated any
+ if self._accumulated_grounding_metadata:
+ await self._process_grounding_metadata(
+ self._accumulated_grounding_metadata, self._search_result_buffer
+ )
+
+ # Reset grounding tracking for next response
+ self._search_result_buffer = ""
+ self._accumulated_grounding_metadata = None
+
+ # Only push the TTSStoppedFrame if the bot is outputting audio
+ # when text is found, modalities is set to TEXT and no audio
+ # is produced.
+ if not text:
+ await self.push_frame(TTSStoppedFrame())
+
+ await self.push_frame(LLMFullResponseEndFrame())
+
+ @traced_stt
+ async def _handle_user_transcription(
+ self, transcript: str, is_final: bool, language: Optional[Language] = None
+ ):
+ """Handle a transcription result with tracing."""
+ pass
+
+ async def _handle_msg_input_transcription(self, message: LiveServerMessage):
+ """Handle the input transcription message.
+
+ Gemini Live sends user transcriptions in either single words or multi-word
+ phrases. As a result, we have to aggregate the input transcription. This handler
+ aggregates into sentences, splitting on the end of sentence markers.
+ """
+ if not message.server_content.input_transcription:
+ return
+
+ text = message.server_content.input_transcription.text
+
+ if not text:
+ return
+
+ # Strip leading space from sentence starts if buffer is empty
+ if text.startswith(" ") and not self._user_transcription_buffer:
+ text = text.lstrip()
+
+ # Accumulate text in the buffer
+ self._user_transcription_buffer += text
+
+ # Check for complete sentences
+ while True:
+ eos_end_marker = match_endofsentence(self._user_transcription_buffer)
+ if not eos_end_marker:
+ break
+
+ # Extract the complete sentence
+ complete_sentence = self._user_transcription_buffer[:eos_end_marker]
+ # Keep the remainder for the next chunk
+ self._user_transcription_buffer = self._user_transcription_buffer[eos_end_marker:]
+
+ # Send a TranscriptionFrame with the complete sentence
+ logger.debug(f"[Transcription:user] [{complete_sentence}]")
+ await self._handle_user_transcription(
+ complete_sentence, True, self._settings["language"]
+ )
+ await self.push_frame(
+ TranscriptionFrame(
+ text=complete_sentence,
+ user_id="",
+ timestamp=time_now_iso8601(),
+ result=message,
+ ),
+ FrameDirection.UPSTREAM,
+ )
+
+ async def _handle_msg_output_transcription(self, message: LiveServerMessage):
+ """Handle the output transcription message."""
+ if not message.server_content.output_transcription:
+ return
+
+ # This is the output transcription text when modalities is set to AUDIO.
+ # In this case, we push LLMTextFrame and TTSTextFrame to be handled by the
+ # downstream assistant context aggregator.
+ text = message.server_content.output_transcription.text
+
+ if not text:
+ return
+
+ # Accumulate text for grounding as well
+ self._search_result_buffer += text
+
+ # Check for grounding metadata in server content
+ if message.server_content and message.server_content.grounding_metadata:
+ self._accumulated_grounding_metadata = message.server_content.grounding_metadata
+ # Collect text for tracing
+ self._llm_output_buffer += text
+
+ await self.push_frame(LLMTextFrame(text=text))
+ await self.push_frame(TTSTextFrame(text=text))
+
+ async def _handle_msg_grounding_metadata(self, message: LiveServerMessage):
+ """Handle dedicated grounding metadata messages."""
+ if message.server_content and message.server_content.grounding_metadata:
+ grounding_metadata = message.server_content.grounding_metadata
+ # Process the grounding metadata immediately
+ await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer)
+
+ async def _process_grounding_metadata(
+ self, grounding_metadata: GroundingMetadata, search_result: str = ""
+ ):
+ """Process grounding metadata and emit LLMSearchResponseFrame."""
+ if not grounding_metadata:
+ return
+
+ # Extract rendered content for search suggestions
+ rendered_content = None
+ if (
+ grounding_metadata.search_entry_point
+ and grounding_metadata.search_entry_point.rendered_content
+ ):
+ rendered_content = grounding_metadata.search_entry_point.rendered_content
+
+ # Convert grounding chunks and supports to LLMSearchOrigin format
+ origins = []
+
+ if grounding_metadata.grounding_chunks and grounding_metadata.grounding_supports:
+ # Create a mapping of chunk indices to origins
+ chunk_to_origin: Dict[int, LLMSearchOrigin] = {}
+
+ for index, chunk in enumerate(grounding_metadata.grounding_chunks):
+ if chunk.web:
+ origin = LLMSearchOrigin(
+ site_uri=chunk.web.uri, site_title=chunk.web.title, results=[]
+ )
+ chunk_to_origin[index] = origin
+ origins.append(origin)
+
+ # Add grounding support results to the appropriate origins
+ for support in grounding_metadata.grounding_supports:
+ if support.segment and support.grounding_chunk_indices:
+ text = support.segment.text or ""
+ confidence_scores = support.confidence_scores or []
+
+ # Add this result to all origins referenced by this support
+ for chunk_index in support.grounding_chunk_indices:
+ if chunk_index in chunk_to_origin:
+ result = LLMSearchResult(text=text, confidence=confidence_scores)
+ chunk_to_origin[chunk_index].results.append(result)
+
+ # Create and push the search response frame
+ search_frame = LLMSearchResponseFrame(
+ search_result=search_result, origins=origins, rendered_content=rendered_content
+ )
+
+ await self.push_frame(search_frame)
+
+ async def _handle_msg_usage_metadata(self, message: LiveServerMessage):
+ """Handle the usage metadata message."""
+ if not message.usage_metadata:
+ return
+
+ usage = message.usage_metadata
+
+ # Ensure we have valid integers for all token counts
+ prompt_tokens = usage.prompt_token_count or 0
+ completion_tokens = usage.response_token_count or 0
+ total_tokens = usage.total_token_count or (prompt_tokens + completion_tokens)
+
+ tokens = LLMTokenUsage(
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ total_tokens=total_tokens,
+ )
+
+ await self.start_llm_usage_metrics(tokens)
+
+ def _handle_msg_resumption_update(self, message: LiveServerMessage):
+ update = message.session_resumption_update
+ if update.resumable and update.new_handle:
+ self._session_resumption_handle = update.new_handle
+
+ async def _handle_send_error(self, error: Exception):
+ # In server-to-server contexts, a WebSocket error should be quite rare.
+ # Given how hard it is to recover from a send-side error with proper
+ # state management, and that exponential backoff for retries can have
+ # cost/stability implications for a service cluster, let's just treat a
+ # send-side error as fatal.
+ if not self._disconnecting:
+ await self.push_error(ErrorFrame(error=f"{self} Send error: {error}", fatal=True))
+
+ def create_context_aggregator(
+ self,
+ context: OpenAILLMContext,
+ *,
+ user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
+ assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
+ ) -> GeminiLiveContextAggregatorPair:
+ """Create an instance of GeminiLiveContextAggregatorPair from an OpenAILLMContext.
+
+ Constructor keyword arguments for both the user and assistant aggregators can be provided.
+
+ Args:
+ context: The LLM context to use.
+ user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams().
+ assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams().
+
+ Returns:
+ GeminiLiveContextAggregatorPair: A pair of context
+ aggregators, one for the user and one for the assistant,
+ encapsulated in an GeminiLiveContextAggregatorPair.
+ """
+ context.set_llm_adapter(self.get_llm_adapter())
+
+ GeminiLiveContext.upgrade(context)
+ user = GeminiLiveUserContextAggregator(context, params=user_params)
+
+ assistant_params.expect_stripped_words = False
+ assistant = GeminiLiveAssistantContextAggregator(context, params=assistant_params)
+ return GeminiLiveContextAggregatorPair(_user=user, _assistant=assistant)
diff --git a/src/pipecat/services/gemini_multimodal_live/vertex.py b/src/pipecat/services/gemini_live/vertex.py
similarity index 89%
rename from src/pipecat/services/gemini_multimodal_live/vertex.py
rename to src/pipecat/services/gemini_live/vertex.py
index e07eb474f..c03f589b8 100644
--- a/src/pipecat/services/gemini_multimodal_live/vertex.py
+++ b/src/pipecat/services/gemini_live/vertex.py
@@ -4,9 +4,9 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
-"""Google Vertex AI Gemini Multimodal Live service.
+"""Service for accessing Gemini Live via Google Vertex AI.
-This module provides integration with Google's Gemini Multimodal Live model via
+This module provides integration with Google's Gemini Live model via
Vertex AI, supporting both text and audio modalities with voice transcription,
streaming responses, and tool usage.
"""
@@ -18,8 +18,8 @@ from typing import List, Optional, Union
from loguru import logger
from pipecat.adapters.schemas.tools_schema import ToolsSchema
-from pipecat.services.gemini_multimodal_live.gemini import (
- GeminiMultimodalLiveLLMService,
+from pipecat.services.gemini_live.gemini import (
+ GeminiLiveLLMService,
HttpOptions,
InputParams,
)
@@ -39,12 +39,12 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
-class GeminiVertexMultimodalLiveLLMService(GeminiMultimodalLiveLLMService):
- """Google Vertex AI Gemini Multimodal Live service.
+class GeminiLiveVertexLLMService(GeminiLiveLLMService):
+ """Provides access to Google's Gemini Live model via Vertex AI.
- Provides access to Google's Gemini Multimodal Live model via Vertex AI,
- supporting both text and audio modalities. It handles voice transcription,
- streaming audio responses, and tool usage.
+ This service enables real-time conversations with Gemini, supporting both
+ text and audio modalities. It handles voice transcription, streaming audio
+ responses, and tool usage.
"""
def __init__(
@@ -66,7 +66,7 @@ class GeminiVertexMultimodalLiveLLMService(GeminiMultimodalLiveLLMService):
http_options: Optional[HttpOptions] = None,
**kwargs,
):
- """Initialize the Google Vertex AI Gemini Multimodal Live service.
+ """Initialize the service for accessing Gemini Live via Google Vertex AI.
Args:
credentials: JSON string of service account credentials.
@@ -85,13 +85,13 @@ class GeminiVertexMultimodalLiveLLMService(GeminiMultimodalLiveLLMService):
is first set. Defaults to True.
file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint.
http_options: HTTP options for the client.
- **kwargs: Additional arguments passed to parent GeminiMultimodalLiveLLMService.
+ **kwargs: Additional arguments passed to parent GeminiLiveLLMService.
"""
# Check if user incorrectly passed api_key, which is used by parent
# class but not here.
if "api_key" in kwargs:
logger.error(
- "GeminiVertexMultimodalLiveLLMService does not accept 'api_key' parameter. "
+ "GeminiLiveVertexLLMService does not accept 'api_key' parameter. "
"Use 'credentials' or 'credentials_path' instead for Vertex AI authentication."
)
raise ValueError(
diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py
index 9705cf9f2..be69033aa 100644
--- a/src/pipecat/services/gemini_multimodal_live/events.py
+++ b/src/pipecat/services/gemini_multimodal_live/events.py
@@ -30,12 +30,15 @@ except ModuleNotFoundError as e:
# These aliases are just here for backward compatibility, since we used to
# define public-facing StartSensitivity and EndSensitivity enums in this
# module.
-warnings.warn(
- "Importing StartSensitivity and EndSensitivity from "
- "pipecat.services.gemini_multimodal_live.events is deprecated. "
- "Please import them directly from google.genai.types instead.",
- DeprecationWarning,
- stacklevel=2,
-)
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Importing StartSensitivity and EndSensitivity from "
+ "pipecat.services.gemini_multimodal_live.events is deprecated. "
+ "Please import them directly from google.genai.types instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
StartSensitivity = _StartSensitivity
EndSensitivity = _EndSensitivity
diff --git a/src/pipecat/services/gemini_multimodal_live/file_api.py b/src/pipecat/services/gemini_multimodal_live/file_api.py
index 5ae7fdbb7..53f8442fc 100644
--- a/src/pipecat/services/gemini_multimodal_live/file_api.py
+++ b/src/pipecat/services/gemini_multimodal_live/file_api.py
@@ -9,181 +9,31 @@
This module provides a client for Google's Gemini File API, enabling file
uploads, metadata retrieval, listing, and deletion. Files uploaded through
this API can be referenced in Gemini generative model calls.
+
+.. deprecated:: 0.0.90
+ Importing GeminiFileAPI from this module is deprecated.
+ Import it from pipecat.services.gemini_live.file_api instead.
"""
-import mimetypes
-from typing import Any, Dict, Optional
+import warnings
-import aiohttp
from loguru import logger
+try:
+ from pipecat.services.gemini_live.file_api import GeminiFileAPI as _GeminiFileAPI
+except ModuleNotFoundError as e:
+ logger.error(f"Exception: {e}")
+ logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
+ raise Exception(f"Missing module: {e}")
-class GeminiFileAPI:
- """Client for the Gemini File API.
-
- This class provides methods for uploading, fetching, listing, and deleting files
- through Google's Gemini File API.
-
- Files uploaded through this API remain available for 48 hours and can be referenced
- in calls to the Gemini generative models. Maximum file size is 2GB, with total
- project storage limited to 20GB.
- """
-
- def __init__(
- self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files"
- ):
- """Initialize the Gemini File API client.
-
- Args:
- api_key: Google AI API key
- base_url: Base URL for the Gemini File API (default is the v1beta endpoint)
- """
- self._api_key = api_key
- self._base_url = base_url
- # Upload URL uses the /upload/ path
- self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files"
-
- async def upload_file(
- self, file_path: str, display_name: Optional[str] = None
- ) -> Dict[str, Any]:
- """Upload a file to the Gemini File API using the correct resumable upload protocol.
-
- Args:
- file_path: Path to the file to upload
- display_name: Optional display name for the file
-
- Returns:
- File metadata including uri, name, and display_name
- """
- logger.info(f"Uploading file: {file_path}")
-
- async with aiohttp.ClientSession() as session:
- # Determine the file's MIME type
- mime_type, _ = mimetypes.guess_type(file_path)
- if not mime_type:
- mime_type = "application/octet-stream"
-
- # Read the file
- with open(file_path, "rb") as f:
- file_data = f.read()
-
- # Create the metadata payload
- metadata = {}
- if display_name:
- metadata = {"file": {"display_name": display_name}}
-
- # Step 1: Initial resumable request to get upload URL
- headers = {
- "X-Goog-Upload-Protocol": "resumable",
- "X-Goog-Upload-Command": "start",
- "X-Goog-Upload-Header-Content-Length": str(len(file_data)),
- "X-Goog-Upload-Header-Content-Type": mime_type,
- "Content-Type": "application/json",
- }
-
- logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}")
- async with session.post(
- f"{self.upload_base_url}?key={self._api_key}", headers=headers, json=metadata
- ) as response:
- if response.status != 200:
- error_text = await response.text()
- logger.error(f"Error initiating file upload: {error_text}")
- raise Exception(f"Failed to initiate upload: {response.status} - {error_text}")
-
- # Get the upload URL from the response header
- upload_url = response.headers.get("X-Goog-Upload-URL")
- if not upload_url:
- logger.error(f"Response headers: {dict(response.headers)}")
- raise Exception("No upload URL in response headers")
-
- logger.debug(f"Got upload URL: {upload_url}")
-
- # Step 2: Upload the actual file data
- upload_headers = {
- "Content-Length": str(len(file_data)),
- "X-Goog-Upload-Offset": "0",
- "X-Goog-Upload-Command": "upload, finalize",
- }
-
- logger.debug(f"Step 2: Uploading file data to {upload_url}")
- async with session.post(upload_url, headers=upload_headers, data=file_data) as response:
- if response.status != 200:
- error_text = await response.text()
- logger.error(f"Error uploading file data: {error_text}")
- raise Exception(f"Failed to upload file: {response.status} - {error_text}")
-
- file_info = await response.json()
- logger.info(f"File uploaded successfully: {file_info.get('file', {}).get('name')}")
- return file_info
-
- async def get_file(self, name: str) -> Dict[str, Any]:
- """Get metadata for a file.
-
- Args:
- name: File name (or full path)
-
- Returns:
- File metadata
- """
- # Extract just the name part if a full path is provided
- if "/" in name:
- name = name.split("/")[-1]
-
- async with aiohttp.ClientSession() as session:
- async with session.get(f"{self._base_url}/{name}?key={self._api_key}") as response:
- if response.status != 200:
- error_text = await response.text()
- logger.error(f"Error getting file metadata: {error_text}")
- raise Exception(f"Failed to get file metadata: {response.status}")
-
- file_info = await response.json()
- return file_info
-
- async def list_files(
- self, page_size: int = 10, page_token: Optional[str] = None
- ) -> Dict[str, Any]:
- """List uploaded files.
-
- Args:
- page_size: Number of files to return per page
- page_token: Token for pagination
-
- Returns:
- List of files and next page token if available
- """
- params = {"key": self._api_key, "pageSize": page_size}
-
- if page_token:
- params["pageToken"] = page_token
-
- async with aiohttp.ClientSession() as session:
- async with session.get(self._base_url, params=params) as response:
- if response.status != 200:
- error_text = await response.text()
- logger.error(f"Error listing files: {error_text}")
- raise Exception(f"Failed to list files: {response.status}")
-
- result = await response.json()
- return result
-
- async def delete_file(self, name: str) -> bool:
- """Delete a file.
-
- Args:
- name: File name (or full path)
-
- Returns:
- True if deleted successfully
- """
- # Extract just the name part if a full path is provided
- if "/" in name:
- name = name.split("/")[-1]
-
- async with aiohttp.ClientSession() as session:
- async with session.delete(f"{self._base_url}/{name}?key={self._api_key}") as response:
- if response.status != 200:
- error_text = await response.text()
- logger.error(f"Error deleting file: {error_text}")
- raise Exception(f"Failed to delete file: {response.status}")
-
- return True
+# These aliases are just here for backward compatibility, since we used to
+# define public-facing StartSensitivity and EndSensitivity enums in this
+# module.
+warnings.warn(
+ "Importing GeminiFileAPI from "
+ "pipecat.services.gemini_multimodal_live.file_api is deprecated. "
+ "Please import it from pipecat.services.gemini_live.file_api instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
+GeminiFileAPI = _GeminiFileAPI
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index 8ab836173..4ac51e02e 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -4,1587 +4,54 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
-"""Google Gemini Multimodal Live API service implementation.
+"""Google Gemini Live API service implementation.
This module provides real-time conversational AI capabilities using Google's
-Gemini Multimodal Live API, supporting both text and audio modalities with
+Gemini Live API, supporting both text and audio modalities with
voice transcription, streaming responses, and tool usage.
+
+.. deprecated:: 0.0.90
+ This module is deprecated. Please use the equivalent types from
+ pipecat.services.gemini_live.gemini instead. Note that the new type names
+ do not include 'Multimodal'.
"""
-import base64
-import io
-import json
-import random
-import time
-import uuid
-from dataclasses import dataclass
-from enum import Enum
-from typing import Any, Dict, List, Optional, Union
+import warnings
-from loguru import logger
-from PIL import Image
-from pydantic import BaseModel, Field
+from pipecat.services.gemini_live.gemini import (
+ ContextWindowCompressionParams as _ContextWindowCompressionParams,
+)
+from pipecat.services.gemini_live.gemini import (
+ GeminiLiveAssistantContextAggregator,
+ GeminiLiveContext,
+ GeminiLiveContextAggregatorPair,
+ GeminiLiveLLMService,
+ GeminiLiveUserContextAggregator,
+ GeminiModalities,
+)
+from pipecat.services.gemini_live.gemini import GeminiMediaResolution as _GeminiMediaResolution
+from pipecat.services.gemini_live.gemini import GeminiVADParams as _GeminiVADParams
+from pipecat.services.gemini_live.gemini import InputParams as _InputParams
-from pipecat.adapters.schemas.tools_schema import ToolsSchema
-from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
-from pipecat.frames.frames import (
- BotStartedSpeakingFrame,
- BotStoppedSpeakingFrame,
- CancelFrame,
- EndFrame,
- ErrorFrame,
- Frame,
- InputAudioRawFrame,
- InputImageRawFrame,
- InputTextRawFrame,
- InterruptionFrame,
- LLMContextFrame,
- LLMFullResponseEndFrame,
- LLMFullResponseStartFrame,
- LLMMessagesAppendFrame,
- LLMSetToolsFrame,
- LLMTextFrame,
- LLMUpdateSettingsFrame,
- StartFrame,
- TranscriptionFrame,
- TTSAudioRawFrame,
- TTSStartedFrame,
- TTSStoppedFrame,
- TTSTextFrame,
- UserImageRawFrame,
- UserStartedSpeakingFrame,
- UserStoppedSpeakingFrame,
-)
-from pipecat.metrics.metrics import LLMTokenUsage
-from pipecat.processors.aggregators.llm_response import (
- LLMAssistantAggregatorParams,
- LLMUserAggregatorParams,
-)
-from pipecat.processors.aggregators.openai_llm_context import (
- OpenAILLMContext,
- OpenAILLMContextFrame,
-)
-from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult
-from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
-from pipecat.services.openai.llm import (
- OpenAIAssistantContextAggregator,
- OpenAIUserContextAggregator,
-)
-from pipecat.transcriptions.language import Language
-from pipecat.utils.string import match_endofsentence
-from pipecat.utils.time import time_now_iso8601
-from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt
-
-from .file_api import GeminiFileAPI
-
-try:
- from google.genai import Client
- from google.genai.live import AsyncSession
- from google.genai.types import (
- AudioTranscriptionConfig,
- AutomaticActivityDetection,
- Blob,
- Content,
- ContextWindowCompressionConfig,
- EndSensitivity,
- FileData,
- FunctionResponse,
- GenerationConfig,
- GroundingMetadata,
- HttpOptions,
- LiveConnectConfig,
- LiveServerMessage,
- MediaResolution,
- Modality,
- Part,
- ProactivityConfig,
- RealtimeInputConfig,
- SessionResumptionConfig,
- SlidingWindow,
- SpeechConfig,
- StartSensitivity,
- ThinkingConfig,
- VoiceConfig,
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.gemini_multimodal_live.gemini are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.gemini_live.gemini instead. Note that the new type "
+ "names do not include 'Multimodal' "
+ "(e.g. `GeminiMultimodalLiveLLMService` is now `GeminiLiveLLMService`).",
+ DeprecationWarning,
+ stacklevel=2,
)
-except ModuleNotFoundError as e:
- logger.error(f"Exception: {e}")
- logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
- raise Exception(f"Missing module: {e}")
-
-# Connection management constants
-MAX_CONSECUTIVE_FAILURES = 3
-CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds
-
-
-def language_to_gemini_language(language: Language) -> Optional[str]:
- """Maps a Language enum value to a Gemini Live supported language code.
-
- Source:
- https://ai.google.dev/api/generate-content#MediaResolution
-
- Args:
- language: The language enum value to convert.
-
- Returns:
- The Gemini language code string, or None if the language is not supported.
- """
- language_map = {
- # Arabic
- Language.AR: "ar-XA",
- # Bengali
- Language.BN_IN: "bn-IN",
- # Chinese (Mandarin)
- Language.CMN: "cmn-CN",
- Language.CMN_CN: "cmn-CN",
- Language.ZH: "cmn-CN", # Map general Chinese to Mandarin for Gemini
- Language.ZH_CN: "cmn-CN", # Map Simplified Chinese to Mandarin for Gemini
- # German
- Language.DE: "de-DE",
- Language.DE_DE: "de-DE",
- # English
- Language.EN: "en-US", # Default to US English (though not explicitly listed in supported codes)
- Language.EN_US: "en-US",
- Language.EN_AU: "en-AU",
- Language.EN_GB: "en-GB",
- Language.EN_IN: "en-IN",
- # Spanish
- Language.ES: "es-ES", # Default to Spain Spanish
- Language.ES_ES: "es-ES",
- Language.ES_US: "es-US",
- # French
- Language.FR: "fr-FR", # Default to France French
- Language.FR_FR: "fr-FR",
- Language.FR_CA: "fr-CA",
- # Gujarati
- Language.GU: "gu-IN",
- Language.GU_IN: "gu-IN",
- # Hindi
- Language.HI: "hi-IN",
- Language.HI_IN: "hi-IN",
- # Indonesian
- Language.ID: "id-ID",
- Language.ID_ID: "id-ID",
- # Italian
- Language.IT: "it-IT",
- Language.IT_IT: "it-IT",
- # Japanese
- Language.JA: "ja-JP",
- Language.JA_JP: "ja-JP",
- # Kannada
- Language.KN: "kn-IN",
- Language.KN_IN: "kn-IN",
- # Korean
- Language.KO: "ko-KR",
- Language.KO_KR: "ko-KR",
- # Malayalam
- Language.ML: "ml-IN",
- Language.ML_IN: "ml-IN",
- # Marathi
- Language.MR: "mr-IN",
- Language.MR_IN: "mr-IN",
- # Dutch
- Language.NL: "nl-NL",
- Language.NL_NL: "nl-NL",
- # Polish
- Language.PL: "pl-PL",
- Language.PL_PL: "pl-PL",
- # Portuguese (Brazil)
- Language.PT_BR: "pt-BR",
- # Russian
- Language.RU: "ru-RU",
- Language.RU_RU: "ru-RU",
- # Tamil
- Language.TA: "ta-IN",
- Language.TA_IN: "ta-IN",
- # Telugu
- Language.TE: "te-IN",
- Language.TE_IN: "te-IN",
- # Thai
- Language.TH: "th-TH",
- Language.TH_TH: "th-TH",
- # Turkish
- Language.TR: "tr-TR",
- Language.TR_TR: "tr-TR",
- # Vietnamese
- Language.VI: "vi-VN",
- Language.VI_VN: "vi-VN",
- }
- return language_map.get(language)
-
-
-class GeminiMultimodalLiveContext(OpenAILLMContext):
- """Extended OpenAI context for Gemini Multimodal Live API.
-
- Provides Gemini-specific context management including system instruction
- extraction and message format conversion for the Live API.
- """
-
- @staticmethod
- def upgrade(obj: OpenAILLMContext) -> "GeminiMultimodalLiveContext":
- """Upgrade an OpenAI context to Gemini context.
-
- Args:
- obj: The OpenAI context to upgrade.
-
- Returns:
- The upgraded Gemini context instance.
- """
- if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiMultimodalLiveContext):
- logger.debug(f"Upgrading to Gemini Multimodal Live Context: {obj}")
- obj.__class__ = GeminiMultimodalLiveContext
- obj._restructure_from_openai_messages()
- return obj
-
- def _restructure_from_openai_messages(self):
- pass
-
- def extract_system_instructions(self):
- """Extract system instructions from context messages.
-
- Returns:
- Combined system instruction text from all system messages.
- """
- system_instruction = ""
- for item in self.messages:
- if item.get("role") == "system":
- content = item.get("content", "")
- if content:
- if system_instruction and not system_instruction.endswith("\n"):
- system_instruction += "\n"
- system_instruction += str(content)
- return system_instruction
-
- def add_file_reference(self, file_uri: str, mime_type: str, text: Optional[str] = None):
- """Add a file reference to the context.
-
- This adds a user message with a file reference that will be sent during context initialization.
-
- Args:
- file_uri: URI of the uploaded file
- mime_type: MIME type of the file
- text: Optional text prompt to accompany the file
- """
- # Create parts list with file reference
- parts = []
- if text:
- parts.append({"type": "text", "text": text})
-
- # Add file reference part
- parts.append(
- {"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}}
- )
-
- # Add to messages
- message = {"role": "user", "content": parts}
- self.messages.append(message)
- logger.info(f"Added file reference to context: {file_uri}")
-
- def get_messages_for_initializing_history(self) -> List[Content]:
- """Get messages formatted for Gemini history initialization.
-
- Returns:
- List of messages in Gemini format for conversation history.
- """
- messages: List[Content] = []
- for item in self.messages:
- role = item.get("role")
-
- if role == "system":
- continue
-
- elif role == "assistant":
- role = "model"
-
- content = item.get("content")
- parts: List[Part] = []
- if isinstance(content, str):
- parts = [Part(text=content)]
- elif isinstance(content, list):
- for part in content:
- if part.get("type") == "text":
- parts.append(Part(text=part.get("text")))
- elif part.get("type") == "file_data":
- file_data = part.get("file_data", {})
- parts.append(
- Part(
- file_data=FileData(
- mime_type=file_data.get("mime_type"),
- file_uri=file_data.get("file_uri"),
- )
- )
- )
- else:
- logger.warning(f"Unsupported content type: {str(part)[:80]}")
- else:
- logger.warning(f"Unsupported content type: {str(content)[:80]}")
- messages.append(Content(role=role, parts=parts))
- return messages
-
-
-class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator):
- """User context aggregator for Gemini Multimodal Live.
-
- Extends OpenAI user aggregator to handle Gemini-specific message passing
- while maintaining compatibility with the standard aggregation pipeline.
- """
-
- async def process_frame(self, frame, direction):
- """Process incoming frames for user context aggregation.
-
- Args:
- frame: The frame to process.
- direction: The frame processing direction.
- """
- await super().process_frame(frame, direction)
- # kind of a hack just to pass the LLMMessagesAppendFrame through, but it's fine for now
- if isinstance(frame, LLMMessagesAppendFrame):
- await self.push_frame(frame, direction)
-
-
-class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator):
- """Assistant context aggregator for Gemini Multimodal Live.
-
- Handles assistant response aggregation while filtering out LLMTextFrames
- to prevent duplicate context entries, as Gemini Live pushes both
- LLMTextFrames and TTSTextFrames.
- """
-
- async def process_frame(self, frame: Frame, direction: FrameDirection):
- """Process incoming frames for assistant context aggregation.
-
- Args:
- frame: The frame to process.
- direction: The frame processing direction.
- """
- # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output,
- # but the GeminiMultimodalLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We
- # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames
- # are process. This ensures that the context gets only one set of messages.
- if not isinstance(frame, LLMTextFrame):
- await super().process_frame(frame, direction)
-
- async def handle_user_image_frame(self, frame: UserImageRawFrame):
- """Handle user image frames.
-
- Args:
- frame: The user image frame to handle.
- """
- # We don't want to store any images in the context. Revisit this later
- # when the API evolves.
- pass
-
-
-@dataclass
-class GeminiMultimodalLiveContextAggregatorPair:
- """Pair of user and assistant context aggregators for Gemini Multimodal Live.
-
- Parameters:
- _user: The user context aggregator instance.
- _assistant: The assistant context aggregator instance.
- """
-
- _user: GeminiMultimodalLiveUserContextAggregator
- _assistant: GeminiMultimodalLiveAssistantContextAggregator
-
- def user(self) -> GeminiMultimodalLiveUserContextAggregator:
- """Get the user context aggregator.
-
- Returns:
- The user context aggregator instance.
- """
- return self._user
-
- def assistant(self) -> GeminiMultimodalLiveAssistantContextAggregator:
- """Get the assistant context aggregator.
-
- Returns:
- The assistant context aggregator instance.
- """
- return self._assistant
-
-
-class GeminiMultimodalModalities(Enum):
- """Supported modalities for Gemini Multimodal Live.
-
- Parameters:
- TEXT: Text responses.
- AUDIO: Audio responses.
- """
-
- TEXT = "TEXT"
- AUDIO = "AUDIO"
-
-
-class GeminiMediaResolution(str, Enum):
- """Media resolution options for Gemini Multimodal Live.
-
- Parameters:
- UNSPECIFIED: Use default resolution setting.
- LOW: Low resolution with 64 tokens.
- MEDIUM: Medium resolution with 256 tokens.
- HIGH: High resolution with zoomed reframing and 256 tokens.
- """
-
- UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default
- LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens
- MEDIUM = "MEDIA_RESOLUTION_MEDIUM" # 256 tokens
- HIGH = "MEDIA_RESOLUTION_HIGH" # Zoomed reframing with 256 tokens
-
-
-class GeminiVADParams(BaseModel):
- """Voice Activity Detection parameters for Gemini Live.
-
- Parameters:
- disabled: Whether to disable VAD. Defaults to None.
- start_sensitivity: Sensitivity for speech start detection. Defaults to None.
- end_sensitivity: Sensitivity for speech end detection. Defaults to None.
- prefix_padding_ms: Prefix padding in milliseconds. Defaults to None.
- silence_duration_ms: Silence duration threshold in milliseconds. Defaults to None.
- """
-
- disabled: Optional[bool] = Field(default=None)
- start_sensitivity: Optional[StartSensitivity] = Field(default=None)
- end_sensitivity: Optional[EndSensitivity] = Field(default=None)
- prefix_padding_ms: Optional[int] = Field(default=None)
- silence_duration_ms: Optional[int] = Field(default=None)
-
-
-class ContextWindowCompressionParams(BaseModel):
- """Parameters for context window compression in Gemini Live.
-
- Parameters:
- enabled: Whether compression is enabled. Defaults to False.
- trigger_tokens: Token count to trigger compression. None uses 80% of context window.
- """
-
- enabled: bool = Field(default=False)
- trigger_tokens: Optional[int] = Field(
- default=None
- ) # None = use default (80% of context window)
-
-
-class InputParams(BaseModel):
- """Input parameters for Gemini Multimodal Live generation.
-
- Parameters:
- frequency_penalty: Frequency penalty for generation (0.0-2.0). Defaults to None.
- max_tokens: Maximum tokens to generate. Must be >= 1. Defaults to 4096.
- presence_penalty: Presence penalty for generation (0.0-2.0). Defaults to None.
- temperature: Sampling temperature (0.0-2.0). Defaults to None.
- top_k: Top-k sampling parameter. Must be >= 0. Defaults to None.
- top_p: Top-p sampling parameter (0.0-1.0). Defaults to None.
- modalities: Response modalities. Defaults to AUDIO.
- language: Language for generation. Defaults to EN_US.
- media_resolution: Media resolution setting. Defaults to UNSPECIFIED.
- vad: Voice activity detection parameters. Defaults to None.
- context_window_compression: Context compression settings. Defaults to None.
- thinking: Thinking settings. Defaults to None.
- Note that these settings may require specifying a model that
- supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025".
- enable_affective_dialog: Enable affective dialog, which allows Gemini
- to adapt to expression and tone. Defaults to None.
- Note that these settings may require specifying a model that
- supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025".
- Also note that this setting may require specifying an API version that
- supports it, e.g. HttpOptions(api_version="v1alpha").
- proactivity: Proactivity settings, which allows Gemini to proactively
- decide how to behave, such as whether to avoid responding to
- content that is not relevant. Defaults to None.
- Note that these settings may require specifying a model that
- supports them, e.g. "gemini-2.5-flash-native-audio-preview-09-2025".
- Also note that this setting may require specifying an API version that
- supports it, e.g. HttpOptions(api_version="v1alpha").
- extra: Additional parameters. Defaults to empty dict.
- """
-
- frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0)
- max_tokens: Optional[int] = Field(default=4096, ge=1)
- presence_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0)
- temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
- top_k: Optional[int] = Field(default=None, ge=0)
- top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0)
- modalities: Optional[GeminiMultimodalModalities] = Field(
- default=GeminiMultimodalModalities.AUDIO
- )
- language: Optional[Language] = Field(default=Language.EN_US)
- media_resolution: Optional[GeminiMediaResolution] = Field(
- default=GeminiMediaResolution.UNSPECIFIED
- )
- vad: Optional[GeminiVADParams] = Field(default=None)
- context_window_compression: Optional[ContextWindowCompressionParams] = Field(default=None)
- thinking: Optional[ThinkingConfig] = Field(default=None)
- enable_affective_dialog: Optional[bool] = Field(default=None)
- proactivity: Optional[ProactivityConfig] = Field(default=None)
- extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
-
-
-class GeminiMultimodalLiveLLMService(LLMService):
- """Provides access to Google's Gemini Multimodal Live API.
-
- This service enables real-time conversations with Gemini, supporting both
- text and audio modalities. It handles voice transcription, streaming audio
- responses, and tool usage.
- """
-
- # Overriding the default adapter to use the Gemini one.
- adapter_class = GeminiLLMAdapter
-
- def __init__(
- self,
- *,
- api_key: str,
- base_url: Optional[str] = None,
- model="models/gemini-2.0-flash-live-001",
- voice_id: str = "Charon",
- start_audio_paused: bool = False,
- start_video_paused: bool = False,
- system_instruction: Optional[str] = None,
- tools: Optional[Union[List[dict], ToolsSchema]] = None,
- params: Optional[InputParams] = None,
- inference_on_context_initialization: bool = True,
- file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files",
- http_options: Optional[HttpOptions] = None,
- **kwargs,
- ):
- """Initialize the Gemini Multimodal Live LLM service.
-
- Args:
- api_key: Google AI API key for authentication.
- base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint.
-
- .. deprecated:: 0.0.90
- This parameter is deprecated and no longer has any effect.
- Please use `http_options` to customize requests made by the
- API client.
-
- model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001".
- voice_id: TTS voice identifier. Defaults to "Charon".
- start_audio_paused: Whether to start with audio input paused. Defaults to False.
- start_video_paused: Whether to start with video input paused. Defaults to False.
- system_instruction: System prompt for the model. Defaults to None.
- tools: Tools/functions available to the model. Defaults to None.
- params: Configuration parameters for the model. Defaults to InputParams().
- inference_on_context_initialization: Whether to generate a response when context
- is first set. Defaults to True.
- file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint.
- http_options: HTTP options for the client.
- **kwargs: Additional arguments passed to parent LLMService.
- """
- # Check for deprecated parameter usage
- if base_url is not None:
- import warnings
-
- with warnings.catch_warnings():
- warnings.simplefilter("always")
- warnings.warn(
- "Parameter 'base_url' is deprecated and no longer has any effect. Please use 'http_options' to customize requests made by the API client.",
- DeprecationWarning,
- stacklevel=2,
- )
-
- super().__init__(base_url=base_url, **kwargs)
-
- params = params or InputParams()
-
- self._last_sent_time = 0
- self._base_url = base_url
- self.set_model_name(model)
- self._voice_id = voice_id
- self._language_code = params.language
-
- self._system_instruction = system_instruction
- self._tools = tools
- self._inference_on_context_initialization = inference_on_context_initialization
- self._needs_turn_complete_message = False
-
- self._audio_input_paused = start_audio_paused
- self._video_input_paused = start_video_paused
- self._context = None
- self._api_key = api_key
- self._http_options = http_options
- self._session: AsyncSession = None
- self._connection_task = None
-
- self._disconnecting = False
- self._run_llm_when_session_ready = False
-
- self._user_is_speaking = False
- self._bot_is_speaking = False
- self._user_audio_buffer = bytearray()
- self._user_transcription_buffer = ""
- self._last_transcription_sent = ""
- self._bot_audio_buffer = bytearray()
- self._bot_text_buffer = ""
- self._llm_output_buffer = ""
-
- self._sample_rate = 24000
-
- self._language = params.language
- self._language_code = (
- language_to_gemini_language(params.language) if params.language else "en-US"
- )
- self._vad_params = params.vad
-
- # Reconnection tracking
- self._consecutive_failures = 0
- self._connection_start_time = None
-
- self._settings = {
- "frequency_penalty": params.frequency_penalty,
- "max_tokens": params.max_tokens,
- "presence_penalty": params.presence_penalty,
- "temperature": params.temperature,
- "top_k": params.top_k,
- "top_p": params.top_p,
- "modalities": params.modalities,
- "language": self._language_code,
- "media_resolution": params.media_resolution,
- "vad": params.vad,
- "context_window_compression": params.context_window_compression.model_dump()
- if params.context_window_compression
- else {},
- "thinking": params.thinking or {},
- "enable_affective_dialog": params.enable_affective_dialog or False,
- "proactivity": params.proactivity or {},
- "extra": params.extra if isinstance(params.extra, dict) else {},
- }
-
- self._file_api_base_url = file_api_base_url
- self._file_api: Optional[GeminiFileAPI] = None
-
- # Grounding metadata tracking
- self._search_result_buffer = ""
- self._accumulated_grounding_metadata = None
-
- # Session resumption
- self._session_resumption_handle: Optional[str] = None
-
- # Bookkeeping for ending gracefully (i.e. after the bot is finished)
- self._end_frame_pending_bot_turn_finished: Optional[EndFrame] = None
-
- # Initialize the API client. Subclasses can override this if needed.
- self.create_client()
-
- def create_client(self):
- """Create the Gemini API client instance. Subclasses can override this."""
- self._client = Client(api_key=self._api_key, http_options=self._http_options)
-
- @property
- def file_api(self) -> GeminiFileAPI:
- """Get the Gemini File API client instance. Subclasses can override this.
-
- Returns:
- The Gemini File API client.
- """
- if not self._file_api:
- self._file_api = GeminiFileAPI(api_key=self._api_key, base_url=self._file_api_base_url)
- return self._file_api
-
- def can_generate_metrics(self) -> bool:
- """Check if the service can generate usage metrics.
-
- Returns:
- True as Gemini Live supports token usage metrics.
- """
- return True
-
- def needs_mcp_alternate_schema(self) -> bool:
- """Check if this LLM service requires alternate MCP schema.
-
- Google/Gemini has stricter JSON schema validation and requires
- certain properties to be removed or modified for compatibility.
-
- Returns:
- True for Google/Gemini services.
- """
- return True
-
- def set_audio_input_paused(self, paused: bool):
- """Set the audio input pause state.
-
- Args:
- paused: Whether to pause audio input.
- """
- self._audio_input_paused = paused
-
- def set_video_input_paused(self, paused: bool):
- """Set the video input pause state.
-
- Args:
- paused: Whether to pause video input.
- """
- self._video_input_paused = paused
-
- def set_model_modalities(self, modalities: GeminiMultimodalModalities):
- """Set the model response modalities.
-
- Args:
- modalities: The modalities to use for responses.
- """
- self._settings["modalities"] = modalities
-
- def set_language(self, language: Language):
- """Set the language for generation.
-
- Args:
- language: The language to use for generation.
- """
- self._language = language
- self._language_code = language_to_gemini_language(language) or "en-US"
- self._settings["language"] = self._language_code
- logger.info(f"Set Gemini language to: {self._language_code}")
-
- async def set_context(self, context: OpenAILLMContext):
- """Set the context explicitly from outside the pipeline.
-
- This is useful when initializing a conversation because in server-side VAD mode we might not have a
- way to trigger the pipeline. This sends the history to the server. The `inference_on_context_initialization`
- flag controls whether to set the turnComplete flag when we do this. Without that flag, the model will
- not respond. This is often what we want when setting the context at the beginning of a conversation.
-
- Args:
- context: The OpenAI LLM context to set.
- """
- if self._context:
- logger.error(
- "Context already set. Can only set up Gemini Multimodal Live context once."
- )
- return
- self._context = GeminiMultimodalLiveContext.upgrade(context)
- await self._create_initial_response()
-
- #
- # standard AIService frame handling
- #
-
- async def start(self, frame: StartFrame):
- """Start the service and establish connection.
-
- Args:
- frame: The start frame.
- """
- await super().start(frame)
- await self._connect()
-
- async def stop(self, frame: EndFrame):
- """Stop the service and close connections.
-
- Args:
- frame: The end frame.
- """
- await super().stop(frame)
- await self._disconnect()
-
- async def cancel(self, frame: CancelFrame):
- """Cancel the service and close connections.
-
- Args:
- frame: The cancel frame.
- """
- await super().cancel(frame)
- await self._disconnect()
-
- #
- # speech and interruption handling
- #
-
- async def _handle_interruption(self):
- await self._set_bot_is_speaking(False)
- await self.push_frame(TTSStoppedFrame())
- await self.push_frame(LLMFullResponseEndFrame())
-
- async def _handle_user_started_speaking(self, frame):
- self._user_is_speaking = True
- pass
-
- async def _handle_user_stopped_speaking(self, frame):
- self._user_is_speaking = False
- self._user_audio_buffer = bytearray()
- await self.start_ttfb_metrics()
- if self._needs_turn_complete_message:
- self._needs_turn_complete_message = False
- # NOTE: without this, the model ignores the context it's been
- # seeded with before the user started speaking
- await self._session.send_client_content(turn_complete=True)
-
- #
- # frame processing
- #
- # StartFrame, StopFrame, CancelFrame implemented in base class
- #
-
- async def process_frame(self, frame: Frame, direction: FrameDirection):
- """Process incoming frames for the Gemini Live service.
-
- Args:
- frame: The frame to process.
- direction: The frame processing direction.
- """
- # Defer EndFrame handling until after the bot turn is finished
- if isinstance(frame, EndFrame):
- if self._bot_is_speaking:
- logger.debug("Deferring handling EndFrame until bot turn is finished")
- self._end_frame_pending_bot_turn_finished = frame
- return
-
- await super().process_frame(frame, direction)
-
- if isinstance(frame, TranscriptionFrame):
- await self.push_frame(frame, direction)
- elif isinstance(frame, OpenAILLMContextFrame):
- context: GeminiMultimodalLiveContext = GeminiMultimodalLiveContext.upgrade(
- frame.context
- )
- # For now, we'll only trigger inference here when either:
- # 1. We have not seen a context frame before
- # 2. The last message is a tool call result
- if not self._context:
- self._context = context
- if frame.context.tools:
- self._tools = frame.context.tools
- await self._create_initial_response()
- elif context.messages and context.messages[-1].get("role") == "tool":
- # Support just one tool call per context frame for now
- tool_result_message = context.messages[-1]
- await self._tool_result(tool_result_message)
- elif isinstance(frame, LLMContextFrame):
- raise NotImplementedError(
- "Universal LLMContext is not yet supported for Gemini Multimodal Live."
- )
- elif isinstance(frame, InputTextRawFrame):
- await self._send_user_text(frame.text)
- await self.push_frame(frame, direction)
- elif isinstance(frame, InputAudioRawFrame):
- await self._send_user_audio(frame)
- await self.push_frame(frame, direction)
- elif isinstance(frame, InputImageRawFrame):
- await self._send_user_video(frame)
- await self.push_frame(frame, direction)
- elif isinstance(frame, InterruptionFrame):
- await self._handle_interruption()
- await self.push_frame(frame, direction)
- elif isinstance(frame, UserStartedSpeakingFrame):
- await self._handle_user_started_speaking(frame)
- await self.push_frame(frame, direction)
- elif isinstance(frame, UserStoppedSpeakingFrame):
- await self._handle_user_stopped_speaking(frame)
- await self.push_frame(frame, direction)
- elif isinstance(frame, BotStartedSpeakingFrame):
- # Ignore this frame. Use the serverContent API message instead
- await self.push_frame(frame, direction)
- elif isinstance(frame, BotStoppedSpeakingFrame):
- # ignore this frame. Use the serverContent.turnComplete API message
- await self.push_frame(frame, direction)
- elif isinstance(frame, LLMMessagesAppendFrame):
- # NOTE: handling LLMMessagesAppendFrame here in the LLMService is
- # unusual - typically this would be handled in the user context
- # aggregator. Leaving this handling here so that user code that
- # uses this frame *without* a user context aggregator still works
- # (we have an example that does just that, actually).
- await self._create_single_response(frame.messages)
- elif isinstance(frame, LLMUpdateSettingsFrame):
- await self._update_settings(frame.settings)
- elif isinstance(frame, LLMSetToolsFrame):
- await self._update_settings()
- else:
- await self.push_frame(frame, direction)
-
- async def _set_bot_is_speaking(self, speaking: bool):
- if self._bot_is_speaking == speaking:
- return
-
- self._bot_is_speaking = speaking
-
- if not self._bot_is_speaking and self._end_frame_pending_bot_turn_finished:
- await self.queue_frame(self._end_frame_pending_bot_turn_finished)
- self._end_frame_pending_bot_turn_finished = None
-
- async def _connect(self, session_resumption_handle: Optional[str] = None):
- """Establish client connection to Gemini Live API."""
- if self._session:
- # Here we assume that if we have a client, we are connected. We
- # handle disconnections in the send/recv code paths.
- return
-
- if session_resumption_handle:
- logger.info(
- f"Connecting to Gemini service with session_resumption_handle: {session_resumption_handle}"
- )
- else:
- logger.info("Connecting to Gemini service")
- try:
- # Assemble basic configuration
- config = LiveConnectConfig(
- generation_config=GenerationConfig(
- frequency_penalty=self._settings["frequency_penalty"],
- max_output_tokens=self._settings["max_tokens"],
- presence_penalty=self._settings["presence_penalty"],
- temperature=self._settings["temperature"],
- top_k=self._settings["top_k"],
- top_p=self._settings["top_p"],
- response_modalities=[Modality(self._settings["modalities"].value)],
- speech_config=SpeechConfig(
- voice_config=VoiceConfig(
- prebuilt_voice_config={"voice_name": self._voice_id}
- ),
- language_code=self._settings["language"],
- ),
- media_resolution=MediaResolution(self._settings["media_resolution"].value),
- ),
- input_audio_transcription=AudioTranscriptionConfig(),
- output_audio_transcription=AudioTranscriptionConfig(),
- session_resumption=SessionResumptionConfig(handle=session_resumption_handle),
- )
-
- # Add context window compression to configuration, if enabled
- if self._settings.get("context_window_compression", {}).get("enabled", False):
- compression_config = ContextWindowCompressionConfig()
-
- # Add sliding window (always true if compression is enabled)
- compression_config.sliding_window = SlidingWindow()
-
- # Add trigger_tokens if specified
- trigger_tokens = self._settings.get("context_window_compression", {}).get(
- "trigger_tokens"
- )
- if trigger_tokens is not None:
- compression_config.trigger_tokens = trigger_tokens
-
- config.context_window_compression = compression_config
-
- # Add thinking configuration to configuration, if provided
- if self._settings.get("thinking"):
- config.thinking_config = self._settings["thinking"]
-
- # Add affective dialog setting, if provided
- if self._settings.get("enable_affective_dialog", False):
- config.enable_affective_dialog = self._settings["enable_affective_dialog"]
-
- # Add proactivity configuration to configuration, if provided
- if self._settings.get("proactivity"):
- config.proactivity = self._settings["proactivity"]
-
- # Add VAD configuration to configuration, if provided
- if self._settings.get("vad"):
- vad_config = AutomaticActivityDetection()
- vad_params = self._settings["vad"]
- has_vad_settings = False
-
- # Only add parameters that are explicitly set
- if vad_params.disabled is not None:
- vad_config.disabled = vad_params.disabled
- has_vad_settings = True
-
- if vad_params.start_sensitivity:
- vad_config.start_of_speech_sensitivity = vad_params.start_sensitivity
- has_vad_settings = True
-
- if vad_params.end_sensitivity:
- vad_config.end_of_speech_sensitivity = vad_params.end_sensitivity
- has_vad_settings = True
-
- if vad_params.prefix_padding_ms is not None:
- vad_config.prefix_padding_ms = vad_params.prefix_padding_ms
- has_vad_settings = True
-
- if vad_params.silence_duration_ms is not None:
- vad_config.silence_duration_ms = vad_params.silence_duration_ms
- has_vad_settings = True
-
- # Only add automatic_activity_detection if we have VAD settings
- if has_vad_settings:
- config.realtime_input_config = RealtimeInputConfig(
- automatic_activity_detection=vad_config
- )
-
- # Add system instruction to configuration, if provided
- system_instruction = self._system_instruction or ""
- if self._context and hasattr(self._context, "extract_system_instructions"):
- system_instruction += "\n" + self._context.extract_system_instructions()
- if system_instruction:
- logger.debug(f"Setting system instruction: {system_instruction}")
- config.system_instruction = system_instruction
-
- # Add tools to configuration, if provided
- if self._tools:
- logger.debug(f"Setting tools: {self._tools}")
- config.tools = self.get_llm_adapter().from_standard_tools(self._tools)
-
- # Start the connection
- self._connection_task = self.create_task(self._connection_task_handler(config=config))
-
- except Exception as e:
- await self.push_error(ErrorFrame(error=f"{self} Initialization error: {e}", fatal=True))
-
- async def _connection_task_handler(self, config: LiveConnectConfig):
- async with self._client.aio.live.connect(model=self._model_name, config=config) as session:
- logger.info("Connected to Gemini service")
-
- # Mark connection start time
- self._connection_start_time = time.time()
-
- await self._handle_session_ready(session)
-
- while True:
- try:
- turn = self._session.receive()
- async for message in turn:
- # Reset failure counter if connection has been stable
- self._check_and_reset_failure_counter()
-
- if message.server_content and message.server_content.model_turn:
- await self._handle_msg_model_turn(message)
- elif (
- message.server_content
- and message.server_content.turn_complete
- and message.usage_metadata
- ):
- await self._handle_msg_turn_complete(message)
- await self._handle_msg_usage_metadata(message)
- elif message.server_content and message.server_content.input_transcription:
- await self._handle_msg_input_transcription(message)
- elif message.server_content and message.server_content.output_transcription:
- await self._handle_msg_output_transcription(message)
- elif message.server_content and message.server_content.grounding_metadata:
- await self._handle_msg_grounding_metadata(message)
- elif message.tool_call:
- await self._handle_msg_tool_call(message)
- elif message.session_resumption_update:
- self._handle_msg_resumption_update(message)
- except Exception as e:
- if not self._disconnecting:
- should_reconnect = await self._handle_connection_error(e)
- if should_reconnect:
- await self._reconnect()
- return # Exit this connection handler, _reconnect will start a new one
- break
-
- def _check_and_reset_failure_counter(self):
- """Check if connection has been stable long enough to reset the failure counter.
-
- If the connection has been active for longer than the established threshold
- and there are accumulated failures, reset the counter to 0.
- """
- if (
- self._connection_start_time
- and self._consecutive_failures > 0
- and time.time() - self._connection_start_time >= CONNECTION_ESTABLISHED_THRESHOLD
- ):
- logger.info(
- f"Connection stable for {CONNECTION_ESTABLISHED_THRESHOLD}s, "
- f"resetting failure counter from {self._consecutive_failures} to 0"
- )
- self._consecutive_failures = 0
-
- async def _handle_connection_error(self, error: Exception) -> bool:
- """Handle a connection error and determine if reconnection should be attempted.
-
- Args:
- error: The exception that caused the connection error.
-
- Returns:
- True if reconnection should be attempted, False if a fatal error should be pushed.
- """
- self._consecutive_failures += 1
- logger.warning(
- f"Connection error (failure {self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES}): {error}"
- )
-
- if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES:
- logger.error(
- f"Max consecutive failures ({MAX_CONSECUTIVE_FAILURES}) reached, "
- "treating as fatal error"
- )
- await self.push_error(
- ErrorFrame(error=f"{self} Error in receive loop: {error}", fatal=True)
- )
- return False
- else:
- logger.info(
- f"Attempting reconnection ({self._consecutive_failures}/{MAX_CONSECUTIVE_FAILURES})"
- )
- return True
-
- async def _reconnect(self):
- """Reconnect to Gemini Live API."""
- await self._disconnect()
- await self._connect(session_resumption_handle=self._session_resumption_handle)
-
- async def _disconnect(self):
- """Disconnect from Gemini Live API and clean up resources."""
- logger.info("Disconnecting from Gemini service")
- try:
- self._disconnecting = True
- await self.stop_all_metrics()
- if self._connection_task:
- await self.cancel_task(self._connection_task, timeout=1.0)
- self._connection_task = None
- if self._session:
- await self._session.close()
- self._session = None
- self._disconnecting = False
- except Exception as e:
- logger.error(f"{self} error disconnecting: {e}")
-
- async def _send_user_audio(self, frame):
- """Send user audio frame to Gemini Live API."""
- if self._audio_input_paused or self._disconnecting or not self._session:
- return
-
- # Send all audio to Gemini
- try:
- await self._session.send_realtime_input(
- audio=Blob(data=frame.audio, mime_type=f"audio/pcm;rate={frame.sample_rate}")
- )
- except Exception as e:
- await self._handle_send_error(e)
-
- # Manage a buffer of audio to use for transcription
- audio = frame.audio
- if self._user_is_speaking:
- self._user_audio_buffer.extend(audio)
- else:
- # Keep 1/2 second of audio in the buffer even when not speaking.
- self._user_audio_buffer.extend(audio)
- length = int((frame.sample_rate * frame.num_channels * 2) * 0.5)
- self._user_audio_buffer = self._user_audio_buffer[-length:]
-
- async def _send_user_text(self, text: str):
- """Send user text via Gemini Live API's realtime input stream.
-
- This method sends text through the realtimeInput stream (via TextInputMessage)
- rather than the clientContent stream. This ensures text input is synchronized
- with audio and video inputs, preventing temporal misalignment that can occur
- when different modalities are processed through separate API pathways.
-
- For realtimeInput, turn completion is automatically inferred by the API based
- on user activity, so no explicit turnComplete signal is needed.
-
- Args:
- text: The text to send as user input.
- """
- if self._disconnecting or not self._session:
- return
-
- try:
- await self._session.send_realtime_input(text=text)
- except Exception as e:
- await self._handle_send_error(e)
-
- async def _send_user_video(self, frame):
- """Send user video frame to Gemini Live API."""
- if self._video_input_paused or self._disconnecting or not self._session:
- return
-
- now = time.time()
- if now - self._last_sent_time < 1:
- return # Ignore if less than 1 second has passed
-
- self._last_sent_time = now # Update last sent time
- logger.debug(f"Sending video frame to Gemini: {frame}")
-
- buffer = io.BytesIO()
- Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
- data = base64.b64encode(buffer.getvalue()).decode("utf-8")
-
- try:
- await self._session.send_realtime_input(video=Blob(data=data, mime_type="image/jpeg"))
- except Exception as e:
- await self._handle_send_error(e)
-
- async def _create_initial_response(self):
- """Create initial response based on context history."""
- if self._disconnecting:
- return
-
- if not self._session:
- self._run_llm_when_session_ready = True
- return
-
- messages = self._context.get_messages_for_initializing_history()
- if not messages:
- return
-
- logger.debug(f"Creating initial response: {messages}")
-
- await self.start_ttfb_metrics()
-
- try:
- await self._session.send_client_content(
- turns=messages, turn_complete=self._inference_on_context_initialization
- )
- except Exception as e:
- await self._handle_send_error(e)
-
- # If we're generating a response right away upon initializing
- # conversation history, set a flag saying that we need a turn complete
- # message when the user stops speaking.
- if not self._inference_on_context_initialization:
- self._needs_turn_complete_message = True
-
- async def _create_single_response(self, messages_list):
- """Create a single response from a list of messages."""
- if self._disconnecting or not self._session:
- return
-
- # Create a throwaway context just for the purpose of getting messages
- # in the right format
- context = GeminiMultimodalLiveContext.upgrade(OpenAILLMContext(messages=messages_list))
- messages = context.get_messages_for_initializing_history()
-
- if not messages:
- return
-
- logger.debug(f"Creating response: {messages}")
-
- await self.start_ttfb_metrics()
-
- try:
- await self._session.send_client_content(turns=messages, turn_complete=True)
- except Exception as e:
- await self._handle_send_error(e)
-
- @traced_gemini_live(operation="llm_tool_result")
- async def _tool_result(self, tool_result_message):
- """Send tool result back to the API."""
- if self._disconnecting or not self._session:
- return
-
- # For now we're shoving the name into the tool_call_id field, so this
- # will work until we revisit that.
- id = tool_result_message.get("tool_call_id")
- name = tool_result_message.get("tool_call_name")
- result = json.loads(tool_result_message.get("content") or "")
- response = FunctionResponse(name=name, id=id, response=result)
-
- try:
- await self._session.send_tool_response(function_responses=response)
- except Exception as e:
- await self._handle_send_error(e)
-
- @traced_gemini_live(operation="llm_setup")
- async def _handle_session_ready(self, session: AsyncSession):
- """Handle the session being ready."""
- self._session = session
- # If we were just waititng for the session to be ready to run the LLM,
- # do that now.
- if self._run_llm_when_session_ready:
- self._run_llm_when_session_ready = False
- await self._create_initial_response()
-
- async def _handle_msg_model_turn(self, msg: LiveServerMessage):
- """Handle the model turn message."""
- part = msg.server_content.model_turn.parts[0]
- if not part:
- return
-
- await self.stop_ttfb_metrics()
-
- # part.text is added when `modalities` is set to TEXT; otherwise, it's None
- text = part.text
- if text:
- if not self._bot_text_buffer:
- await self.push_frame(LLMFullResponseStartFrame())
-
- self._bot_text_buffer += text
- self._search_result_buffer += text # Also accumulate for grounding
- await self.push_frame(LLMTextFrame(text=text))
-
- # Check for grounding metadata in server content
- if msg.server_content and msg.server_content.grounding_metadata:
- self._accumulated_grounding_metadata = msg.server_content.grounding_metadata
-
- inline_data = part.inline_data
- if not inline_data:
- return
-
- # Check if mime type matches expected format
- expected_mime_type = f"audio/pcm;rate={self._sample_rate}"
- if inline_data.mime_type == expected_mime_type:
- # Perfect match, continue processing
- pass
- elif inline_data.mime_type == "audio/pcm":
- # Sample rate not provided in mime type, assume default
- if not hasattr(self, "_sample_rate_warning_logged"):
- logger.warning(
- f"Sample rate not provided in mime type '{inline_data.mime_type}', assuming rate of {self._sample_rate}"
- )
- self._sample_rate_warning_logged = True
- else:
- # Unrecognized format
- logger.warning(f"Unrecognized server_content format {inline_data.mime_type}")
- return
-
- audio = inline_data.data
- if not audio:
- return
-
- if not self._bot_is_speaking:
- await self._set_bot_is_speaking(True)
- await self.push_frame(TTSStartedFrame())
- await self.push_frame(LLMFullResponseStartFrame())
-
- self._bot_audio_buffer.extend(audio)
- frame = TTSAudioRawFrame(
- audio=audio,
- sample_rate=self._sample_rate,
- num_channels=1,
- )
- await self.push_frame(frame)
-
- @traced_gemini_live(operation="llm_tool_call")
- async def _handle_msg_tool_call(self, message: LiveServerMessage):
- """Handle tool call messages."""
- function_calls = message.tool_call.function_calls
- if not function_calls:
- return
- if not self._context:
- logger.error("Function calls are not supported without a context object.")
-
- function_calls_llm = [
- FunctionCallFromLLM(
- context=self._context,
- tool_call_id=(
- # NOTE: when using Vertex AI we don't get server-provided
- # tool call IDs here
- f.id or str(uuid.uuid4())
- ),
- function_name=f.name,
- arguments=f.args,
- )
- for f in function_calls
- ]
-
- await self.run_function_calls(function_calls_llm)
-
- @traced_gemini_live(operation="llm_response")
- async def _handle_msg_turn_complete(self, message: LiveServerMessage):
- """Handle the turn complete message."""
- await self._set_bot_is_speaking(False)
- text = self._bot_text_buffer
-
- # Trace the complete LLM response (this will be handled by the decorator)
- # The decorator will extract the output text and usage metadata from the message
-
- self._bot_text_buffer = ""
- self._llm_output_buffer = ""
-
- # Process grounding metadata if we have accumulated any
- if self._accumulated_grounding_metadata:
- await self._process_grounding_metadata(
- self._accumulated_grounding_metadata, self._search_result_buffer
- )
-
- # Reset grounding tracking for next response
- self._search_result_buffer = ""
- self._accumulated_grounding_metadata = None
-
- # Only push the TTSStoppedFrame if the bot is outputting audio
- # when text is found, modalities is set to TEXT and no audio
- # is produced.
- if not text:
- await self.push_frame(TTSStoppedFrame())
-
- await self.push_frame(LLMFullResponseEndFrame())
-
- @traced_stt
- async def _handle_user_transcription(
- self, transcript: str, is_final: bool, language: Optional[Language] = None
- ):
- """Handle a transcription result with tracing."""
- pass
-
- async def _handle_msg_input_transcription(self, message: LiveServerMessage):
- """Handle the input transcription message.
-
- Gemini Live sends user transcriptions in either single words or multi-word
- phrases. As a result, we have to aggregate the input transcription. This handler
- aggregates into sentences, splitting on the end of sentence markers.
- """
- if not message.server_content.input_transcription:
- return
-
- text = message.server_content.input_transcription.text
-
- if not text:
- return
-
- # Strip leading space from sentence starts if buffer is empty
- if text.startswith(" ") and not self._user_transcription_buffer:
- text = text.lstrip()
-
- # Accumulate text in the buffer
- self._user_transcription_buffer += text
-
- # Check for complete sentences
- while True:
- eos_end_marker = match_endofsentence(self._user_transcription_buffer)
- if not eos_end_marker:
- break
-
- # Extract the complete sentence
- complete_sentence = self._user_transcription_buffer[:eos_end_marker]
- # Keep the remainder for the next chunk
- self._user_transcription_buffer = self._user_transcription_buffer[eos_end_marker:]
-
- # Send a TranscriptionFrame with the complete sentence
- logger.debug(f"[Transcription:user] [{complete_sentence}]")
- await self._handle_user_transcription(
- complete_sentence, True, self._settings["language"]
- )
- await self.push_frame(
- TranscriptionFrame(
- text=complete_sentence,
- user_id="",
- timestamp=time_now_iso8601(),
- result=message,
- ),
- FrameDirection.UPSTREAM,
- )
-
- async def _handle_msg_output_transcription(self, message: LiveServerMessage):
- """Handle the output transcription message."""
- if not message.server_content.output_transcription:
- return
-
- # This is the output transcription text when modalities is set to AUDIO.
- # In this case, we push LLMTextFrame and TTSTextFrame to be handled by the
- # downstream assistant context aggregator.
- text = message.server_content.output_transcription.text
-
- if not text:
- return
-
- # Accumulate text for grounding as well
- self._search_result_buffer += text
-
- # Check for grounding metadata in server content
- if message.server_content and message.server_content.grounding_metadata:
- self._accumulated_grounding_metadata = message.server_content.grounding_metadata
- # Collect text for tracing
- self._llm_output_buffer += text
-
- await self.push_frame(LLMTextFrame(text=text))
- await self.push_frame(TTSTextFrame(text=text))
-
- async def _handle_msg_grounding_metadata(self, message: LiveServerMessage):
- """Handle dedicated grounding metadata messages."""
- if message.server_content and message.server_content.grounding_metadata:
- grounding_metadata = message.server_content.grounding_metadata
- # Process the grounding metadata immediately
- await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer)
-
- async def _process_grounding_metadata(
- self, grounding_metadata: GroundingMetadata, search_result: str = ""
- ):
- """Process grounding metadata and emit LLMSearchResponseFrame."""
- if not grounding_metadata:
- return
-
- # Extract rendered content for search suggestions
- rendered_content = None
- if (
- grounding_metadata.search_entry_point
- and grounding_metadata.search_entry_point.rendered_content
- ):
- rendered_content = grounding_metadata.search_entry_point.rendered_content
-
- # Convert grounding chunks and supports to LLMSearchOrigin format
- origins = []
-
- if grounding_metadata.grounding_chunks and grounding_metadata.grounding_supports:
- # Create a mapping of chunk indices to origins
- chunk_to_origin: Dict[int, LLMSearchOrigin] = {}
-
- for index, chunk in enumerate(grounding_metadata.grounding_chunks):
- if chunk.web:
- origin = LLMSearchOrigin(
- site_uri=chunk.web.uri, site_title=chunk.web.title, results=[]
- )
- chunk_to_origin[index] = origin
- origins.append(origin)
-
- # Add grounding support results to the appropriate origins
- for support in grounding_metadata.grounding_supports:
- if support.segment and support.grounding_chunk_indices:
- text = support.segment.text or ""
- confidence_scores = support.confidence_scores or []
-
- # Add this result to all origins referenced by this support
- for chunk_index in support.grounding_chunk_indices:
- if chunk_index in chunk_to_origin:
- result = LLMSearchResult(text=text, confidence=confidence_scores)
- chunk_to_origin[chunk_index].results.append(result)
-
- # Create and push the search response frame
- search_frame = LLMSearchResponseFrame(
- search_result=search_result, origins=origins, rendered_content=rendered_content
- )
-
- await self.push_frame(search_frame)
-
- async def _handle_msg_usage_metadata(self, message: LiveServerMessage):
- """Handle the usage metadata message."""
- if not message.usage_metadata:
- return
-
- usage = message.usage_metadata
-
- # Ensure we have valid integers for all token counts
- prompt_tokens = usage.prompt_token_count or 0
- completion_tokens = usage.response_token_count or 0
- total_tokens = usage.total_token_count or (prompt_tokens + completion_tokens)
-
- tokens = LLMTokenUsage(
- prompt_tokens=prompt_tokens,
- completion_tokens=completion_tokens,
- total_tokens=total_tokens,
- )
-
- await self.start_llm_usage_metrics(tokens)
-
- def _handle_msg_resumption_update(self, message: LiveServerMessage):
- update = message.session_resumption_update
- if update.resumable and update.new_handle:
- self._session_resumption_handle = update.new_handle
-
- async def _handle_send_error(self, error: Exception):
- # In server-to-server contexts, a WebSocket error should be quite rare.
- # Given how hard it is to recover from a send-side error with proper
- # state management, and that exponential backoff for retries can have
- # cost/stability implications for a service cluster, let's just treat a
- # send-side error as fatal.
- if not self._disconnecting:
- await self.push_error(ErrorFrame(error=f"{self} Send error: {error}", fatal=True))
-
- def create_context_aggregator(
- self,
- context: OpenAILLMContext,
- *,
- user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
- assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
- ) -> GeminiMultimodalLiveContextAggregatorPair:
- """Create an instance of GeminiMultimodalLiveContextAggregatorPair from an OpenAILLMContext.
-
- Constructor keyword arguments for both the user and assistant aggregators can be provided.
-
- Args:
- context: The LLM context to use.
- user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams().
- assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams().
-
- Returns:
- GeminiMultimodalLiveContextAggregatorPair: A pair of context
- aggregators, one for the user and one for the assistant,
- encapsulated in an GeminiMultimodalLiveContextAggregatorPair.
- """
- context.set_llm_adapter(self.get_llm_adapter())
-
- GeminiMultimodalLiveContext.upgrade(context)
- user = GeminiMultimodalLiveUserContextAggregator(context, params=user_params)
-
- assistant_params.expect_stripped_words = False
- assistant = GeminiMultimodalLiveAssistantContextAggregator(context, params=assistant_params)
- return GeminiMultimodalLiveContextAggregatorPair(_user=user, _assistant=assistant)
+GeminiMultimodalLiveContext = GeminiLiveContext
+GeminiMultimodalLiveUserContextAggregator = GeminiLiveUserContextAggregator
+GeminiMultimodalLiveAssistantContextAggregator = GeminiLiveAssistantContextAggregator
+GeminiMultimodalLiveContextAggregatorPair = GeminiLiveContextAggregatorPair
+GeminiMultimodalModalities = GeminiModalities
+GeminiMediaResolution = _GeminiMediaResolution
+GeminiVADParams = _GeminiVADParams
+ContextWindowCompressionParams = _ContextWindowCompressionParams
+InputParams = _InputParams
+GeminiMultimodalLiveLLMService = GeminiLiveLLMService
From 10069719e4f3018463e6e93049803daf2ad2f099 Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Thu, 9 Oct 2025 15:48:59 -0400
Subject: [PATCH 08/51] Make `pause_processing_frames()` and
`pause_processing_system_frames()` more robust in `FrameProcessor`.
To understand this fix, let's look exclusively at `pause_processing_frames()` (`pause_processing_system_frames()` works the same way).
`pause_processing_frames()` works by setting a `__should_block_frames` flag, which is then read each time through the loop in the long-running `__process_frame_task_handler`. if `__should_block_frames` is `True`, it pauses processing frames until it's resumed.
Prior to this fix, the check for `__should_block_frames` was before `await self.__process_queue.get()`. The problem is that a lot of the time spent in the loop is waiting for a frame from the process queue. So if `pause_processing_frames()` is set at any time other than within `process_frame()` itself, it actually won't have an effect by the next frame, only on the frame *after* the next, which is later than intended.
Because thus far in the Pipecat codebase we've only ever called `pause_processing_frames()` and `pause_processing_system_frames()` from within `process_frame()`, this change should have no behavioral effect. But it will be helpful if we ever need to call it from anywhere else. I noticed this issue while developing a feature that did exactly that (though I later abandoned that code).
---
src/pipecat/processors/frame_processor.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py
index 0b55082aa..1ca3333b5 100644
--- a/src/pipecat/processors/frame_processor.py
+++ b/src/pipecat/processors/frame_processor.py
@@ -877,6 +877,8 @@ class FrameProcessor(BaseObject):
"""
while True:
+ (frame, direction, callback) = await self.__input_queue.get()
+
if self.__should_block_system_frames and self.__input_event:
logger.trace(f"{self}: system frame processing paused")
await self.__input_event.wait()
@@ -884,8 +886,6 @@ class FrameProcessor(BaseObject):
self.__should_block_system_frames = False
logger.trace(f"{self}: system frame processing resumed")
- (frame, direction, callback) = await self.__input_queue.get()
-
if isinstance(frame, SystemFrame):
await self.__process_frame(frame, direction, callback)
elif self.__process_queue:
@@ -900,6 +900,8 @@ class FrameProcessor(BaseObject):
async def __process_frame_task_handler(self):
"""Handle non-system frames from the process queue."""
while True:
+ (frame, direction, callback) = await self.__process_queue.get()
+
if self.__should_block_frames and self.__process_event:
logger.trace(f"{self}: frame processing paused")
await self.__process_event.wait()
@@ -907,8 +909,6 @@ class FrameProcessor(BaseObject):
self.__should_block_frames = False
logger.trace(f"{self}: frame processing resumed")
- (frame, direction, callback) = await self.__process_queue.get()
-
await self.__process_frame(frame, direction, callback)
self.__process_queue.task_done()
From 2994640f475c8a65d671d970d48460715f345669 Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Thu, 9 Oct 2025 16:45:47 -0400
Subject: [PATCH 09/51] Move `location` and `project_id` out of `InputParams`
in `GoogleVertexLLMService`, making them top-level init args instead. We do
this for two reasons:
- Conceptually, these args comprise project-level setup, akin to credentials, whereas everything in `InputParams` is concerned with model configuration
- Providing a `project_id` when initializing `GoogleVertexLLMService` should not be optional, but prior to the change in this commit it was (erroneously) treated as optional by dint of `InputParams` being optional
This improvement was discussed [in this comment](https://github.com/pipecat-ai/pipecat/pull/2795#discussion_r2408279142).
---
src/pipecat/services/google/llm_vertex.py | 67 ++++++++++++++++++++---
1 file changed, 58 insertions(+), 9 deletions(-)
diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py
index ee66c1f20..3a3bf9ea9 100644
--- a/src/pipecat/services/google/llm_vertex.py
+++ b/src/pipecat/services/google/llm_vertex.py
@@ -51,6 +51,10 @@ class GoogleVertexLLMService(OpenAILLMService):
class InputParams(OpenAILLMService.InputParams):
"""Input parameters specific to Vertex AI.
+ .. deprecated:: 0.0.90
+ Use `OpenAILLMService.InputParams` instead and provide `location` and
+ `project_id` as direct arguments to `GoogleVertexLLMService.__init__()`.
+
Parameters:
location: GCP region for Vertex AI endpoint (e.g., "us-east4").
project_id: Google Cloud project ID.
@@ -60,12 +64,29 @@ class GoogleVertexLLMService(OpenAILLMService):
location: str = "us-east4"
project_id: str
+ def __init__(self, **kwargs):
+ """Initializes the InputParams."""
+ import warnings
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+ warnings.warn(
+ "GoogleVertexLLMService.InputParams is deprecated. "
+ "Please use OpenAILLMService.InputParams instead and provide "
+ "'location' and 'project_id' as direct arguments to __init__.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ super().__init__(**kwargs)
+
def __init__(
self,
*,
credentials: Optional[str] = None,
credentials_path: Optional[str] = None,
model: str = "google/gemini-2.0-flash-001",
+ location: Optional[str] = None,
+ project_id: Optional[str] = None,
params: Optional[InputParams] = None,
**kwargs,
):
@@ -75,11 +96,42 @@ class GoogleVertexLLMService(OpenAILLMService):
credentials: JSON string of service account credentials.
credentials_path: Path to the service account JSON file.
model: Model identifier (e.g., "google/gemini-2.0-flash-001").
+ location: GCP region for Vertex AI endpoint (e.g., "us-east4").
+ project_id: Google Cloud project ID.
params: Vertex AI input parameters including location and project.
+
+ .. deprecated:: 0.0.90
+ Use `OpenAILLMService.InputParams` instead and provide `location`
+ and `project_id` as direct arguments to `__init__()`.
+
**kwargs: Additional arguments passed to OpenAILLMService.
"""
- params = params or OpenAILLMService.InputParams()
- base_url = self._get_base_url(params)
+ # Handle deprecated InputParams
+ if params is not None and isinstance(params, GoogleVertexLLMService.InputParams):
+ # Extract location and project_id from params if not provided directly
+ if project_id is None:
+ project_id = params.project_id
+ if location is None:
+ location = params.location
+ # Convert to base InputParams
+ params = OpenAILLMService.InputParams(
+ **params.model_dump(exclude={"location", "project_id"}, exclude_unset=True)
+ )
+ else:
+ params = params or OpenAILLMService.InputParams()
+
+ # Validate parameters
+ # NOTE: once we remove deprecated InputParams, we can update __init__()
+ # signature with the following:
+ # - location: str = "us-east4",
+ # - project_id: str,
+ # For now, we need them as-is to maintain backward compatibility.
+ if project_id is None:
+ raise ValueError("project_id is required")
+ if location is None:
+ location = "us-east4" # Default location if not provided
+
+ base_url = self._get_base_url(location, project_id)
self._api_key = self._get_api_token(credentials, credentials_path)
super().__init__(
@@ -91,17 +143,14 @@ class GoogleVertexLLMService(OpenAILLMService):
)
@staticmethod
- def _get_base_url(params: InputParams) -> str:
+ def _get_base_url(location: str, project_id: str) -> str:
"""Construct the base URL for Vertex AI API."""
# Determine the correct API host based on location
- if params.location == "global":
+ if location == "global":
api_host = "aiplatform.googleapis.com"
else:
- api_host = f"{params.location}-aiplatform.googleapis.com"
- return (
- f"https://{api_host}/v1/"
- f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi"
- )
+ api_host = f"{location}-aiplatform.googleapis.com"
+ return f"https://{api_host}/v1/projects/{project_id}/locations/{location}/endpoints/openapi"
@staticmethod
def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str:
From aeace9b9be1005b85fa9147412f8045dc3447c80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 16:02:52 -0700
Subject: [PATCH 10/51] google: move gemini_live inside google service
---
CHANGELOG.md | 23 +++++++++++--------
.../foundational/26-gemini-multimodal-live.py | 2 +-
...6a-gemini-multimodal-live-transcription.py | 2 +-
...gemini-multimodal-live-function-calling.py | 2 +-
.../26c-gemini-multimodal-live-video.py | 2 +-
.../26d-gemini-multimodal-live-text.py | 2 +-
.../26e-gemini-multimodal-google-search.py | 2 +-
.../26f-gemini-multimodal-live-files-api.py | 2 +-
...emini-multimodal-live-groundingMetadata.py | 2 +-
.../26h-gemini-multimodal-live-vertex.py | 4 ++--
...26i-gemini-multimodal-live-graceful-end.py | 4 +---
examples/foundational/46-video-processing.py | 2 +-
.../gemini_multimodal_live/file_api.py | 2 +-
.../services/gemini_multimodal_live/gemini.py | 12 ++++++----
.../{ => google}/gemini_live/__init__.py | 0
.../{ => google}/gemini_live/file_api.py | 0
.../{ => google}/gemini_live/gemini.py | 0
.../{ => google}/gemini_live/vertex.py | 2 +-
18 files changed, 35 insertions(+), 30 deletions(-)
rename src/pipecat/services/{ => google}/gemini_live/__init__.py (100%)
rename src/pipecat/services/{ => google}/gemini_live/file_api.py (100%)
rename src/pipecat/services/{ => google}/gemini_live/gemini.py (100%)
rename src/pipecat/services/{ => google}/gemini_live/vertex.py (99%)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6999bd37c..10c749dbc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,10 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
-- Added `GeminiVertexMultimodalLiveLLMService`, for accessing Gemini Live via
- Google Vertex AI.
+- Added `GeminiLiveVertexLLMService`, for accessing Gemini Live via Google
+ Vertex AI.
-- Added some new configuration options to `GeminiMultimodalLiveLLMService`:
+- Added some new configuration options to `GeminiLiveLLMService`:
- `thinking`
- `enable_affective_dialog`
@@ -37,16 +37,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
-- Updated `GeminiMultimodalLiveLLMService` to use the `google-genai` library
- rather than use WebSockets directly.
+- Updated `GeminiLiveLLMService` to use the `google-genai` library rather than
+ use WebSockets directly.
+
+### Deprecated
+
+- `GeminiMultimodalLiveLLMService` is now deprecated, use
+ `GeminiLiveLLMService`.
### Fixed
-- `GeminiMultimodalLiveLLMService` will now end gracefully (i.e. after the bot
- has finished) upon receiving an `EndFrame`.
+- `GeminiLiveLLMService` will now end gracefully (i.e. after the bot has
+ finished) upon receiving an `EndFrame`.
-- `GeminiMultimodalLiveLLMService` will try to seamlessly reconnect when it
- loses its connection.
+- `GeminiLiveLLMService` will try to seamlessly reconnect when it loses its
+ connection.
## [0.0.89] - 2025-10-07
diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py
index f063c247f..fc7ad9d35 100644
--- a/examples/foundational/26-gemini-multimodal-live.py
+++ b/examples/foundational/26-gemini-multimodal-live.py
@@ -17,7 +17,7 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py
index 3c0eb19ce..cf8660edf 100644
--- a/examples/foundational/26a-gemini-multimodal-live-transcription.py
+++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py
@@ -20,7 +20,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.transcript_processor import TranscriptProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
index a86822a20..a08f5a352 100644
--- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py
+++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
@@ -22,7 +22,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py
index 7eac0bc0b..1ac4eb923 100644
--- a/examples/foundational/26c-gemini-multimodal-live-video.py
+++ b/examples/foundational/26c-gemini-multimodal-live-video.py
@@ -24,7 +24,7 @@ from pipecat.runner.utils import (
maybe_capture_participant_camera,
maybe_capture_participant_screen,
)
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py
index 625ca17d0..68fd79c54 100644
--- a/examples/foundational/26d-gemini-multimodal-live-text.py
+++ b/examples/foundational/26d-gemini-multimodal-live-text.py
@@ -20,7 +20,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.cartesia.tts import CartesiaTTSService
-from pipecat.services.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.gemini import (
GeminiLiveLLMService,
GeminiModalities,
InputParams,
diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-multimodal-google-search.py
index 1bcf02db9..87c754b75 100644
--- a/examples/foundational/26e-gemini-multimodal-google-search.py
+++ b/examples/foundational/26e-gemini-multimodal-google-search.py
@@ -19,7 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26f-gemini-multimodal-live-files-api.py b/examples/foundational/26f-gemini-multimodal-live-files-api.py
index 7ce4a483d..51dad712d 100644
--- a/examples/foundational/26f-gemini-multimodal-live-files-api.py
+++ b/examples/foundational/26f-gemini-multimodal-live-files-api.py
@@ -19,7 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.gemini import (
GeminiLiveLLMService,
)
from pipecat.transports.base_transport import BaseTransport, TransportParams
diff --git a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py b/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
index 9b8b25315..2b424eb18 100644
--- a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
+++ b/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
@@ -14,8 +14,8 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
from pipecat.services.google.frames import LLMSearchResponseFrame
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26h-gemini-multimodal-live-vertex.py b/examples/foundational/26h-gemini-multimodal-live-vertex.py
index 4210dc0e2..f8dfc8c52 100644
--- a/examples/foundational/26h-gemini-multimodal-live-vertex.py
+++ b/examples/foundational/26h-gemini-multimodal-live-vertex.py
@@ -17,8 +17,8 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
-from pipecat.services.gemini_live.vertex import GeminiLiveVertexLLMService
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.vertex import GeminiLiveVertexLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py b/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
index 9451d0ef0..3527e89e3 100644
--- a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
+++ b/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
@@ -4,8 +4,6 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
-
-import asyncio
import os
from datetime import datetime
@@ -24,7 +22,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py
index 7f7363538..0152f3211 100644
--- a/examples/foundational/46-video-processing.py
+++ b/examples/foundational/46-video-processing.py
@@ -20,7 +20,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.gemini_live import GeminiLiveLLMService
+from pipecat.services.google.gemini_live import GeminiLiveLLMService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.daily.transport import DailyParams, DailyTransport
diff --git a/src/pipecat/services/gemini_multimodal_live/file_api.py b/src/pipecat/services/gemini_multimodal_live/file_api.py
index 53f8442fc..ed4d6c522 100644
--- a/src/pipecat/services/gemini_multimodal_live/file_api.py
+++ b/src/pipecat/services/gemini_multimodal_live/file_api.py
@@ -20,7 +20,7 @@ import warnings
from loguru import logger
try:
- from pipecat.services.gemini_live.file_api import GeminiFileAPI as _GeminiFileAPI
+ from pipecat.services.google.gemini_live.file_api import GeminiFileAPI as _GeminiFileAPI
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index 4ac51e02e..a3547551a 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -18,10 +18,10 @@ voice transcription, streaming responses, and tool usage.
import warnings
-from pipecat.services.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.gemini import (
ContextWindowCompressionParams as _ContextWindowCompressionParams,
)
-from pipecat.services.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.gemini import (
GeminiLiveAssistantContextAggregator,
GeminiLiveContext,
GeminiLiveContextAggregatorPair,
@@ -29,9 +29,11 @@ from pipecat.services.gemini_live.gemini import (
GeminiLiveUserContextAggregator,
GeminiModalities,
)
-from pipecat.services.gemini_live.gemini import GeminiMediaResolution as _GeminiMediaResolution
-from pipecat.services.gemini_live.gemini import GeminiVADParams as _GeminiVADParams
-from pipecat.services.gemini_live.gemini import InputParams as _InputParams
+from pipecat.services.google.gemini_live.gemini import (
+ GeminiMediaResolution as _GeminiMediaResolution,
+)
+from pipecat.services.google.gemini_live.gemini import GeminiVADParams as _GeminiVADParams
+from pipecat.services.google.gemini_live.gemini import InputParams as _InputParams
with warnings.catch_warnings():
warnings.simplefilter("always")
diff --git a/src/pipecat/services/gemini_live/__init__.py b/src/pipecat/services/google/gemini_live/__init__.py
similarity index 100%
rename from src/pipecat/services/gemini_live/__init__.py
rename to src/pipecat/services/google/gemini_live/__init__.py
diff --git a/src/pipecat/services/gemini_live/file_api.py b/src/pipecat/services/google/gemini_live/file_api.py
similarity index 100%
rename from src/pipecat/services/gemini_live/file_api.py
rename to src/pipecat/services/google/gemini_live/file_api.py
diff --git a/src/pipecat/services/gemini_live/gemini.py b/src/pipecat/services/google/gemini_live/gemini.py
similarity index 100%
rename from src/pipecat/services/gemini_live/gemini.py
rename to src/pipecat/services/google/gemini_live/gemini.py
diff --git a/src/pipecat/services/gemini_live/vertex.py b/src/pipecat/services/google/gemini_live/vertex.py
similarity index 99%
rename from src/pipecat/services/gemini_live/vertex.py
rename to src/pipecat/services/google/gemini_live/vertex.py
index c03f589b8..29ff63a0b 100644
--- a/src/pipecat/services/gemini_live/vertex.py
+++ b/src/pipecat/services/google/gemini_live/vertex.py
@@ -18,7 +18,7 @@ from typing import List, Optional, Union
from loguru import logger
from pipecat.adapters.schemas.tools_schema import ToolsSchema
-from pipecat.services.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.gemini import (
GeminiLiveLLMService,
HttpOptions,
InputParams,
From 59217eae38ccf4a12039a3c85c78b2beda552f75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Wed, 8 Oct 2025 19:10:47 -0700
Subject: [PATCH 11/51] runner: add --folder argument to allow file downloads
---
CHANGELOG.md | 3 ++
src/pipecat/runner/run.py | 83 ++++++++++++++++++++++++++++++++++-----
2 files changed, 76 insertions(+), 10 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 10c749dbc..829991a85 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- Added `--folder` argument to the runner, allowing files saved in that folder
+ to be downloaded from `http://HOST:PORT/file/FILE`.
+
- Added `GeminiLiveVertexLLMService`, for accessing Gemini Live via Google
Vertex AI.
diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py
index 1ab923314..fd1a8a952 100644
--- a/src/pipecat/runner/run.py
+++ b/src/pipecat/runner/run.py
@@ -67,12 +67,15 @@ To run locally:
import argparse
import asyncio
+import mimetypes
import os
import sys
from contextlib import asynccontextmanager
+from pathlib import Path
from typing import Optional
import aiohttp
+from fastapi.responses import FileResponse
from loguru import logger
from pipecat.runner.types import (
@@ -98,6 +101,12 @@ except ImportError as e:
load_dotenv(override=True)
os.environ["ENV"] = "local"
+TELEPHONY_TRANSPORTS = ["twilio", "telnyx", "plivo", "exotel"]
+
+RUNNER_DOWNLOADS_FOLDER: Optional[str] = None
+RUNNER_HOST: str = "localhost"
+RUNNER_PORT: int = 7860
+
def _get_bot_module():
"""Get the bot module from the calling script."""
@@ -152,7 +161,12 @@ async def _run_telephony_bot(websocket: WebSocket):
def _create_server_app(
- transport_type: str, host: str = "localhost", proxy: str = None, esp32_mode: bool = False
+ *,
+ transport_type: str,
+ host: str = "localhost",
+ proxy: str,
+ esp32_mode: bool = False,
+ folder: Optional[str] = None,
):
"""Create FastAPI app with transport-specific routes."""
app = FastAPI()
@@ -167,19 +181,21 @@ def _create_server_app(
# Set up transport-specific routes
if transport_type == "webrtc":
- _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host)
+ _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host, folder=folder)
_setup_whatsapp_routes(app)
elif transport_type == "daily":
_setup_daily_routes(app)
- elif transport_type in ["twilio", "telnyx", "plivo", "exotel"]:
- _setup_telephony_routes(app, transport_type, proxy)
+ elif transport_type in TELEPHONY_TRANSPORTS:
+ _setup_telephony_routes(app, transport_type=transport_type, proxy=proxy)
else:
logger.warning(f"Unknown transport type: {transport_type}")
return app
-def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "localhost"):
+def _setup_webrtc_routes(
+ app: FastAPI, *, esp32_mode: bool = False, host: str = "localhost", folder: Optional[str] = None
+):
"""Set up WebRTC-specific routes."""
try:
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
@@ -201,6 +217,21 @@ def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "lo
"""Redirect root requests to client interface."""
return RedirectResponse(url="/client/")
+ @app.get("/files/{filename}")
+ async def download_file(filename: str):
+ """Handle file downloads."""
+ if not folder:
+ logger.warning(f"Attempting to dowload {filename}, but downloads folder not setup.")
+ return
+
+ file_path = Path(folder) / filename
+ if not os.path.exists(file_path):
+ raise HTTPException(404)
+
+ media_type, _ = mimetypes.guess_type(file_path)
+
+ return FileResponse(path=file_path, media_type=media_type, filename=filename)
+
# Initialize the SmallWebRTC request handler
small_webrtc_handler: SmallWebRTCRequestHandler = SmallWebRTCRequestHandler(
esp32_mode=esp32_mode, host=host
@@ -489,7 +520,7 @@ def _setup_daily_routes(app: FastAPI):
return await _handle_rtvi_request(request)
-def _setup_telephony_routes(app: FastAPI, transport_type: str, proxy: str):
+def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str):
"""Set up telephony-specific routes."""
# XML response templates (Exotel doesn't use XML webhooks)
XML_TEMPLATES = {
@@ -592,6 +623,21 @@ def _validate_and_clean_proxy(proxy: str) -> str:
return proxy
+def runner_downloads_folder() -> Optional[str]:
+ """Returns the folder where files are stored for later download."""
+ return RUNNER_DOWNLOADS_FOLDER
+
+
+def runner_host() -> str:
+ """Returns the host name of this runner."""
+ return RUNNER_HOST
+
+
+def runner_port() -> int:
+ """Returns the port of this runner."""
+ return RUNNER_PORT
+
+
def main():
"""Start the Pipecat development runner.
@@ -612,14 +658,16 @@ def main():
The bot file must contain a `bot(runner_args)` function as the entry point.
"""
+ global RUNNER_DOWNLOADS_FOLDER, RUNNER_HOST, RUNNER_PORT
+
parser = argparse.ArgumentParser(description="Pipecat Development Runner")
- parser.add_argument("--host", type=str, default="localhost", help="Host address")
- parser.add_argument("--port", type=int, default=7860, help="Port number")
+ parser.add_argument("--host", type=str, default=RUNNER_HOST, help="Host address")
+ parser.add_argument("--port", type=int, default=RUNNER_PORT, help="Port number")
parser.add_argument(
"-t",
"--transport",
type=str,
- choices=["daily", "webrtc", "twilio", "telnyx", "plivo", "exotel"],
+ choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS],
default="webrtc",
help="Transport type",
)
@@ -637,6 +685,7 @@ def main():
default=False,
help="Connect directly to Daily room (automatically sets transport to daily)",
)
+ parser.add_argument("-f", "--folder", type=str, help="Path to downloads folder")
parser.add_argument(
"--verbose", "-v", action="count", default=0, help="Increase logging verbosity"
)
@@ -659,6 +708,10 @@ def main():
logger.error("For ESP32, you need to specify `--host IP` so we can do SDP munging.")
return
+ if args.transport in TELEPHONY_TRANSPORTS and not args.proxy:
+ logger.error(f"For telephony transports, you need to specify `--proxy PROXY`.")
+ return
+
# Log level
logger.remove()
logger.add(sys.stderr, level="TRACE" if args.verbose else "DEBUG")
@@ -689,8 +742,18 @@ def main():
print(f" → Open http://{args.host}:{args.port} in your browser to start a session")
print()
+ RUNNER_DOWNLOADS_FOLDER = args.folder
+ RUNNER_HOST = args.host
+ RUNNER_PORT = args.port
+
# Create the app with transport-specific setup
- app = _create_server_app(args.transport, args.host, args.proxy, args.esp32)
+ app = _create_server_app(
+ transport_type=args.transport,
+ host=args.host,
+ proxy=args.proxy,
+ esp32_mode=args.esp32,
+ folder=args.folder,
+ )
# Run the server
uvicorn.run(app, host=args.host, port=args.port)
From cdc86db8cebb064c1fa01d29e0ad44672303d226 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 16:58:22 -0700
Subject: [PATCH 12/51] update CHANGELOG with GoogleVertexLLMService token fix
---
CHANGELOG.md | 3 +++
1 file changed, 3 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 829991a85..28da49e20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,6 +50,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Fixed a `GoogleVertexLLMService` issue that would generate an error if no
+ token information was returned.
+
- `GeminiLiveLLMService` will now end gracefully (i.e. after the bot has
finished) upon receiving an `EndFrame`.
From 15bf5b15339b8af6fe2a955b471c5b673da9cf33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 17:28:16 -0700
Subject: [PATCH 13/51] aws: move aws_nova_sonic to aws.nova_sonic
---
.../20e-persistent-context-aws-nova-sonic.py | 2 +-
examples/foundational/40-aws-nova-sonic.py | 2 +-
src/pipecat/services/aws/__init__.py | 1 +
src/pipecat/services/aws/nova_sonic/__init__.py | 0
.../{aws_nova_sonic => aws/nova_sonic}/context.py | 2 +-
.../{aws_nova_sonic => aws/nova_sonic}/frames.py | 0
.../aws.py => aws/nova_sonic/llm.py} | 6 +++---
.../{aws_nova_sonic => aws/nova_sonic}/ready.wav | Bin
src/pipecat/services/aws_nova_sonic/__init__.py | 1 -
9 files changed, 7 insertions(+), 7 deletions(-)
create mode 100644 src/pipecat/services/aws/nova_sonic/__init__.py
rename src/pipecat/services/{aws_nova_sonic => aws/nova_sonic}/context.py (99%)
rename src/pipecat/services/{aws_nova_sonic => aws/nova_sonic}/frames.py (100%)
rename src/pipecat/services/{aws_nova_sonic/aws.py => aws/nova_sonic/llm.py} (99%)
rename src/pipecat/services/{aws_nova_sonic => aws/nova_sonic}/ready.wav (100%)
delete mode 100644 src/pipecat/services/aws_nova_sonic/__init__.py
diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py
index bd3d9d545..2161aff8f 100644
--- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py
+++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py
@@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.aws_nova_sonic.aws import AWSNovaSonicLLMService
+from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/40-aws-nova-sonic.py b/examples/foundational/40-aws-nova-sonic.py
index de7bbf638..d9bd2257d 100644
--- a/examples/foundational/40-aws-nova-sonic.py
+++ b/examples/foundational/40-aws-nova-sonic.py
@@ -21,7 +21,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.aws_nova_sonic import AWSNovaSonicLLMService
+from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/src/pipecat/services/aws/__init__.py b/src/pipecat/services/aws/__init__.py
index b1f157bd3..3cdd4cc5a 100644
--- a/src/pipecat/services/aws/__init__.py
+++ b/src/pipecat/services/aws/__init__.py
@@ -9,6 +9,7 @@ import sys
from pipecat.services import DeprecatedModuleProxy
from .llm import *
+from .nova_sonic import *
from .stt import *
from .tts import *
diff --git a/src/pipecat/services/aws/nova_sonic/__init__.py b/src/pipecat/services/aws/nova_sonic/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws/nova_sonic/context.py
similarity index 99%
rename from src/pipecat/services/aws_nova_sonic/context.py
rename to src/pipecat/services/aws/nova_sonic/context.py
index 0ce5ce033..86aa0f0b5 100644
--- a/src/pipecat/services/aws_nova_sonic/context.py
+++ b/src/pipecat/services/aws/nova_sonic/context.py
@@ -33,7 +33,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame
+from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame
from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
diff --git a/src/pipecat/services/aws_nova_sonic/frames.py b/src/pipecat/services/aws/nova_sonic/frames.py
similarity index 100%
rename from src/pipecat/services/aws_nova_sonic/frames.py
rename to src/pipecat/services/aws/nova_sonic/frames.py
diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws/nova_sonic/llm.py
similarity index 99%
rename from src/pipecat/services/aws_nova_sonic/aws.py
rename to src/pipecat/services/aws/nova_sonic/llm.py
index 66b020055..2801de688 100644
--- a/src/pipecat/services/aws_nova_sonic/aws.py
+++ b/src/pipecat/services/aws/nova_sonic/llm.py
@@ -54,14 +54,14 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.aws_nova_sonic.context import (
+from pipecat.services.aws.nova_sonic.context import (
AWSNovaSonicAssistantContextAggregator,
AWSNovaSonicContextAggregatorPair,
AWSNovaSonicLLMContext,
AWSNovaSonicUserContextAggregator,
Role,
)
-from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame
+from pipecat.services.aws.nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame
from pipecat.services.llm_service import LLMService
from pipecat.utils.time import time_now_iso8601
@@ -251,7 +251,7 @@ class AWSNovaSonicLLMService(LLMService):
self._connected_time: Optional[float] = None
self._wants_connection = False
- file_path = files("pipecat.services.aws_nova_sonic").joinpath("ready.wav")
+ file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav")
with wave.open(file_path.open("rb"), "rb") as wav_file:
self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes())
diff --git a/src/pipecat/services/aws_nova_sonic/ready.wav b/src/pipecat/services/aws/nova_sonic/ready.wav
similarity index 100%
rename from src/pipecat/services/aws_nova_sonic/ready.wav
rename to src/pipecat/services/aws/nova_sonic/ready.wav
diff --git a/src/pipecat/services/aws_nova_sonic/__init__.py b/src/pipecat/services/aws_nova_sonic/__init__.py
deleted file mode 100644
index 4da394cf6..000000000
--- a/src/pipecat/services/aws_nova_sonic/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .aws import AWSNovaSonicLLMService, Params
From 2a79b2c85392479e93922519ad2f92d67b728e4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 17:29:02 -0700
Subject: [PATCH 14/51] aws: deprecate aws_nova_sonic
---
.../services/aws_nova_sonic/__init__.py | 19 ++++++++++++++
src/pipecat/services/aws_nova_sonic/aws.py | 25 +++++++++++++++++++
.../services/aws_nova_sonic/context.py | 25 +++++++++++++++++++
src/pipecat/services/aws_nova_sonic/frames.py | 21 ++++++++++++++++
4 files changed, 90 insertions(+)
create mode 100644 src/pipecat/services/aws_nova_sonic/__init__.py
create mode 100644 src/pipecat/services/aws_nova_sonic/aws.py
create mode 100644 src/pipecat/services/aws_nova_sonic/context.py
create mode 100644 src/pipecat/services/aws_nova_sonic/frames.py
diff --git a/src/pipecat/services/aws_nova_sonic/__init__.py b/src/pipecat/services/aws_nova_sonic/__init__.py
new file mode 100644
index 000000000..e1cb912b6
--- /dev/null
+++ b/src/pipecat/services/aws_nova_sonic/__init__.py
@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import warnings
+
+from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService, Params
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.aws_nova_sonic are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.aws.nova_sonic.llm instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py
new file mode 100644
index 000000000..0524829df
--- /dev/null
+++ b/src/pipecat/services/aws_nova_sonic/aws.py
@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""AWS Nova Sonic LLM service implementation for Pipecat AI framework.
+
+This module provides a speech-to-speech LLM service using AWS Nova Sonic, which supports
+bidirectional audio streaming, text generation, and function calling capabilities.
+"""
+
+import warnings
+
+from pipecat.services.aws.nova_sonic.llm import *
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.aws_nova_sonic.aws are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.aws.nova_sonic.llm instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py
new file mode 100644
index 000000000..05a24f337
--- /dev/null
+++ b/src/pipecat/services/aws_nova_sonic/context.py
@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Context management for AWS Nova Sonic LLM service.
+
+This module provides specialized context aggregators and message handling for AWS Nova Sonic,
+including conversation history management and role-specific message processing.
+"""
+
+import warnings
+
+from pipecat.services.aws.nova_sonic.context import *
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.aws_nova_sonic.context are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.aws.nova_sonic.context instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/src/pipecat/services/aws_nova_sonic/frames.py b/src/pipecat/services/aws_nova_sonic/frames.py
new file mode 100644
index 000000000..def5f26c4
--- /dev/null
+++ b/src/pipecat/services/aws_nova_sonic/frames.py
@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Custom frames for AWS Nova Sonic LLM service."""
+
+import warnings
+
+from pipecat.services.aws.nova_sonic.frames import *
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.aws_nova_sonic.frames are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.aws.nova_sonic.frames instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
From aac4ce2d12e310d8cb652ef5d942c90e9dc68050 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 17:29:42 -0700
Subject: [PATCH 15/51] update CHANGELOG with aws_nova_sonic deprecation
---
CHANGELOG.md | 3 +++
1 file changed, 3 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 28da49e20..9493c9d06 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deprecated
+- `pipecat.service.aws_nova_sonic` is now deprecated, use
+ `pipecat.services.aws.nova_sonic` instead.
+
- `GeminiMultimodalLiveLLMService` is now deprecated, use
`GeminiLiveLLMService`.
From aa6e81648a7cac657771c97abd42aaf2551455ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 19:00:54 -0700
Subject: [PATCH 16/51] move openai_realtime to openai.realtime
---
examples/foundational/19-openai-realtime.py | 7 ++++---
examples/foundational/19b-openai-realtime-text.py | 7 ++++---
.../20b-persistent-context-openai-realtime.py | 7 ++++---
src/pipecat/services/openai/__init__.py | 1 +
src/pipecat/services/openai/realtime/__init__.py | 0
.../{openai_realtime => openai/realtime}/azure.py | 0
.../{openai_realtime => openai/realtime}/context.py | 0
.../{openai_realtime => openai/realtime}/events.py | 0
.../{openai_realtime => openai/realtime}/frames.py | 2 +-
.../{openai_realtime => openai/realtime}/openai.py | 0
src/pipecat/services/openai_realtime/__init__.py | 9 ---------
11 files changed, 14 insertions(+), 19 deletions(-)
create mode 100644 src/pipecat/services/openai/realtime/__init__.py
rename src/pipecat/services/{openai_realtime => openai/realtime}/azure.py (100%)
rename src/pipecat/services/{openai_realtime => openai/realtime}/context.py (100%)
rename src/pipecat/services/{openai_realtime => openai/realtime}/events.py (100%)
rename src/pipecat/services/{openai_realtime => openai/realtime}/frames.py (93%)
rename src/pipecat/services/{openai_realtime => openai/realtime}/openai.py (100%)
delete mode 100644 src/pipecat/services/openai_realtime/__init__.py
diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py
index 0f9309f3d..aff4e15d7 100644
--- a/examples/foundational/19-openai-realtime.py
+++ b/examples/foundational/19-openai-realtime.py
@@ -24,14 +24,15 @@ from pipecat.processors.transcript_processor import TranscriptProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.llm_service import FunctionCallParams
-from pipecat.services.openai_realtime import (
+from pipecat.services.openai.realtime.events import (
+ AudioConfiguration,
+ AudioInput,
InputAudioNoiseReduction,
InputAudioTranscription,
- OpenAIRealtimeLLMService,
SemanticTurnDetection,
SessionProperties,
)
-from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput
+from pipecat.services.openai.realtime.openai import OpenAIRealtimeLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py
index b5d1d73e1..8ac916b23 100644
--- a/examples/foundational/19b-openai-realtime-text.py
+++ b/examples/foundational/19b-openai-realtime-text.py
@@ -24,14 +24,15 @@ from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.cartesia import CartesiaTTSService
from pipecat.services.llm_service import FunctionCallParams
-from pipecat.services.openai_realtime import (
+from pipecat.services.openai.realtime.events import (
+ AudioConfiguration,
+ AudioInput,
InputAudioNoiseReduction,
InputAudioTranscription,
- OpenAIRealtimeLLMService,
SemanticTurnDetection,
SessionProperties,
)
-from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput
+from pipecat.services.openai.realtime.openai import OpenAIRealtimeLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py
index 52c7d8f49..100109de3 100644
--- a/examples/foundational/20b-persistent-context-openai-realtime.py
+++ b/examples/foundational/20b-persistent-context-openai-realtime.py
@@ -25,13 +25,14 @@ from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.llm_service import FunctionCallParams
-from pipecat.services.openai_realtime import (
+from pipecat.services.openai.realtime.events import (
+ AudioConfiguration,
+ AudioInput,
InputAudioTranscription,
- OpenAIRealtimeLLMService,
SessionProperties,
TurnDetection,
)
-from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput
+from pipecat.services.openai.realtime.openai import OpenAIRealtimeLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/src/pipecat/services/openai/__init__.py b/src/pipecat/services/openai/__init__.py
index 4decac126..d913c5ee0 100644
--- a/src/pipecat/services/openai/__init__.py
+++ b/src/pipecat/services/openai/__init__.py
@@ -10,6 +10,7 @@ from pipecat.services import DeprecatedModuleProxy
from .image import *
from .llm import *
+from .realtime import *
from .stt import *
from .tts import *
diff --git a/src/pipecat/services/openai/realtime/__init__.py b/src/pipecat/services/openai/realtime/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pipecat/services/openai_realtime/azure.py b/src/pipecat/services/openai/realtime/azure.py
similarity index 100%
rename from src/pipecat/services/openai_realtime/azure.py
rename to src/pipecat/services/openai/realtime/azure.py
diff --git a/src/pipecat/services/openai_realtime/context.py b/src/pipecat/services/openai/realtime/context.py
similarity index 100%
rename from src/pipecat/services/openai_realtime/context.py
rename to src/pipecat/services/openai/realtime/context.py
diff --git a/src/pipecat/services/openai_realtime/events.py b/src/pipecat/services/openai/realtime/events.py
similarity index 100%
rename from src/pipecat/services/openai_realtime/events.py
rename to src/pipecat/services/openai/realtime/events.py
diff --git a/src/pipecat/services/openai_realtime/frames.py b/src/pipecat/services/openai/realtime/frames.py
similarity index 93%
rename from src/pipecat/services/openai_realtime/frames.py
rename to src/pipecat/services/openai/realtime/frames.py
index 290e025f9..8617c6efd 100644
--- a/src/pipecat/services/openai_realtime/frames.py
+++ b/src/pipecat/services/openai/realtime/frames.py
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING
from pipecat.frames.frames import DataFrame, FunctionCallResultFrame
if TYPE_CHECKING:
- from pipecat.services.openai_realtime.context import OpenAIRealtimeLLMContext
+ from pipecat.services.openai.realtime.context import OpenAIRealtimeLLMContext
@dataclass
diff --git a/src/pipecat/services/openai_realtime/openai.py b/src/pipecat/services/openai/realtime/openai.py
similarity index 100%
rename from src/pipecat/services/openai_realtime/openai.py
rename to src/pipecat/services/openai/realtime/openai.py
diff --git a/src/pipecat/services/openai_realtime/__init__.py b/src/pipecat/services/openai_realtime/__init__.py
deleted file mode 100644
index 6f3154f1c..000000000
--- a/src/pipecat/services/openai_realtime/__init__.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from .azure import AzureRealtimeLLMService
-from .events import (
- InputAudioNoiseReduction,
- InputAudioTranscription,
- SemanticTurnDetection,
- SessionProperties,
- TurnDetection,
-)
-from .openai import OpenAIRealtimeLLMService
From 303dd2ec355e4a1e2804ad071d85c44342d62595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 19:02:40 -0700
Subject: [PATCH 17/51] move openai.realtime.openai to openai.realtime.llm
---
examples/foundational/19-openai-realtime.py | 2 +-
examples/foundational/19b-openai-realtime-text.py | 2 +-
examples/foundational/20b-persistent-context-openai-realtime.py | 2 +-
src/pipecat/services/openai/realtime/{openai.py => llm.py} | 0
4 files changed, 3 insertions(+), 3 deletions(-)
rename src/pipecat/services/openai/realtime/{openai.py => llm.py} (100%)
diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py
index aff4e15d7..f182d7c8c 100644
--- a/examples/foundational/19-openai-realtime.py
+++ b/examples/foundational/19-openai-realtime.py
@@ -32,7 +32,7 @@ from pipecat.services.openai.realtime.events import (
SemanticTurnDetection,
SessionProperties,
)
-from pipecat.services.openai.realtime.openai import OpenAIRealtimeLLMService
+from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py
index 8ac916b23..33f3a1cb2 100644
--- a/examples/foundational/19b-openai-realtime-text.py
+++ b/examples/foundational/19b-openai-realtime-text.py
@@ -32,7 +32,7 @@ from pipecat.services.openai.realtime.events import (
SemanticTurnDetection,
SessionProperties,
)
-from pipecat.services.openai.realtime.openai import OpenAIRealtimeLLMService
+from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py
index 100109de3..629a17c67 100644
--- a/examples/foundational/20b-persistent-context-openai-realtime.py
+++ b/examples/foundational/20b-persistent-context-openai-realtime.py
@@ -32,7 +32,7 @@ from pipecat.services.openai.realtime.events import (
SessionProperties,
TurnDetection,
)
-from pipecat.services.openai.realtime.openai import OpenAIRealtimeLLMService
+from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/src/pipecat/services/openai/realtime/openai.py b/src/pipecat/services/openai/realtime/llm.py
similarity index 100%
rename from src/pipecat/services/openai/realtime/openai.py
rename to src/pipecat/services/openai/realtime/llm.py
From 7b05c9283b2f4f0db086daa5a373f794df527319 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 19:06:09 -0700
Subject: [PATCH 18/51] move openai.realtime.azure to azure.realtime.llm
---
examples/foundational/19a-azure-realtime.py | 7 ++++---
src/pipecat/services/azure/realtime/__init__.py | 0
.../{openai/realtime/azure.py => azure/realtime/llm.py} | 6 ++----
3 files changed, 6 insertions(+), 7 deletions(-)
create mode 100644 src/pipecat/services/azure/realtime/__init__.py
rename src/pipecat/services/{openai/realtime/azure.py => azure/realtime/llm.py} (91%)
diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py
index a39826b81..c4b0fc02a 100644
--- a/examples/foundational/19a-azure-realtime.py
+++ b/examples/foundational/19a-azure-realtime.py
@@ -21,13 +21,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
+from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService
from pipecat.services.llm_service import FunctionCallParams
-from pipecat.services.openai_realtime import (
- AzureRealtimeLLMService,
+from pipecat.services.openai.realtime.events import (
+ AudioConfiguration,
+ AudioInput,
InputAudioTranscription,
SessionProperties,
)
-from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/src/pipecat/services/azure/realtime/__init__.py b/src/pipecat/services/azure/realtime/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pipecat/services/openai/realtime/azure.py b/src/pipecat/services/azure/realtime/llm.py
similarity index 91%
rename from src/pipecat/services/openai/realtime/azure.py
rename to src/pipecat/services/azure/realtime/llm.py
index aed53b94c..fb420e10b 100644
--- a/src/pipecat/services/openai/realtime/azure.py
+++ b/src/pipecat/services/azure/realtime/llm.py
@@ -8,15 +8,13 @@
from loguru import logger
-from .openai import OpenAIRealtimeLLMService
+from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService
try:
from websockets.asyncio.client import connect as websocket_connect
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
- logger.error(
- "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
- )
+ logger.error("In order to use Azure Realtime, you need to `pip install pipecat-ai[openai]`.")
raise Exception(f"Missing module: {e}")
From 9698b008da9363675b7c07cce49e252ce9e56e56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 19:13:25 -0700
Subject: [PATCH 19/51] deprecate openai_realtime
---
.../services/openai_realtime/__init__.py | 27 +++++++++++++++++++
src/pipecat/services/openai_realtime/azure.py | 21 +++++++++++++++
.../services/openai_realtime/context.py | 21 +++++++++++++++
.../services/openai_realtime/events.py | 21 +++++++++++++++
.../services/openai_realtime/frames.py | 21 +++++++++++++++
5 files changed, 111 insertions(+)
create mode 100644 src/pipecat/services/openai_realtime/__init__.py
create mode 100644 src/pipecat/services/openai_realtime/azure.py
create mode 100644 src/pipecat/services/openai_realtime/context.py
create mode 100644 src/pipecat/services/openai_realtime/events.py
create mode 100644 src/pipecat/services/openai_realtime/frames.py
diff --git a/src/pipecat/services/openai_realtime/__init__.py b/src/pipecat/services/openai_realtime/__init__.py
new file mode 100644
index 000000000..f1fbefeda
--- /dev/null
+++ b/src/pipecat/services/openai_realtime/__init__.py
@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import warnings
+
+from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService
+from pipecat.services.openai.realtime.events import (
+ InputAudioNoiseReduction,
+ InputAudioTranscription,
+ SemanticTurnDetection,
+ SessionProperties,
+ TurnDetection,
+)
+from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.openai_realtime are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.openai.realtime instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/src/pipecat/services/openai_realtime/azure.py b/src/pipecat/services/openai_realtime/azure.py
new file mode 100644
index 000000000..dae6ef496
--- /dev/null
+++ b/src/pipecat/services/openai_realtime/azure.py
@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Azure OpenAI Realtime LLM service implementation."""
+
+import warnings
+
+from pipecat.services.azure.realtime.llm import *
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.openai_realtime.azure are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.azure.realtime.llm instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/src/pipecat/services/openai_realtime/context.py b/src/pipecat/services/openai_realtime/context.py
new file mode 100644
index 000000000..58f1cfe75
--- /dev/null
+++ b/src/pipecat/services/openai_realtime/context.py
@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""OpenAI Realtime LLM context and aggregator implementations."""
+
+import warnings
+
+from pipecat.services.openai.realtime.context import *
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.openai_realtime.context are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.openai.realtime.context instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/src/pipecat/services/openai_realtime/events.py b/src/pipecat/services/openai_realtime/events.py
new file mode 100644
index 000000000..53b4b0dff
--- /dev/null
+++ b/src/pipecat/services/openai_realtime/events.py
@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Event models and data structures for OpenAI Realtime API communication."""
+
+import warnings
+
+from pipecat.services.openai.realtime.events import *
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.openai_realtime.events are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.openai.realtime.events instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
diff --git a/src/pipecat/services/openai_realtime/frames.py b/src/pipecat/services/openai_realtime/frames.py
new file mode 100644
index 000000000..e7e4d7d9f
--- /dev/null
+++ b/src/pipecat/services/openai_realtime/frames.py
@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Custom frame types for OpenAI Realtime API integration."""
+
+import warnings
+
+from pipecat.services.openai.realtime.frames import *
+
+with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "Types in pipecat.services.openai_realtime.frames are deprecated. "
+ "Please use the equivalent types from "
+ "pipecat.services.openai.realtime.frames instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
From d81b0f63689d0b3c106379dee5b12924f8b6a2dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 19:14:43 -0700
Subject: [PATCH 20/51] update CHANGELOG with openai_realtime deprecation
---
CHANGELOG.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9493c9d06..7079f21f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deprecated
+- `pipecat.service.openai_realtime` is now deprecated, use
+ `pipecat.services.openai.realtime` instead or
+ `pipecat.services.azure.realtime` for Azure Realtime.
+
- `pipecat.service.aws_nova_sonic` is now deprecated, use
`pipecat.services.aws.nova_sonic` instead.
From 5679dde70fa259c7070e554a37ce9ec44dc8e9d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 22:09:11 -0700
Subject: [PATCH 21/51] ai_service: use openai.realtime.events instead of
openai_realtime_beta.events
---
src/pipecat/services/ai_service.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py
index eebdbfb0b..759efcf00 100644
--- a/src/pipecat/services/ai_service.py
+++ b/src/pipecat/services/ai_service.py
@@ -97,9 +97,7 @@ class AIService(FrameProcessor):
pass
async def _update_settings(self, settings: Mapping[str, Any]):
- from pipecat.services.openai_realtime_beta.events import (
- SessionProperties,
- )
+ from pipecat.services.openai.realtime.events import SessionProperties
for key, value in settings.items():
logger.debug("Update request for:", key, value)
@@ -111,9 +109,7 @@ class AIService(FrameProcessor):
logger.debug("Attempting to update", key, value)
try:
- from pipecat.services.openai_realtime_beta.events import (
- TurnDetection,
- )
+ from pipecat.services.openai.realtime.events import TurnDetection
if isinstance(self._session_properties, SessionProperties):
current_properties = self._session_properties
From 9935a68018c43b5388fa07e41553eb9aca229112 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 22:14:52 -0700
Subject: [PATCH 22/51] examples(19b): fix deprecations
---
examples/foundational/19b-openai-realtime-text.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py
index 33f3a1cb2..bb63a4814 100644
--- a/examples/foundational/19b-openai-realtime-text.py
+++ b/examples/foundational/19b-openai-realtime-text.py
@@ -22,7 +22,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.transcript_processor import TranscriptProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.cartesia import CartesiaTTSService
+from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.realtime.events import (
AudioConfiguration,
From 7cec013666d5620fbfae62c175d6576435463728 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Thu, 9 Oct 2025 21:56:54 -0700
Subject: [PATCH 23/51] google: rename google.gemini_live.gemini to
google.gemini_live.llm
---
examples/foundational/26-gemini-multimodal-live.py | 2 +-
.../26a-gemini-multimodal-live-transcription.py | 2 +-
.../26b-gemini-multimodal-live-function-calling.py | 2 +-
.../foundational/26c-gemini-multimodal-live-video.py | 2 +-
.../foundational/26d-gemini-multimodal-live-text.py | 2 +-
.../26e-gemini-multimodal-google-search.py | 2 +-
.../26f-gemini-multimodal-live-files-api.py | 4 +---
.../26g-gemini-multimodal-live-groundingMetadata.py | 4 ++--
.../26h-gemini-multimodal-live-vertex.py | 1 -
.../26i-gemini-multimodal-live-graceful-end.py | 2 +-
examples/foundational/46-video-processing.py | 2 +-
.../services/gemini_multimodal_live/gemini.py | 12 +++++-------
src/pipecat/services/google/__init__.py | 1 +
src/pipecat/services/google/gemini_live/__init__.py | 2 +-
.../google/gemini_live/{gemini.py => llm.py} | 0
src/pipecat/services/google/gemini_live/vertex.py | 7 ++-----
16 files changed, 20 insertions(+), 27 deletions(-)
rename src/pipecat/services/google/gemini_live/{gemini.py => llm.py} (100%)
diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py
index fc7ad9d35..dd5dfcffc 100644
--- a/examples/foundational/26-gemini-multimodal-live.py
+++ b/examples/foundational/26-gemini-multimodal-live.py
@@ -17,7 +17,7 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py
index cf8660edf..de277156b 100644
--- a/examples/foundational/26a-gemini-multimodal-live-transcription.py
+++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py
@@ -20,7 +20,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.transcript_processor import TranscriptProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
index a08f5a352..8fecf0de6 100644
--- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py
+++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py
@@ -22,7 +22,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py
index 1ac4eb923..7b765075e 100644
--- a/examples/foundational/26c-gemini-multimodal-live-video.py
+++ b/examples/foundational/26c-gemini-multimodal-live-video.py
@@ -24,7 +24,7 @@ from pipecat.runner.utils import (
maybe_capture_participant_camera,
maybe_capture_participant_screen,
)
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py
index 68fd79c54..05b792c00 100644
--- a/examples/foundational/26d-gemini-multimodal-live-text.py
+++ b/examples/foundational/26d-gemini-multimodal-live-text.py
@@ -20,7 +20,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.cartesia.tts import CartesiaTTSService
-from pipecat.services.google.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.llm import (
GeminiLiveLLMService,
GeminiModalities,
InputParams,
diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-multimodal-google-search.py
index 87c754b75..178fdd282 100644
--- a/examples/foundational/26e-gemini-multimodal-google-search.py
+++ b/examples/foundational/26e-gemini-multimodal-google-search.py
@@ -19,7 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26f-gemini-multimodal-live-files-api.py b/examples/foundational/26f-gemini-multimodal-live-files-api.py
index 51dad712d..eeda16f52 100644
--- a/examples/foundational/26f-gemini-multimodal-live-files-api.py
+++ b/examples/foundational/26f-gemini-multimodal-live-files-api.py
@@ -19,9 +19,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.gemini import (
- GeminiLiveLLMService,
-)
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py b/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
index 2b424eb18..bea1756b2 100644
--- a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
+++ b/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
@@ -9,13 +9,13 @@ from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.frames.frames import Frame, LLMRunFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.google.frames import LLMSearchResponseFrame
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/examples/foundational/26h-gemini-multimodal-live-vertex.py b/examples/foundational/26h-gemini-multimodal-live-vertex.py
index f8dfc8c52..97ebbbbde 100644
--- a/examples/foundational/26h-gemini-multimodal-live-vertex.py
+++ b/examples/foundational/26h-gemini-multimodal-live-vertex.py
@@ -17,7 +17,6 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
from pipecat.services.google.gemini_live.vertex import GeminiLiveVertexLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py b/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
index 3527e89e3..e51bbb032 100644
--- a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
+++ b/examples/foundational/26i-gemini-multimodal-live-graceful-end.py
@@ -22,7 +22,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.gemini import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
diff --git a/examples/foundational/46-video-processing.py b/examples/foundational/46-video-processing.py
index 0152f3211..62ea5debe 100644
--- a/examples/foundational/46-video-processing.py
+++ b/examples/foundational/46-video-processing.py
@@ -20,7 +20,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.daily.transport import DailyParams, DailyTransport
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index a3547551a..8d6847425 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -18,10 +18,10 @@ voice transcription, streaming responses, and tool usage.
import warnings
-from pipecat.services.google.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.llm import (
ContextWindowCompressionParams as _ContextWindowCompressionParams,
)
-from pipecat.services.google.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.llm import (
GeminiLiveAssistantContextAggregator,
GeminiLiveContext,
GeminiLiveContextAggregatorPair,
@@ -29,11 +29,9 @@ from pipecat.services.google.gemini_live.gemini import (
GeminiLiveUserContextAggregator,
GeminiModalities,
)
-from pipecat.services.google.gemini_live.gemini import (
- GeminiMediaResolution as _GeminiMediaResolution,
-)
-from pipecat.services.google.gemini_live.gemini import GeminiVADParams as _GeminiVADParams
-from pipecat.services.google.gemini_live.gemini import InputParams as _InputParams
+from pipecat.services.google.gemini_live.llm import GeminiMediaResolution as _GeminiMediaResolution
+from pipecat.services.google.gemini_live.llm import GeminiVADParams as _GeminiVADParams
+from pipecat.services.google.gemini_live.llm import InputParams as _InputParams
with warnings.catch_warnings():
warnings.simplefilter("always")
diff --git a/src/pipecat/services/google/__init__.py b/src/pipecat/services/google/__init__.py
index ec187000f..08e295b9e 100644
--- a/src/pipecat/services/google/__init__.py
+++ b/src/pipecat/services/google/__init__.py
@@ -9,6 +9,7 @@ import sys
from pipecat.services import DeprecatedModuleProxy
from .frames import *
+from .gemini_live import *
from .image import *
from .llm import *
from .llm_openai import *
diff --git a/src/pipecat/services/google/gemini_live/__init__.py b/src/pipecat/services/google/gemini_live/__init__.py
index 6a2d33bd8..1ab28050e 100644
--- a/src/pipecat/services/google/gemini_live/__init__.py
+++ b/src/pipecat/services/google/gemini_live/__init__.py
@@ -1,3 +1,3 @@
from .file_api import GeminiFileAPI
-from .gemini import GeminiLiveLLMService
+from .llm import GeminiLiveLLMService
from .vertex import GeminiLiveVertexLLMService
diff --git a/src/pipecat/services/google/gemini_live/gemini.py b/src/pipecat/services/google/gemini_live/llm.py
similarity index 100%
rename from src/pipecat/services/google/gemini_live/gemini.py
rename to src/pipecat/services/google/gemini_live/llm.py
diff --git a/src/pipecat/services/google/gemini_live/vertex.py b/src/pipecat/services/google/gemini_live/vertex.py
index 29ff63a0b..4f1b57f4c 100644
--- a/src/pipecat/services/google/gemini_live/vertex.py
+++ b/src/pipecat/services/google/gemini_live/vertex.py
@@ -12,13 +12,12 @@ streaming responses, and tool usage.
"""
import json
-import os
from typing import List, Optional, Union
from loguru import logger
from pipecat.adapters.schemas.tools_schema import ToolsSchema
-from pipecat.services.google.gemini_live.gemini import (
+from pipecat.services.google.gemini_live.llm import (
GeminiLiveLLMService,
HttpOptions,
InputParams,
@@ -33,9 +32,7 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
- logger.error(
- "In order to use Google Vertex AI, you need to `pip install pipecat-ai[google]`. Also, set up your Google credentials properly."
- )
+ logger.error("In order to use Google Vertex AI, you need to `pip install pipecat-ai[google]`.")
raise Exception(f"Missing module: {e}")
From 1b3afb551196ef8d0a1db459da7c40e809a264ae Mon Sep 17 00:00:00 2001
From: Filipi Fuchter
Date: Fri, 10 Oct 2025 09:44:47 -0300
Subject: [PATCH 24/51] Added audio filter KrispVivaFilter using the Krisp VIVA
SDK
---
CHANGELOG.md | 2 +
env.example | 3 +
.../07p-interruptible-krisp-viva.py | 129 ++++++++++++
.../audio/filters/krisp_viva_filter.py | 193 ++++++++++++++++++
4 files changed, 327 insertions(+)
create mode 100644 examples/foundational/07p-interruptible-krisp-viva.py
create mode 100644 src/pipecat/audio/filters/krisp_viva_filter.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7079f21f5..02b31d93f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- Added audio filter `KrispVivaFilter` using the Krisp VIVA SDK.
+
- Added `--folder` argument to the runner, allowing files saved in that folder
to be downloaded from `http://HOST:PORT/file/FILE`.
diff --git a/env.example b/env.example
index 22768494b..f707f49c9 100644
--- a/env.example
+++ b/env.example
@@ -90,6 +90,9 @@ SIMLI_FACE_ID=...
# Krisp
KRISP_MODEL_PATH=...
+# Krisp Viva
+KRISP_VIVA_MODEL_PATH=...
+
# DeepSeek
DEEPSEEK_API_KEY=...
diff --git a/examples/foundational/07p-interruptible-krisp-viva.py b/examples/foundational/07p-interruptible-krisp-viva.py
new file mode 100644
index 000000000..c7ca15b40
--- /dev/null
+++ b/examples/foundational/07p-interruptible-krisp-viva.py
@@ -0,0 +1,129 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter
+from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import LLMRunFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.llm_context import LLMContext
+from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
+from pipecat.runner.types import RunnerArguments
+from pipecat.runner.utils import create_transport
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.deepgram.tts import DeepgramTTSService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.transports.daily.transport import DailyParams
+from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
+
+load_dotenv(override=True)
+
+# We store functions so objects (e.g. SileroVADAnalyzer) don't get
+# instantiated. The function will be called when the desired transport gets
+# selected.
+transport_params = {
+ "daily": lambda: DailyParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+ turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+ audio_in_filter=KrispVivaFilter(),
+ ),
+ "twilio": lambda: FastAPIWebsocketParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+ turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+ audio_in_filter=KrispVivaFilter(),
+ ),
+ "webrtc": lambda: TransportParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+ turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+ audio_in_filter=KrispVivaFilter(),
+ ),
+}
+
+
+async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
+ logger.info(f"Starting bot")
+
+ stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+ tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en")
+
+ llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+ messages = [
+ {
+ "role": "system",
+ "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+ },
+ ]
+
+ context = LLMContext(messages)
+ context_aggregator = LLMContextAggregatorPair(context)
+
+ pipeline = Pipeline(
+ [
+ transport.input(), # Transport user input
+ stt, # STT
+ context_aggregator.user(), # User responses
+ llm, # LLM
+ tts, # TTS
+ transport.output(), # Transport bot output
+ context_aggregator.assistant(), # Assistant spoken responses
+ ]
+ )
+
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ enable_metrics=True,
+ enable_usage_metrics=True,
+ ),
+ idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
+ )
+
+ @transport.event_handler("on_client_connected")
+ async def on_client_connected(transport, client):
+ logger.info(f"Client connected")
+ # Kick off the conversation.
+ messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+ await task.queue_frames([LLMRunFrame()])
+
+ @transport.event_handler("on_client_disconnected")
+ async def on_client_disconnected(transport, client):
+ logger.info(f"Client disconnected")
+ await task.cancel()
+
+ runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
+
+ await runner.run(task)
+
+
+async def bot(runner_args: RunnerArguments):
+ """Main bot entry point compatible with Pipecat Cloud."""
+ transport = await create_transport(runner_args, transport_params)
+ await run_bot(transport, runner_args)
+
+
+if __name__ == "__main__":
+ from pipecat.runner.run import main
+
+ main()
diff --git a/src/pipecat/audio/filters/krisp_viva_filter.py b/src/pipecat/audio/filters/krisp_viva_filter.py
new file mode 100644
index 000000000..ddb489168
--- /dev/null
+++ b/src/pipecat/audio/filters/krisp_viva_filter.py
@@ -0,0 +1,193 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Krisp noise reduction audio filter for Pipecat.
+
+This module provides an audio filter implementation using Krisp VIVA SDK.
+"""
+
+import os
+
+import numpy as np
+from loguru import logger
+
+from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
+from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
+
+try:
+ import krisp_audio
+except ModuleNotFoundError as e:
+ logger.error(f"Exception: {e}")
+ logger.error("In order to use the Krisp filter, you need to install krisp_audio.")
+ raise Exception(f"Missing module: {e}")
+
+
+def _log_callback(log_message, log_level):
+ logger.info(f"[{log_level}] {log_message}")
+
+
+class KrispVivaFilter(BaseAudioFilter):
+ """Audio filter using the Krisp VIVA SDK.
+
+ Provides real-time noise reduction for audio streams using Krisp's
+ proprietary noise suppression algorithms. This filter requires a
+ valid Krisp model file to operate.
+
+ Supported sample rates:
+ - 8000 Hz
+ - 16000 Hz
+ - 24000 Hz
+ - 32000 Hz
+ - 44100 Hz
+ - 48000 Hz
+ """
+
+ # Initialize Krisp Audio SDK globally
+ krisp_audio.globalInit("", _log_callback, krisp_audio.LogLevel.Off)
+ SDK_VERSION = krisp_audio.getVersion()
+ logger.debug(
+ f"Krisp Audio Python SDK Version: {SDK_VERSION.major}."
+ f"{SDK_VERSION.minor}.{SDK_VERSION.patch}"
+ )
+
+ SAMPLE_RATES = {
+ 8000: krisp_audio.SamplingRate.Sr8000Hz,
+ 16000: krisp_audio.SamplingRate.Sr16000Hz,
+ 24000: krisp_audio.SamplingRate.Sr24000Hz,
+ 32000: krisp_audio.SamplingRate.Sr32000Hz,
+ 44100: krisp_audio.SamplingRate.Sr44100Hz,
+ 48000: krisp_audio.SamplingRate.Sr48000Hz,
+ }
+
+ FRAME_SIZE_MS = 10 # Krisp requires audio frames of 10ms duration for processing.
+
+ def __init__(self, model_path: str = None, noise_suppression_level: int = 100) -> None:
+ """Initialize the Krisp noise reduction filter.
+
+ Args:
+ model_path: Path to the Krisp model file (.kef extension).
+ If None, uses KRISP_VIVA_MODEL_PATH environment variable.
+ noise_suppression_level: Noise suppression level.
+
+ Raises:
+ ValueError: If model_path is not provided and KRISP_VIVA_MODEL_PATH is not set.
+ Exception: If model file doesn't have .kef extension.
+ FileNotFoundError: If model file doesn't exist.
+ """
+ super().__init__()
+
+ # Set model path, checking environment if not specified
+ self._model_path = model_path or os.getenv("KRISP_VIVA_MODEL_PATH")
+ if not self._model_path:
+ logger.error("Model path is not provided and KRISP_VIVA_MODEL_PATH is not set.")
+ raise ValueError("Model path for KrispAudioProcessor must be provided.")
+
+ if not self._model_path.endswith(".kef"):
+ raise Exception("Model is expected with .kef extension")
+
+ if not os.path.isfile(self._model_path):
+ raise FileNotFoundError(f"Model file not found: {self._model_path}")
+
+ self._filtering = True
+ self._session = None
+ self._samples_per_frame = None
+ self._noise_suppression_level = noise_suppression_level
+
+ # Audio buffer to accumulate samples for complete frames
+ self._audio_buffer = bytearray()
+
+ def _int_to_sample_rate(self, sample_rate):
+ """Convert integer sample rate to krisp_audio SamplingRate enum.
+
+ Args:
+ sample_rate: Sample rate as integer
+
+ Returns:
+ krisp_audio.SamplingRate enum value
+
+ Raises:
+ ValueError: If sample rate is not supported
+ """
+ if sample_rate not in self.SAMPLE_RATES:
+ raise ValueError("Unsupported sample rate")
+ return self.SAMPLE_RATES[sample_rate]
+
+ async def start(self, sample_rate: int):
+ """Initialize the Krisp processor with the transport's sample rate.
+
+ Args:
+ sample_rate: The sample rate of the input transport in Hz.
+ """
+ model_info = krisp_audio.ModelInfo()
+ model_info.path = self._model_path
+
+ nc_cfg = krisp_audio.NcSessionConfig()
+ nc_cfg.inputSampleRate = self._int_to_sample_rate(sample_rate)
+ nc_cfg.inputFrameDuration = krisp_audio.FrameDuration.Fd10ms
+ nc_cfg.outputSampleRate = nc_cfg.inputSampleRate
+ nc_cfg.modelInfo = model_info
+
+ self._samples_per_frame = int((sample_rate * self.FRAME_SIZE_MS) / 1000)
+ self._session = krisp_audio.NcInt16.create(nc_cfg)
+
+ async def stop(self):
+ """Clean up the Krisp processor when stopping."""
+ self._session = None
+
+ async def process_frame(self, frame: FilterControlFrame):
+ """Process control frames to enable/disable filtering.
+
+ Args:
+ frame: The control frame containing filter commands.
+ """
+ if isinstance(frame, FilterEnableFrame):
+ self._filtering = frame.enable
+
+ async def filter(self, audio: bytes) -> bytes:
+ """Apply Krisp noise reduction to audio data.
+
+ Args:
+ audio: Raw audio data as bytes to be filtered.
+
+ Returns:
+ Noise-reduced audio data as bytes.
+ """
+ if not self._filtering:
+ return audio
+
+ # Add incoming audio to our buffer
+ self._audio_buffer.extend(audio)
+
+ # Calculate how many complete frames we can process
+ total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample
+ num_complete_frames = total_samples // self._samples_per_frame
+
+ if num_complete_frames == 0:
+ # Not enough samples for a complete frame yet, return empty
+ return b""
+
+ # Calculate how many bytes we need for complete frames
+ complete_samples_count = num_complete_frames * self._samples_per_frame
+ bytes_to_process = complete_samples_count * 2 # 2 bytes per sample
+
+ # Extract the bytes we can process
+ audio_to_process = bytes(self._audio_buffer[:bytes_to_process])
+
+ # Remove processed bytes from buffer, keep the remainder
+ self._audio_buffer = self._audio_buffer[bytes_to_process:]
+
+ # Process the complete frames
+ samples = np.frombuffer(audio_to_process, dtype=np.int16)
+ frames = samples.reshape(-1, self._samples_per_frame)
+ processed_samples = np.empty_like(samples)
+
+ for i, frame in enumerate(frames):
+ cleaned_frame = self._session.process(frame, self._noise_suppression_level)
+ processed_samples[i * self._samples_per_frame : (i + 1) * self._samples_per_frame] = (
+ cleaned_frame
+ )
+
+ return processed_samples.tobytes()
From 8ee28b37cdd31811294ed077c0947a081004fcbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Fri, 10 Oct 2025 06:41:19 -0700
Subject: [PATCH 25/51] google: rename google.gemini_live.vertext to
google.gemini_live.llm_vertex
---
examples/foundational/26h-gemini-multimodal-live-vertex.py | 2 +-
src/pipecat/services/google/gemini_live/__init__.py | 2 +-
.../services/google/gemini_live/{vertex.py => llm_vertex.py} | 0
3 files changed, 2 insertions(+), 2 deletions(-)
rename src/pipecat/services/google/gemini_live/{vertex.py => llm_vertex.py} (100%)
diff --git a/examples/foundational/26h-gemini-multimodal-live-vertex.py b/examples/foundational/26h-gemini-multimodal-live-vertex.py
index 97ebbbbde..79bfac531 100644
--- a/examples/foundational/26h-gemini-multimodal-live-vertex.py
+++ b/examples/foundational/26h-gemini-multimodal-live-vertex.py
@@ -17,7 +17,7 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.vertex import GeminiLiveVertexLLMService
+from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
diff --git a/src/pipecat/services/google/gemini_live/__init__.py b/src/pipecat/services/google/gemini_live/__init__.py
index 1ab28050e..142ca2a83 100644
--- a/src/pipecat/services/google/gemini_live/__init__.py
+++ b/src/pipecat/services/google/gemini_live/__init__.py
@@ -1,3 +1,3 @@
from .file_api import GeminiFileAPI
from .llm import GeminiLiveLLMService
-from .vertex import GeminiLiveVertexLLMService
+from .llm_vertex import GeminiLiveVertexLLMService
diff --git a/src/pipecat/services/google/gemini_live/vertex.py b/src/pipecat/services/google/gemini_live/llm_vertex.py
similarity index 100%
rename from src/pipecat/services/google/gemini_live/vertex.py
rename to src/pipecat/services/google/gemini_live/llm_vertex.py
From 3894d2a4b9fd8c339882720b357656f79ba92545 Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Thu, 9 Oct 2025 22:03:06 -0400
Subject: [PATCH 26/51] Deprecate LivekitFrameSerializer
---
CHANGELOG.md | 2 ++
src/pipecat/serializers/livekit.py | 20 ++++++++++++++++++++
2 files changed, 22 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02b31d93f..9763f78f2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deprecated
+- `LivekitFrameSerializer` is now deprecated. Use `LiveKitTransport` instead.
+
- `pipecat.service.openai_realtime` is now deprecated, use
`pipecat.services.openai.realtime` instead or
`pipecat.services.azure.realtime` for Azure Realtime.
diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py
index 3d4188960..f3a34c434 100644
--- a/src/pipecat/serializers/livekit.py
+++ b/src/pipecat/serializers/livekit.py
@@ -25,11 +25,31 @@ except ModuleNotFoundError as e:
class LivekitFrameSerializer(FrameSerializer):
"""Serializer for converting between Pipecat frames and LiveKit audio frames.
+ .. deprecated:: 0.0.90
+
+ This class is deprecated and will be removed in a future version.
+ Please use LiveKitTransport instead, which handles audio streaming
+ and frame conversion natively.
+
This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects
to LiveKit AudioFrame objects for transmission, and the reverse conversion
for received audio data.
"""
+ def __init__(self):
+ """Initialize the LiveKit frame serializer."""
+ super().__init__()
+ import warnings
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("always")
+ warnings.warn(
+ "LivekitFrameSerializer is deprecated and will be removed in a future version. "
+ "Please use LiveKitTransport instead, which handles audio streaming natively.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
@property
def type(self) -> FrameSerializerType:
"""Get the serializer type.
From 8b62a96878da6863148ed923b2dcf1cb9d246940 Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Fri, 10 Oct 2025 10:12:00 -0400
Subject: [PATCH 27/51] Improve how we're deprecating `location` and
`project_id` in `GoogleVertexLLMService`, allowing user code to (correctly)
continue referring to `GoogleVertexLLMService.InputParams`.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Also fix the slightly wrong (but so far harmless) pattern of initializing `OpenAILLMService.InputParams()` in the `GoogleVertexLLMService` if `params` wasn't provided—we should be letting the superclass decide what to do if the argument isn't specified.
---
src/pipecat/services/google/llm_vertex.py | 69 +++++++++++++----------
1 file changed, 38 insertions(+), 31 deletions(-)
diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py
index 3a3bf9ea9..cc0b8654e 100644
--- a/src/pipecat/services/google/llm_vertex.py
+++ b/src/pipecat/services/google/llm_vertex.py
@@ -51,32 +51,45 @@ class GoogleVertexLLMService(OpenAILLMService):
class InputParams(OpenAILLMService.InputParams):
"""Input parameters specific to Vertex AI.
- .. deprecated:: 0.0.90
- Use `OpenAILLMService.InputParams` instead and provide `location` and
- `project_id` as direct arguments to `GoogleVertexLLMService.__init__()`.
-
Parameters:
location: GCP region for Vertex AI endpoint (e.g., "us-east4").
+
+ .. deprecated:: 0.0.90
+ Use `location` as a direct argument to
+ `GoogleVertexLLMService.__init__()` instead.
+
project_id: Google Cloud project ID.
+
+ .. deprecated:: 0.0.90
+ Use `project_id` as a direct argument to
+ `GoogleVertexLLMService.__init__()` instead.
"""
# https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations
- location: str = "us-east4"
- project_id: str
+ location: Optional[str] = None
+ project_id: Optional[str] = None
def __init__(self, **kwargs):
"""Initializes the InputParams."""
import warnings
with warnings.catch_warnings():
- warnings.simplefilter("ignore", DeprecationWarning)
- warnings.warn(
- "GoogleVertexLLMService.InputParams is deprecated. "
- "Please use OpenAILLMService.InputParams instead and provide "
- "'location' and 'project_id' as direct arguments to __init__.",
- DeprecationWarning,
- stacklevel=2,
- )
+ warnings.simplefilter("always")
+ if "location" in kwargs and kwargs["location"] is not None:
+ warnings.warn(
+ "GoogleVertexLLMService.InputParams.location is deprecated. "
+ "Please provide 'location' as a direct argument to GoogleVertexLLMService.__init__() instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ if "project_id" in kwargs and kwargs["project_id"] is not None:
+ warnings.warn(
+ "GoogleVertexLLMService.InputParams.project_id is deprecated. "
+ "Please provide 'project_id' as a direct argument to GoogleVertexLLMService.__init__() instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
super().__init__(**kwargs)
def __init__(
@@ -87,7 +100,6 @@ class GoogleVertexLLMService(OpenAILLMService):
model: str = "google/gemini-2.0-flash-001",
location: Optional[str] = None,
project_id: Optional[str] = None,
- params: Optional[InputParams] = None,
**kwargs,
):
"""Initializes the VertexLLMService.
@@ -98,17 +110,13 @@ class GoogleVertexLLMService(OpenAILLMService):
model: Model identifier (e.g., "google/gemini-2.0-flash-001").
location: GCP region for Vertex AI endpoint (e.g., "us-east4").
project_id: Google Cloud project ID.
- params: Vertex AI input parameters including location and project.
-
- .. deprecated:: 0.0.90
- Use `OpenAILLMService.InputParams` instead and provide `location`
- and `project_id` as direct arguments to `__init__()`.
-
**kwargs: Additional arguments passed to OpenAILLMService.
"""
- # Handle deprecated InputParams
- if params is not None and isinstance(params, GoogleVertexLLMService.InputParams):
- # Extract location and project_id from params if not provided directly
+ # Handle deprecated InputParams fields
+ if "params" in kwargs and isinstance(kwargs["params"], GoogleVertexLLMService.InputParams):
+ params = kwargs["params"]
+ # Extract location and project_id from params if not provided
+ # directly, for backward compatibility
if project_id is None:
project_id = params.project_id
if location is None:
@@ -117,15 +125,15 @@ class GoogleVertexLLMService(OpenAILLMService):
params = OpenAILLMService.InputParams(
**params.model_dump(exclude={"location", "project_id"}, exclude_unset=True)
)
- else:
- params = params or OpenAILLMService.InputParams()
+ kwargs["params"] = params
- # Validate parameters
- # NOTE: once we remove deprecated InputParams, we can update __init__()
- # signature with the following:
+ # Validate project_id and location parameters
+ # NOTE: once we remove Vertex-spcific InputParams class, we can update
+ # __init__() signature as follows:
# - location: str = "us-east4",
# - project_id: str,
- # For now, we need them as-is to maintain backward compatibility.
+ # But for now, we need them as-is to maintain proper backward
+ # compatibility.
if project_id is None:
raise ValueError("project_id is required")
if location is None:
@@ -138,7 +146,6 @@ class GoogleVertexLLMService(OpenAILLMService):
api_key=self._api_key,
base_url=base_url,
model=model,
- params=params,
**kwargs,
)
From 523e890c8cd0fb5872baa9ab981193082b7096af Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Fri, 10 Oct 2025 10:30:38 -0400
Subject: [PATCH 28/51] Fix deprecation messages pointing users to the new
import paths for Gemini Live
---
src/pipecat/services/gemini_multimodal_live/file_api.py | 4 ++--
src/pipecat/services/gemini_multimodal_live/gemini.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/pipecat/services/gemini_multimodal_live/file_api.py b/src/pipecat/services/gemini_multimodal_live/file_api.py
index ed4d6c522..367d6797a 100644
--- a/src/pipecat/services/gemini_multimodal_live/file_api.py
+++ b/src/pipecat/services/gemini_multimodal_live/file_api.py
@@ -12,7 +12,7 @@ this API can be referenced in Gemini generative model calls.
.. deprecated:: 0.0.90
Importing GeminiFileAPI from this module is deprecated.
- Import it from pipecat.services.gemini_live.file_api instead.
+ Import it from pipecat.services.google.gemini_live.file_api instead.
"""
import warnings
@@ -32,7 +32,7 @@ except ModuleNotFoundError as e:
warnings.warn(
"Importing GeminiFileAPI from "
"pipecat.services.gemini_multimodal_live.file_api is deprecated. "
- "Please import it from pipecat.services.gemini_live.file_api instead.",
+ "Please import it from pipecat.services.google.gemini_live.file_api instead.",
DeprecationWarning,
stacklevel=2,
)
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index 8d6847425..e56e34e9d 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -12,7 +12,7 @@ voice transcription, streaming responses, and tool usage.
.. deprecated:: 0.0.90
This module is deprecated. Please use the equivalent types from
- pipecat.services.gemini_live.gemini instead. Note that the new type names
+ pipecat.services.google.gemini_live.llm instead. Note that the new type names
do not include 'Multimodal'.
"""
@@ -38,7 +38,7 @@ with warnings.catch_warnings():
warnings.warn(
"Types in pipecat.services.gemini_multimodal_live.gemini are deprecated. "
"Please use the equivalent types from "
- "pipecat.services.gemini_live.gemini instead. Note that the new type "
+ "pipecat.services.google.gemini_live.llm instead. Note that the new type "
"names do not include 'Multimodal' "
"(e.g. `GeminiMultimodalLiveLLMService` is now `GeminiLiveLLMService`).",
DeprecationWarning,
From 4b415721e26d039db8e8736b5678b57b0b7764f8 Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Fri, 10 Oct 2025 12:25:23 -0400
Subject: [PATCH 29/51] Update release evals for OpenAI Realtime, Gemini Live
Vertex; shorten 26 foundational names
---
CHANGELOG.md | 10 +-
...i-multimodal-live.py => 26-gemini-live.py} | 0
...on.py => 26a-gemini-live-transcription.py} | 0
...py => 26b-gemini-live-function-calling.py} | 2 +-
...live-video.py => 26c-gemini-live-video.py} | 0
...l-live-text.py => 26d-gemini-live-text.py} | 2 +-
...ch.py => 26e-gemini-live-google-search.py} | 0
...es-api.py => 26f-gemini-live-files-api.py} | 0
...y => 26g-gemini-live-groundingMetadata.py} | 0
...26h-gemini-live-vertex-function-calling.py | 191 ++++++++++++++++++
.../26h-gemini-multimodal-live-vertex.py | 133 ------------
...end.py => 26i-gemini-live-graceful-end.py} | 0
examples/foundational/README.md | 2 +-
scripts/evals/run-release-evals.py | 17 +-
14 files changed, 212 insertions(+), 145 deletions(-)
rename examples/foundational/{26-gemini-multimodal-live.py => 26-gemini-live.py} (100%)
rename examples/foundational/{26a-gemini-multimodal-live-transcription.py => 26a-gemini-live-transcription.py} (100%)
rename examples/foundational/{26b-gemini-multimodal-live-function-calling.py => 26b-gemini-live-function-calling.py} (98%)
rename examples/foundational/{26c-gemini-multimodal-live-video.py => 26c-gemini-live-video.py} (100%)
rename examples/foundational/{26d-gemini-multimodal-live-text.py => 26d-gemini-live-text.py} (98%)
rename examples/foundational/{26e-gemini-multimodal-google-search.py => 26e-gemini-live-google-search.py} (100%)
rename examples/foundational/{26f-gemini-multimodal-live-files-api.py => 26f-gemini-live-files-api.py} (100%)
rename examples/foundational/{26g-gemini-multimodal-live-groundingMetadata.py => 26g-gemini-live-groundingMetadata.py} (100%)
create mode 100644 examples/foundational/26h-gemini-live-vertex-function-calling.py
delete mode 100644 examples/foundational/26h-gemini-multimodal-live-vertex.py
rename examples/foundational/{26i-gemini-multimodal-live-graceful-end.py => 26i-gemini-live-graceful-end.py} (100%)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9763f78f2..70324c65e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1487,7 +1487,7 @@ quality and critical bugs impacting `ParallelPipelines` functionality.**
- Added `session_token` parameter to `AWSNovaSonicLLMService`.
- Added Gemini Multimodal Live File API for uploading, fetching, listing, and
- deleting files. See `26f-gemini-multimodal-live-files-api.py` for example usage.
+ deleting files. See `26f-gemini-live-files-api.py` for example usage.
### Changed
@@ -3493,7 +3493,7 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general"))
- Added the new modalities option and helper function to set Gemini output
modalities.
-- Added `examples/foundational/26d-gemini-multimodal-live-text.py` which is
+- Added `examples/foundational/26d-gemini-live-text.py` which is
using Gemini as TEXT modality and using another TTS provider for TTS process.
### Changed
@@ -3680,9 +3680,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general"))
- Added new foundational examples for `GeminiMultimodalLiveLLMService`:
- `26-gemini-multimodal-live.py`
- - `26a-gemini-multimodal-live-transcription.py`
- - `26b-gemini-multimodal-live-video.py`
- - `26c-gemini-multimodal-live-video.py`
+ - `26a-gemini-live-transcription.py`
+ - `26b-gemini-live-video.py`
+ - `26c-gemini-live-video.py`
- Added `SimliVideoService`. This is an integration for Simli AI avatars.
(see https://www.simli.com)
diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-live.py
similarity index 100%
rename from examples/foundational/26-gemini-multimodal-live.py
rename to examples/foundational/26-gemini-live.py
diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-live-transcription.py
similarity index 100%
rename from examples/foundational/26a-gemini-multimodal-live-transcription.py
rename to examples/foundational/26a-gemini-live-transcription.py
diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-live-function-calling.py
similarity index 98%
rename from examples/foundational/26b-gemini-multimodal-live-function-calling.py
rename to examples/foundational/26b-gemini-live-function-calling.py
index 8fecf0de6..65d159bb0 100644
--- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py
+++ b/examples/foundational/26b-gemini-live-function-calling.py
@@ -122,7 +122,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
required=["location"],
)
search_tool = {"google_search": {}}
- # KNOWN ISSUE: If using GeminiVertexMultimodalLiveLLMService, it appears
+ # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears
# you cannot use the "google_search" tool alongside other tools.
# See https://github.com/googleapis/python-genai/issues/941.
tools = ToolsSchema(
diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-live-video.py
similarity index 100%
rename from examples/foundational/26c-gemini-multimodal-live-video.py
rename to examples/foundational/26c-gemini-live-video.py
diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-live-text.py
similarity index 98%
rename from examples/foundational/26d-gemini-multimodal-live-text.py
rename to examples/foundational/26d-gemini-live-text.py
index 05b792c00..42387f76d 100644
--- a/examples/foundational/26d-gemini-multimodal-live-text.py
+++ b/examples/foundational/26d-gemini-live-text.py
@@ -80,7 +80,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
- # KNOWN ISSUE: If using GeminiVertexMultimodalLiveLLMService, it appears
+ # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears
# you cannot specify a modality other than AUDIO.
llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-live-google-search.py
similarity index 100%
rename from examples/foundational/26e-gemini-multimodal-google-search.py
rename to examples/foundational/26e-gemini-live-google-search.py
diff --git a/examples/foundational/26f-gemini-multimodal-live-files-api.py b/examples/foundational/26f-gemini-live-files-api.py
similarity index 100%
rename from examples/foundational/26f-gemini-multimodal-live-files-api.py
rename to examples/foundational/26f-gemini-live-files-api.py
diff --git a/examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py b/examples/foundational/26g-gemini-live-groundingMetadata.py
similarity index 100%
rename from examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
rename to examples/foundational/26g-gemini-live-groundingMetadata.py
diff --git a/examples/foundational/26h-gemini-live-vertex-function-calling.py b/examples/foundational/26h-gemini-live-vertex-function-calling.py
new file mode 100644
index 000000000..c0344a052
--- /dev/null
+++ b/examples/foundational/26h-gemini-live-vertex-function-calling.py
@@ -0,0 +1,191 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import os
+from datetime import datetime
+
+from dotenv import load_dotenv
+from google.genai.types import HttpOptions
+from loguru import logger
+
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import LLMRunFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.runner.types import RunnerArguments
+from pipecat.runner.utils import create_transport
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService
+from pipecat.services.llm_service import FunctionCallParams
+from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.transports.daily.transport import DailyParams
+from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
+
+load_dotenv(override=True)
+
+
+async def fetch_weather_from_api(params: FunctionCallParams):
+ temperature = 75 if params.arguments["format"] == "fahrenheit" else 24
+ await params.result_callback(
+ {
+ "conditions": "nice",
+ "temperature": temperature,
+ "format": params.arguments["format"],
+ "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"),
+ }
+ )
+
+
+async def fetch_restaurant_recommendation(params: FunctionCallParams):
+ await params.result_callback({"name": "The Golden Dragon"})
+
+
+system_instruction = """
+You are a helpful assistant who can answer questions and use tools.
+
+You have three tools available to you:
+1. get_current_weather: Use this tool to get the current weather in a specific location.
+2. get_restaurant_recommendation: Use this tool to get a restaurant recommendation in a specific location.
+"""
+
+
+# We store functions so objects (e.g. SileroVADAnalyzer) don't get
+# instantiated. The function will be called when the desired transport gets
+# selected.
+transport_params = {
+ "daily": lambda: DailyParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ # set stop_secs to something roughly similar to the internal setting
+ # of the Multimodal Live api, just to align events. This doesn't really
+ # matter because we can only use the Multimodal Live API's phrase
+ # endpointing, for now.
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
+ ),
+ "twilio": lambda: FastAPIWebsocketParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ # set stop_secs to something roughly similar to the internal setting
+ # of the Multimodal Live api, just to align events. This doesn't really
+ # matter because we can only use the Multimodal Live API's phrase
+ # endpointing, for now.
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
+ ),
+ "webrtc": lambda: TransportParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ # set stop_secs to something roughly similar to the internal setting
+ # of the Multimodal Live api, just to align events. This doesn't really
+ # matter because we can only use the Multimodal Live API's phrase
+ # endpointing, for now.
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
+ ),
+}
+
+
+async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
+ logger.info(f"Starting bot")
+
+ weather_function = FunctionSchema(
+ name="get_current_weather",
+ description="Get the current weather",
+ properties={
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA",
+ },
+ "format": {
+ "type": "string",
+ "enum": ["celsius", "fahrenheit"],
+ "description": "The temperature unit to use. Infer this from the user's location.",
+ },
+ },
+ required=["location", "format"],
+ )
+ restaurant_function = FunctionSchema(
+ name="get_restaurant_recommendation",
+ description="Get a restaurant recommendation",
+ properties={
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA",
+ },
+ },
+ required=["location"],
+ )
+ # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears
+ # you cannot use the "google_search" tool alongside other tools.
+ # See https://github.com/googleapis/python-genai/issues/941.
+ tools = ToolsSchema(standard_tools=[weather_function, restaurant_function])
+
+ llm = GeminiLiveVertexLLMService(
+ credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"),
+ project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
+ location=os.getenv("GOOGLE_CLOUD_LOCATION"),
+ system_instruction=system_instruction,
+ voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
+ tools=tools,
+ )
+
+ llm.register_function("get_current_weather", fetch_weather_from_api)
+ llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
+
+ context = OpenAILLMContext(
+ [{"role": "user", "content": "Say hello."}],
+ )
+ context_aggregator = llm.create_context_aggregator(context)
+
+ pipeline = Pipeline(
+ [
+ transport.input(),
+ context_aggregator.user(),
+ llm,
+ transport.output(),
+ context_aggregator.assistant(),
+ ]
+ )
+
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ enable_metrics=True,
+ enable_usage_metrics=True,
+ ),
+ idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
+ )
+
+ @transport.event_handler("on_client_connected")
+ async def on_client_connected(transport, client):
+ logger.info(f"Client connected")
+ # Kick off the conversation.
+ await task.queue_frames([LLMRunFrame()])
+
+ @transport.event_handler("on_client_disconnected")
+ async def on_client_disconnected(transport, client):
+ logger.info(f"Client disconnected")
+ await task.cancel()
+
+ runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
+
+ await runner.run(task)
+
+
+async def bot(runner_args: RunnerArguments):
+ """Main bot entry point compatible with Pipecat Cloud."""
+ transport = await create_transport(runner_args, transport_params)
+ await run_bot(transport, runner_args)
+
+
+if __name__ == "__main__":
+ from pipecat.runner.run import main
+
+ main()
diff --git a/examples/foundational/26h-gemini-multimodal-live-vertex.py b/examples/foundational/26h-gemini-multimodal-live-vertex.py
deleted file mode 100644
index 79bfac531..000000000
--- a/examples/foundational/26h-gemini-multimodal-live-vertex.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#
-# Copyright (c) 2024–2025, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import os
-
-from dotenv import load_dotenv
-from loguru import logger
-
-from pipecat.audio.vad.silero import SileroVADAnalyzer
-from pipecat.audio.vad.vad_analyzer import VADParams
-from pipecat.frames.frames import LLMMessagesAppendFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.runner.types import RunnerArguments
-from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService
-from pipecat.transports.base_transport import BaseTransport, TransportParams
-from pipecat.transports.daily.transport import DailyParams
-from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
-
-# Load environment variables
-load_dotenv(override=True)
-
-
-# We store functions so objects (e.g. SileroVADAnalyzer) don't get
-# instantiated. The function will be called when the desired transport gets
-# selected.
-transport_params = {
- "daily": lambda: DailyParams(
- audio_in_enabled=True,
- audio_out_enabled=True,
- # set stop_secs to something roughly similar to the internal setting
- # of the Multimodal Live api, just to align events.
- vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
- ),
- "twilio": lambda: FastAPIWebsocketParams(
- audio_in_enabled=True,
- audio_out_enabled=True,
- # set stop_secs to something roughly similar to the internal setting
- # of the Multimodal Live api, just to align events.
- vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
- ),
- "webrtc": lambda: TransportParams(
- audio_in_enabled=True,
- audio_out_enabled=True,
- # set stop_secs to something roughly similar to the internal setting
- # of the Multimodal Live api, just to align events.
- vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
- ),
-}
-
-
-async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
- logger.info(f"Starting bot")
-
- # Create the Gemini Vertex Multimodal Live LLM service
- system_instruction = f"""
- You are a helpful AI assistant.
- Your goal is to demonstrate your capabilities in a helpful and engaging way.
- Your output will be converted to audio so don't include special characters in your answers.
- Respond to what the user said in a creative and helpful way.
- """
-
- llm = GeminiLiveVertexLLMService(
- credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"),
- project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
- location=os.getenv("GOOGLE_CLOUD_LOCATION"),
- system_instruction=system_instruction,
- voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
- )
-
- # Build the pipeline
- pipeline = Pipeline(
- [
- transport.input(),
- llm,
- transport.output(),
- ]
- )
-
- # Configure the pipeline task
- task = PipelineTask(
- pipeline,
- params=PipelineParams(
- enable_metrics=True,
- enable_usage_metrics=True,
- ),
- idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
- )
-
- # Handle client connection event
- @transport.event_handler("on_client_connected")
- async def on_client_connected(transport, client):
- logger.info(f"Client connected")
- # Kick off the conversation.
- await task.queue_frames(
- [
- LLMMessagesAppendFrame(
- messages=[
- {
- "role": "user",
- "content": f"Greet the user and introduce yourself.",
- }
- ]
- )
- ]
- )
-
- # Handle client disconnection events
- @transport.event_handler("on_client_disconnected")
- async def on_client_disconnected(transport, client):
- logger.info(f"Client disconnected")
- await task.cancel()
-
- # Run the pipeline
- runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
- await runner.run(task)
-
-
-async def bot(runner_args: RunnerArguments):
- """Main bot entry point compatible with Pipecat Cloud."""
- transport = await create_transport(runner_args, transport_params)
- await run_bot(transport, runner_args)
-
-
-if __name__ == "__main__":
- from pipecat.runner.run import main
-
- main()
diff --git a/examples/foundational/26i-gemini-multimodal-live-graceful-end.py b/examples/foundational/26i-gemini-live-graceful-end.py
similarity index 100%
rename from examples/foundational/26i-gemini-multimodal-live-graceful-end.py
rename to examples/foundational/26i-gemini-live-graceful-end.py
diff --git a/examples/foundational/README.md b/examples/foundational/README.md
index 9a6c26005..c1ead3ece 100644
--- a/examples/foundational/README.md
+++ b/examples/foundational/README.md
@@ -105,7 +105,7 @@ uv run 07-interruptible.py -t twilio -x NGROK_HOST_NAME
### Vision & Multimodal
- **[12a-describe-video-gemini-flash.py](./12a-describe-video-gemini-flash.py)**: Bot describes user's video (Video input, Multimodal LLMs)
-- **[26c-gemini-multimodal-live-video.py](./26c-gemini-multimodal-live-video.py)**: Gemini with video input (Streaming video, Function calls)
+- **[26c-gemini-live-video.py](./26c-gemini-live-video.py)**: Gemini with video input (Streaming video, Function calls)
### Voice & Language
diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py
index 206cf848e..44cb69aed 100644
--- a/scripts/evals/run-release-evals.py
+++ b/scripts/evals/run-release-evals.py
@@ -147,7 +147,10 @@ TESTS_15 = [
]
TESTS_19 = [
+ ("19-openai-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST),
("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST),
+ # OpenAI Realtime not released on Azure yet
+ # ("19a-azure-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST),
("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST),
("19b-openai-realtime-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST),
("19b-openai-realtime-beta-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST),
@@ -160,18 +163,18 @@ TESTS_21 = [
TESTS_26 = [
("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST),
(
- "26a-gemini-multimodal-live-transcription.py",
+ "26a-gemini-live-transcription.py",
PROMPT_SIMPLE_MATH,
EVAL_SIMPLE_MATH,
BOT_SPEAKS_FIRST,
),
(
- "26b-gemini-multimodal-live-function-calling.py",
+ "26b-gemini-live-function-calling.py",
PROMPT_WEATHER,
EVAL_WEATHER,
BOT_SPEAKS_FIRST,
),
- ("26c-gemini-multimodal-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST),
+ ("26c-gemini-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST),
(
"26e-gemini-multimodal-google-search.py",
PROMPT_ONLINE_SEARCH,
@@ -179,7 +182,13 @@ TESTS_26 = [
BOT_SPEAKS_FIRST,
),
# Currently not working.
- # ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST),
+ # ("26d-gemini-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST),
+ (
+ "26h-gemini-live-vertex-function-calling.py",
+ PROMPT_WEATHER,
+ EVAL_WEATHER,
+ BOT_SPEAKS_FIRST,
+ ),
]
TESTS_27 = [
From 21d610cd30b298c473f793ded5fc76af2bef4511 Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Fri, 10 Oct 2025 12:43:31 -0400
Subject: [PATCH 30/51] Docs fixes for 0.0.90 release
---
docs/api/conf.py | 1 +
src/pipecat/services/hume/tts.py | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/docs/api/conf.py b/docs/api/conf.py
index a3062b612..8946a34d3 100644
--- a/docs/api/conf.py
+++ b/docs/api/conf.py
@@ -50,6 +50,7 @@ autodoc_mock_imports = [
# Krisp - has build issues on some platforms
"pipecat_ai_krisp",
"krisp",
+ "krisp_audio",
# System-specific GUI libraries
"_tkinter",
"tkinter",
diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py
index 0629a8909..2701c5d05 100644
--- a/src/pipecat/services/hume/tts.py
+++ b/src/pipecat/services/hume/tts.py
@@ -42,7 +42,7 @@ class HumeTTSService(TTSService):
"""Hume Octave Text-to-Speech service.
Streams PCM audio via Hume's HTTP output streaming (JSON chunks) endpoint
- using the Python SDK and emits `TTSAudioRawFrame`s suitable for Pipecat transports.
+ using the Python SDK and emits ``TTSAudioRawFrame`` frames suitable for Pipecat transports.
Supported features:
@@ -78,7 +78,7 @@ class HumeTTSService(TTSService):
Args:
api_key: Hume API key. If omitted, reads the ``HUME_API_KEY`` environment variable.
- voice_id: ID of the voice to use (ID-only; names are not supported here).
+ voice_id: ID of the voice to use. Only voice IDs are supported; voice names are not.
params: Optional synthesis controls (acting instructions, speed, trailing silence).
sample_rate: Output sample rate for emitted PCM frames. Defaults to 48_000 (Hume).
**kwargs: Additional arguments passed to the parent class.
From 1c25b6fb72c079fae5ecfb186516a3ab157dbf0a Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Fri, 10 Oct 2025 12:56:37 -0400
Subject: [PATCH 31/51] `location` should not be optional when using Google
Vertex.
Also, update `GoogleVertexLLMService` initialization pattern in the example file.
---
.../foundational/14p-function-calling-gemini-vertex-ai.py | 5 ++---
src/pipecat/services/google/gemini_live/llm_vertex.py | 2 +-
src/pipecat/services/google/llm_vertex.py | 4 ++++
3 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py
index f9ad264eb..cba5eee60 100644
--- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py
+++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py
@@ -76,9 +76,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm = GoogleVertexLLMService(
credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"),
- params=GoogleVertexLLMService.InputParams(
- project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
- ),
+ project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
+ location=os.getenv("GOOGLE_CLOUD_LOCATION"),
)
# You can aslo register a function_name of None to get all functions
# sent to the same callback with an additional function_name parameter.
diff --git a/src/pipecat/services/google/gemini_live/llm_vertex.py b/src/pipecat/services/google/gemini_live/llm_vertex.py
index 4f1b57f4c..a38154755 100644
--- a/src/pipecat/services/google/gemini_live/llm_vertex.py
+++ b/src/pipecat/services/google/gemini_live/llm_vertex.py
@@ -49,7 +49,7 @@ class GeminiLiveVertexLLMService(GeminiLiveLLMService):
*,
credentials: Optional[str] = None,
credentials_path: Optional[str] = None,
- location: str = "us-east4",
+ location: str,
project_id: str,
model="google/gemini-2.0-flash-live-preview-04-09",
voice_id: str = "Charon",
diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py
index cc0b8654e..49adb2e9b 100644
--- a/src/pipecat/services/google/llm_vertex.py
+++ b/src/pipecat/services/google/llm_vertex.py
@@ -137,6 +137,10 @@ class GoogleVertexLLMService(OpenAILLMService):
if project_id is None:
raise ValueError("project_id is required")
if location is None:
+ # If location is not provided, default to "us-east4".
+ # Note: this is legacy behavior; ideally location would be
+ # required.
+ logger.warning("location is not provided. Defaulting to 'us-east4'.")
location = "us-east4" # Default location if not provided
base_url = self._get_base_url(location, project_id)
From 55014bdd771fe703efa112ccddc3f833f713b856 Mon Sep 17 00:00:00 2001
From: Paul Kompfner
Date: Fri, 10 Oct 2025 13:18:03 -0400
Subject: [PATCH 32/51] Update a Google Vertex disclaimer for accuracy
---
examples/foundational/26d-gemini-live-text.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/examples/foundational/26d-gemini-live-text.py b/examples/foundational/26d-gemini-live-text.py
index 42387f76d..062c0231b 100644
--- a/examples/foundational/26d-gemini-live-text.py
+++ b/examples/foundational/26d-gemini-live-text.py
@@ -80,8 +80,10 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
- # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears
- # you cannot specify a modality other than AUDIO.
+ # KNOWN ISSUE: If using GeminiLiveVertexLLMService, you cannot specify a
+ # modality other than AUDIO (at least not if using the service's default
+ # model, which is a native audio model:
+ # https://cloud.google.com/vertex-ai/generative-ai/docs/live-api/tools#native-audio).
llm = GeminiLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=SYSTEM_INSTRUCTION,
From 502e7e42a7b2cc5af93bc29402435153121fa755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Fri, 10 Oct 2025 08:33:21 -0700
Subject: [PATCH 33/51] update CHANGELOG for 0.0.90
---
CHANGELOG.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 70324c65e..e92361569 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased]
+## [0.0.90] - 2025-10-10
### Added
From 0473556992f7142fc864b4c44a6455122b59d609 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Fri, 10 Oct 2025 12:09:20 -0700
Subject: [PATCH 34/51] tts: fix RimeHttpTTSService/PiperTTSService 16-bit
audio frames alignment
---
CHANGELOG.md | 8 ++++++
src/pipecat/services/piper/tts.py | 16 +++++------
src/pipecat/services/rime/tts.py | 14 +++++-----
src/pipecat/services/tts_service.py | 42 ++++++++++++++++++++++++++++-
4 files changed, 62 insertions(+), 18 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e92361569..797ca06ab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,14 @@ All notable changes to **Pipecat** will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [Unreleased]
+
+### Fixed
+
+- Fixed an issue where `RimeHttpTTSService` and `PiperTTSService` could generate
+ incorrectly 16-bit aligned audio frames, potentially leading to internal
+ errors or static audio.
+
## [0.0.90] - 2025-10-10
### Added
diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py
index d5b663c77..fa43a720c 100644
--- a/src/pipecat/services/piper/tts.py
+++ b/src/pipecat/services/piper/tts.py
@@ -14,7 +14,6 @@ from loguru import logger
from pipecat.frames.frames import (
ErrorFrame,
Frame,
- TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
@@ -99,16 +98,15 @@ class PiperTTSService(TTSService):
await self.start_tts_usage_metrics(text)
+ yield TTSStartedFrame()
+
CHUNK_SIZE = self.chunk_size
- yield TTSStartedFrame()
- async for chunk in response.content.iter_chunked(CHUNK_SIZE):
- # remove wav header if present
- if chunk.startswith(b"RIFF"):
- chunk = chunk[44:]
- if len(chunk) > 0:
- await self.stop_ttfb_metrics()
- yield TTSAudioRawFrame(chunk, self.sample_rate, 1)
+ async for frame in self._stream_audio_frames_from_iterator(
+ response.content.iter_chunked(CHUNK_SIZE), strip_wav_header=True
+ ):
+ await self.stop_ttfb_metrics()
+ yield frame
except Exception as e:
logger.error(f"Error in run_tts: {e}")
yield ErrorFrame(error=str(e))
diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py
index 917716545..1ac829ebd 100644
--- a/src/pipecat/services/rime/tts.py
+++ b/src/pipecat/services/rime/tts.py
@@ -553,15 +553,13 @@ class RimeHttpTTSService(TTSService):
CHUNK_SIZE = self.chunk_size
- async for chunk in response.content.iter_chunked(CHUNK_SIZE):
- if need_to_strip_wav_header and chunk.startswith(b"RIFF"):
- chunk = chunk[44:]
- need_to_strip_wav_header = False
+ async for frame in self._stream_audio_frames_from_iterator(
+ response.content.iter_chunked(CHUNK_SIZE),
+ strip_wav_header=need_to_strip_wav_header,
+ ):
+ await self.stop_ttfb_metrics()
+ yield frame
- if len(chunk) > 0:
- await self.stop_ttfb_metrics()
- frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
- yield frame
except Exception as e:
logger.exception(f"Error generating TTS: {e}")
yield ErrorFrame(error=f"Rime TTS error: {str(e)}")
diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py
index 02b80b609..a60b50818 100644
--- a/src/pipecat/services/tts_service.py
+++ b/src/pipecat/services/tts_service.py
@@ -8,7 +8,17 @@
import asyncio
from abc import abstractmethod
-from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple
+from typing import (
+ Any,
+ AsyncGenerator,
+ AsyncIterator,
+ Dict,
+ List,
+ Mapping,
+ Optional,
+ Sequence,
+ Tuple,
+)
from loguru import logger
@@ -374,6 +384,36 @@ class TTSService(AIService):
):
await self._stop_frame_queue.put(frame)
+ async def _stream_audio_frames_from_iterator(
+ self, iterator: AsyncIterator[bytes], *, strip_wav_header: bool
+ ) -> AsyncGenerator[Frame, None]:
+ buffer = bytearray()
+ need_to_strip_wav_header = strip_wav_header
+ async for chunk in iterator:
+ if need_to_strip_wav_header and chunk.startswith(b"RIFF"):
+ chunk = chunk[44:]
+ need_to_strip_wav_header = False
+
+ # Append to current buffer.
+ buffer.extend(chunk)
+
+ # Round to nearest even number.
+ aligned_length = len(buffer) & ~1 # 111111111...11110
+ if aligned_length > 0:
+ aligned_chunk = buffer[:aligned_length]
+ buffer = buffer[aligned_length:] # keep any leftover byte
+
+ if len(aligned_chunk) > 0:
+ frame = TTSAudioRawFrame(bytes(aligned_chunk), self.sample_rate, 1)
+ yield frame
+
+ if len(buffer) > 0:
+ # Make sure we don't need an extra padding byte.
+ if len(buffer) % 2 == 1:
+ buffer.extend(b"\x00")
+ frame = TTSAudioRawFrame(bytes(buffer), self.sample_rate, 1)
+ yield frame
+
async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
self._processing_text = False
await self._text_aggregator.handle_interruption()
From 16e9093d5a25a435eda743e542771a4475c1a1d5 Mon Sep 17 00:00:00 2001
From: makosst <191634637+makosst@users.noreply.github.com>
Date: Sat, 11 Oct 2025 09:20:17 -0700
Subject: [PATCH 35/51] Added Manta Graph to README
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 3eb9bf346..50cc72791 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,7 @@
[](https://pypi.org/project/pipecat-ai)  [](https://codecov.io/gh/pipecat-ai/pipecat) [](https://docs.pipecat.ai) [](https://discord.gg/pipecat) [](https://deepwiki.com/pipecat-ai/pipecat)
+[](https://getmanta.ai/pipecat)
# 🎙️ Pipecat: Real-Time Voice & Multimodal AI Agents
From 21d8d148b8adebfaaf5986fc7058a85aa4b8b4ec Mon Sep 17 00:00:00 2001
From: Pyae Sone Myo <66440980+Rickaym@users.noreply.github.com>
Date: Sun, 12 Oct 2025 22:10:11 +0630
Subject: [PATCH 36/51] fix: handle partial words across alignment chunks
gracefully
---
src/pipecat/services/elevenlabs/tts.py | 60 ++++++++++++++++----------
1 file changed, 37 insertions(+), 23 deletions(-)
diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py
index 3d6d5bd4c..4c91c4f38 100644
--- a/src/pipecat/services/elevenlabs/tts.py
+++ b/src/pipecat/services/elevenlabs/tts.py
@@ -168,16 +168,24 @@ def build_elevenlabs_voice_settings(
def calculate_word_times(
- alignment_info: Mapping[str, Any], cumulative_time: float
-) -> List[Tuple[str, float]]:
+ alignment_info: Mapping[str, Any],
+ cumulative_time: float,
+ partial_word: str = "",
+ partial_word_start_time: float = 0.0,
+) -> tuple[List[Tuple[str, float]], str, float]:
"""Calculate word timestamps from character alignment information.
Args:
alignment_info: Character alignment data from ElevenLabs API.
cumulative_time: Base time offset for this chunk.
+ partial_word: Partial word carried over from previous chunk.
+ partial_word_start_time: Start time of the partial word.
Returns:
- List of (word, timestamp) tuples.
+ Tuple of (word_times, new_partial_word, new_partial_word_start_time):
+ - word_times: List of (word, timestamp) tuples for complete words
+ - new_partial_word: Incomplete word at end of chunk (empty if chunk ends with space)
+ - new_partial_word_start_time: Start time of the incomplete word
"""
chars = alignment_info["chars"]
char_start_times_ms = alignment_info["charStartTimesMs"]
@@ -186,41 +194,37 @@ def calculate_word_times(
logger.error(
f"calculate_word_times: length mismatch - chars={len(chars)}, times={len(char_start_times_ms)}"
)
- return []
+ return ([], partial_word, partial_word_start_time)
# Build words and track their start positions
words = []
- word_start_indices = []
- current_word = ""
- word_start_index = None
+ word_start_times = []
+ current_word = partial_word # Start with any partial word from previous chunk
+ word_start_time = partial_word_start_time if partial_word else None
for i, char in enumerate(chars):
if char == " ":
# End of current word
if current_word: # Only add non-empty words
words.append(current_word)
- word_start_indices.append(word_start_index)
+ word_start_times.append(word_start_time)
current_word = ""
- word_start_index = None
+ word_start_time = None
else:
# Building a word
- if word_start_index is None: # First character of new word
- word_start_index = i
+ if word_start_time is None: # First character of new word
+ # Convert from milliseconds to seconds and add cumulative offset
+ word_start_time = cumulative_time + (char_start_times_ms[i] / 1000.0)
current_word += char
- # Handle the last word if there's no trailing space
- if current_word and word_start_index is not None:
- words.append(current_word)
- word_start_indices.append(word_start_index)
+ # Build result for complete words
+ word_times = list(zip(words, word_start_times))
- # Calculate timestamps for each word
- word_times = []
- for word, start_idx in zip(words, word_start_indices):
- # Convert from milliseconds to seconds and add cumulative offset
- start_time_seconds = cumulative_time + (char_start_times_ms[start_idx] / 1000.0)
- word_times.append((word, start_time_seconds))
+ # Return any incomplete word at the end of this chunk
+ new_partial_word = current_word if current_word else ""
+ new_partial_word_start_time = word_start_time if word_start_time is not None else 0.0
- return word_times
+ return (word_times, new_partial_word, new_partial_word_start_time)
class ElevenLabsTTSService(AudioContextWordTTSService):
@@ -332,6 +336,9 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
# there's an interruption or TTSStoppedFrame.
self._started = False
self._cumulative_time = 0
+ # Track partial words that span across alignment chunks
+ self._partial_word = ""
+ self._partial_word_start_time = 0.0
# Context management for v1 multi API
self._context_id = None
@@ -609,7 +616,12 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
if msg.get("alignment"):
alignment = msg["alignment"]
- word_times = calculate_word_times(alignment, self._cumulative_time)
+ word_times, self._partial_word, self._partial_word_start_time = calculate_word_times(
+ alignment,
+ self._cumulative_time,
+ self._partial_word,
+ self._partial_word_start_time,
+ )
if word_times:
await self.add_word_timestamps(word_times)
@@ -683,6 +695,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
yield TTSStartedFrame()
self._started = True
self._cumulative_time = 0
+ self._partial_word = ""
+ self._partial_word_start_time = 0.0
# If a context ID does not exist, create a new one and
# register it. If an ID exists, that means the Pipeline is
# configured for allow_interruptions=False, so continue
From d9580f72a92601fd09e9b4a8f9b9e507a5b40f4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Mon, 13 Oct 2025 18:29:19 -0700
Subject: [PATCH 37/51] runner: allow subdirectories in --folder
---
CHANGELOG.md | 5 +++++
src/pipecat/runner/run.py | 2 +-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 797ca06ab..424d6a420 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Added
+
+- The runner `--folder` argument now supports downloading files from
+ subdirectories.
+
### Fixed
- Fixed an issue where `RimeHttpTTSService` and `PiperTTSService` could generate
diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py
index fd1a8a952..53b39f4e3 100644
--- a/src/pipecat/runner/run.py
+++ b/src/pipecat/runner/run.py
@@ -217,7 +217,7 @@ def _setup_webrtc_routes(
"""Redirect root requests to client interface."""
return RedirectResponse(url="/client/")
- @app.get("/files/{filename}")
+ @app.get("/files/{filename:path}")
async def download_file(filename: str):
"""Handle file downloads."""
if not folder:
From 3b751322d3d18832b4fca299624eae710c7ed2af Mon Sep 17 00:00:00 2001
From: Pyae Sone Myo <66440980+Rickaym@users.noreply.github.com>
Date: Tue, 14 Oct 2025 23:04:09 +0630
Subject: [PATCH 38/51] fix: add interruption reset for partial word states
---
src/pipecat/services/elevenlabs/tts.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py
index 4c91c4f38..9e4154e4d 100644
--- a/src/pipecat/services/elevenlabs/tts.py
+++ b/src/pipecat/services/elevenlabs/tts.py
@@ -577,6 +577,8 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
logger.error(f"Error closing context on interruption: {e}")
self._context_id = None
self._started = False
+ self._partial_word = ""
+ self._partial_word_start_time = 0.0
async def _receive_messages(self):
"""Handle incoming WebSocket messages from ElevenLabs."""
@@ -616,11 +618,13 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
if msg.get("alignment"):
alignment = msg["alignment"]
- word_times, self._partial_word, self._partial_word_start_time = calculate_word_times(
- alignment,
- self._cumulative_time,
- self._partial_word,
- self._partial_word_start_time,
+ word_times, self._partial_word, self._partial_word_start_time = (
+ calculate_word_times(
+ alignment,
+ self._cumulative_time,
+ self._partial_word,
+ self._partial_word_start_time,
+ )
)
if word_times:
From b6b09975534b9f645b35a809f602aa6f1358a15b Mon Sep 17 00:00:00 2001
From: Pyae Sone Myo <66440980+Rickaym@users.noreply.github.com>
Date: Tue, 14 Oct 2025 23:06:13 +0630
Subject: [PATCH 39/51] fix: add support for partial words
---
src/pipecat/services/elevenlabs/tts.py | 36 +++++++++++++++-----------
1 file changed, 21 insertions(+), 15 deletions(-)
diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py
index 9e4154e4d..1544e94be 100644
--- a/src/pipecat/services/elevenlabs/tts.py
+++ b/src/pipecat/services/elevenlabs/tts.py
@@ -827,6 +827,10 @@ class ElevenLabsHttpTTSService(WordTTSService):
# Store previous text for context within a turn
self._previous_text = ""
+ # Track partial words that span across alignment chunks
+ self._partial_word = ""
+ self._partial_word_start_time = 0.0
+
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language to ElevenLabs language code.
@@ -854,6 +858,8 @@ class ElevenLabsHttpTTSService(WordTTSService):
self._cumulative_time = 0
self._started = False
self._previous_text = ""
+ self._partial_word = ""
+ self._partial_word_start_time = 0.0
logger.debug(f"{self}: Reset internal state")
async def start(self, frame: StartFrame):
@@ -888,11 +894,13 @@ class ElevenLabsHttpTTSService(WordTTSService):
def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]:
"""Calculate word timing from character alignment data.
+ This method handles partial words that may span across multiple alignment chunks.
+
Args:
alignment_info: Character timing data from ElevenLabs.
Returns:
- List of (word, timestamp) pairs.
+ List of (word, timestamp) pairs for complete words in this chunk.
Example input data::
@@ -918,30 +926,28 @@ class ElevenLabsHttpTTSService(WordTTSService):
# Build the words and find their start times
words = []
word_start_times = []
- current_word = ""
- first_char_idx = -1
+ # Start with any partial word from previous chunk
+ current_word = self._partial_word
+ word_start_time = self._partial_word_start_time if self._partial_word else None
for i, char in enumerate(chars):
if char == " ":
if current_word: # Only add non-empty words
words.append(current_word)
- # Use time of the first character of the word, offset by cumulative time
- word_start_times.append(
- self._cumulative_time + char_start_times[first_char_idx]
- )
+ word_start_times.append(word_start_time)
current_word = ""
- first_char_idx = -1
+ word_start_time = None
else:
- if not current_word: # This is the first character of a new word
- first_char_idx = i
+ if word_start_time is None: # First character of a new word
+ # Use time of the first character of the word, offset by cumulative time
+ word_start_time = self._cumulative_time + char_start_times[i]
current_word += char
- # Don't forget the last word if there's no trailing space
- if current_word and first_char_idx >= 0:
- words.append(current_word)
- word_start_times.append(self._cumulative_time + char_start_times[first_char_idx])
+ # Store any incomplete word at the end of this chunk
+ self._partial_word = current_word if current_word else ""
+ self._partial_word_start_time = word_start_time if word_start_time is not None else 0.0
- # Create word-time pairs
+ # Create word-time pairs for complete words only
word_times = list(zip(words, word_start_times))
return word_times
From be2858bfbb3df8b814bd4d73ed7316e1f35e4aa9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Tue, 14 Oct 2025 14:09:45 -0700
Subject: [PATCH 40/51] CartesiaSTTService: inherit from WebsocketSTTService
---
CHANGELOG.md | 4 +
src/pipecat/services/cartesia/stt.py | 145 ++++++++++++++-------------
src/pipecat/services/cartesia/tts.py | 2 +-
3 files changed, 80 insertions(+), 71 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 424d6a420..dd3bfa3c6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- The runner `--folder` argument now supports downloading files from
subdirectories.
+### Changed
+
+- `CartesiaSTTService` now inherits from `WebsocketSTTService`.
+
### Fixed
- Fixed an issue where `RimeHttpTTSService` and `PiperTTSService` could generate
diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py
index 5412c422c..97a4f7127 100644
--- a/src/pipecat/services/cartesia/stt.py
+++ b/src/pipecat/services/cartesia/stt.py
@@ -28,13 +28,12 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.stt_service import STTService
+from pipecat.services.stt_service import WebsocketSTTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_stt
try:
- import websockets
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
except ModuleNotFoundError as e:
@@ -124,7 +123,7 @@ class CartesiaLiveOptions:
return cls(**json.loads(json_str))
-class CartesiaSTTService(STTService):
+class CartesiaSTTService(WebsocketSTTService):
"""Speech-to-text service using Cartesia Live API.
Provides real-time speech transcription through WebSocket connection
@@ -176,8 +175,7 @@ class CartesiaSTTService(STTService):
self.set_model_name(merged_options.model)
self._api_key = api_key
self._base_url = base_url or "api.cartesia.ai"
- self._connection = None
- self._receiver_task = None
+ self._receive_task = None
def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
@@ -214,6 +212,27 @@ class CartesiaSTTService(STTService):
await super().cancel(frame)
await self._disconnect()
+ async def start_metrics(self):
+ """Start performance metrics collection for transcription processing."""
+ await self.start_ttfb_metrics()
+ await self.start_processing_metrics()
+
+ async def process_frame(self, frame: Frame, direction: FrameDirection):
+ """Process incoming frames and handle speech events.
+
+ Args:
+ frame: The frame to process.
+ direction: Direction of frame flow in the pipeline.
+ """
+ await super().process_frame(frame, direction)
+
+ if isinstance(frame, UserStartedSpeakingFrame):
+ await self.start_metrics()
+ elif isinstance(frame, UserStoppedSpeakingFrame):
+ # Send finalize command to flush the transcription session
+ if self._websocket and self._websocket.state is State.OPEN:
+ await self._websocket.send("finalize")
+
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text transcription.
@@ -224,45 +243,69 @@ class CartesiaSTTService(STTService):
None - transcription results are handled via WebSocket responses.
"""
# If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again
- if not self._connection or self._connection.state is State.CLOSED:
+ if not self._websocket or self._websocket.state is State.CLOSED:
await self._connect()
- await self._connection.send(audio)
+ await self._websocket.send(audio)
yield None
async def _connect(self):
- params = self._settings.to_dict()
- ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}"
- logger.debug(f"Connecting to Cartesia: {ws_url}")
- headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key}
+ await self._connect_websocket()
+ if self._websocket and not self._receive_task:
+ self._receive_task = asyncio.create_task(self._receive_task_handler(self._report_error))
+
+ async def _disconnect(self):
+ if self._receive_task:
+ await self.cancel_task(self._receive_task)
+ self._receive_task = None
+
+ await self._disconnect_websocket()
+
+ async def _connect_websocket(self):
try:
- self._connection = await websocket_connect(ws_url, additional_headers=headers)
- # Setup the receiver task to handle the incoming messages from the Cartesia server
- if self._receiver_task is None or self._receiver_task.done():
- self._receiver_task = asyncio.create_task(self._receive_messages())
- logger.debug(f"Connected to Cartesia")
+ if self._websocket and self._websocket.state is State.OPEN:
+ return
+ logger.debug("Connecting to Cartesia STT")
+
+ params = self._settings.to_dict()
+ ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}"
+ headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key}
+
+ self._websocket = await websocket_connect(ws_url, additional_headers=headers)
except Exception as e:
logger.error(f"{self}: unable to connect to Cartesia: {e}")
- async def _receive_messages(self):
+ async def _disconnect_websocket(self):
try:
- while True:
- if not self._connection or self._connection.state is State.CLOSED:
- break
-
- message = await self._connection.recv()
- try:
- data = json.loads(message)
- await self._process_response(data)
- except json.JSONDecodeError:
- logger.warning(f"Received non-JSON message: {message}")
- except asyncio.CancelledError:
- pass
- except websockets.exceptions.ConnectionClosed as e:
- logger.debug(f"WebSocket connection closed: {e}")
+ if self._websocket and self._websocket.state is State.OPEN:
+ logger.debug("Disconnecting from Cartesia STT")
+ await self._websocket.close()
except Exception as e:
- logger.error(f"Error in message receiver: {e}")
+ logger.error(f"{self} error closing websocket: {e}")
+ finally:
+ self._websocket = None
+
+ def _get_websocket(self):
+ if self._websocket:
+ return self._websocket
+ raise Exception("Websocket not connected")
+
+ async def _process_messages(self):
+ async for message in self._get_websocket():
+ try:
+ data = json.loads(message)
+ await self._process_response(data)
+ except json.JSONDecodeError:
+ logger.warning(f"Received non-JSON message: {message}")
+
+ async def _receive_messages(self):
+ while True:
+ await self._process_messages()
+ # Cartesia times out after 5 minutes of innactivity (no keepalive
+ # mechanism is available). So, we try to reconnect.
+ logger.debug(f"{self} Cartesia connection was disconnected (timeout?), reconnecting")
+ await self._connect_websocket()
async def _process_response(self, data):
if "type" in data:
@@ -316,41 +359,3 @@ class CartesiaSTTService(STTService):
language,
)
)
-
- async def _disconnect(self):
- if self._receiver_task:
- self._receiver_task.cancel()
- try:
- await self._receiver_task
- except asyncio.CancelledError:
- pass
- except Exception as e:
- logger.exception(f"Unexpected exception while cancelling task: {e}")
- self._receiver_task = None
-
- if self._connection and self._connection.state is State.OPEN:
- logger.debug("Disconnecting from Cartesia")
-
- await self._connection.close()
- self._connection = None
-
- async def start_metrics(self):
- """Start performance metrics collection for transcription processing."""
- await self.start_ttfb_metrics()
- await self.start_processing_metrics()
-
- async def process_frame(self, frame: Frame, direction: FrameDirection):
- """Process incoming frames and handle speech events.
-
- Args:
- frame: The frame to process.
- direction: Direction of frame flow in the pipeline.
- """
- await super().process_frame(frame, direction)
-
- if isinstance(frame, UserStartedSpeakingFrame):
- await self.start_metrics()
- elif isinstance(frame, UserStoppedSpeakingFrame):
- # Send finalize command to flush the transcription session
- if self._connection and self._connection.state is State.OPEN:
- await self._connection.send("finalize")
diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py
index 3b81da5d4..9b2475cb6 100644
--- a/src/pipecat/services/cartesia/tts.py
+++ b/src/pipecat/services/cartesia/tts.py
@@ -344,7 +344,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
try:
if self._websocket and self._websocket.state is State.OPEN:
return
- logger.debug("Connecting to Cartesia")
+ logger.debug("Connecting to Cartesia TTS")
self._websocket = await websocket_connect(
f"{self._url}?api_key={self._api_key}&cartesia_version={self._cartesia_version}"
)
From 0c31b5ef19d21a190b6600d0e2366bb14f9713a9 Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Wed, 15 Oct 2025 11:49:26 -0400
Subject: [PATCH 41/51] Add aggregate_sentences arg to ElevenLabsHttpTTSService
---
CHANGELOG.md | 3 +++
src/pipecat/services/elevenlabs/tts.py | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 424d6a420..73a42dd22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- Added an `aggregate_sentences` arg in `ElevenLabsHttpTTSService`, where the
+ default value is True.
+
- The runner `--folder` argument now supports downloading files from
subdirectories.
diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py
index 1544e94be..a9bf65376 100644
--- a/src/pipecat/services/elevenlabs/tts.py
+++ b/src/pipecat/services/elevenlabs/tts.py
@@ -774,6 +774,7 @@ class ElevenLabsHttpTTSService(WordTTSService):
base_url: str = "https://api.elevenlabs.io",
sample_rate: Optional[int] = None,
params: Optional[InputParams] = None,
+ aggregate_sentences: Optional[bool] = True,
**kwargs,
):
"""Initialize the ElevenLabs HTTP TTS service.
@@ -786,10 +787,11 @@ class ElevenLabsHttpTTSService(WordTTSService):
base_url: Base URL for ElevenLabs HTTP API.
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
+ aggregate_sentences: Whether to aggregate sentences within the TTSService.
**kwargs: Additional arguments passed to the parent service.
"""
super().__init__(
- aggregate_sentences=True,
+ aggregate_sentences=aggregate_sentences,
push_text_frames=False,
push_stop_frames=True,
sample_rate=sample_rate,
From 83b0dc39f715f350e2509171cabff910d7805885 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Wed, 15 Oct 2025 09:22:48 -0700
Subject: [PATCH 42/51] BaseInputTransport: stop audio filter on cancel
---
CHANGELOG.md | 3 +++
src/pipecat/transports/base_input.py | 3 +++
2 files changed, 6 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 424d6a420..aaaf0b54c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Fixed an issue where audio filters' `stop()` would not be called when using
+ `CancelFrame`.
+
- Fixed an issue where `RimeHttpTTSService` and `PiperTTSService` could generate
incorrectly 16-bit aligned audio frames, potentially leading to internal
errors or static audio.
diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py
index 25cc56f68..0f247c6dc 100644
--- a/src/pipecat/transports/base_input.py
+++ b/src/pipecat/transports/base_input.py
@@ -232,6 +232,9 @@ class BaseInputTransport(FrameProcessor):
"""
# Cancel and wait for the audio input task to finish.
await self._cancel_audio_task()
+ # Stop audio filter.
+ if self._params.audio_in_filter:
+ await self._params.audio_in_filter.stop()
async def set_transport_ready(self, frame: StartFrame):
"""Called when the transport is ready to stream.
From fa5d4ecf869621d96d038f1379030cb04a32c0fc Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Wed, 15 Oct 2025 12:45:07 -0400
Subject: [PATCH 43/51] Add foundation 47-sentry-metrics.py
---
CHANGELOG.md | 5 +
examples/foundational/47-sentry-metrics.py | 142 +++++++++++++++++++++
2 files changed, 147 insertions(+)
create mode 100644 examples/foundational/47-sentry-metrics.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 424d6a420..3e37b4ef9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
incorrectly 16-bit aligned audio frames, potentially leading to internal
errors or static audio.
+### Other
+
+- Added foundational example `47-sentry-metrics.py`, demonstrating how to use the
+ `SentryMetrics` processor.
+
## [0.0.90] - 2025-10-10
### Added
diff --git a/examples/foundational/47-sentry-metrics.py b/examples/foundational/47-sentry-metrics.py
new file mode 100644
index 000000000..ae7b7a59d
--- /dev/null
+++ b/examples/foundational/47-sentry-metrics.py
@@ -0,0 +1,142 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+import sentry_sdk
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import LLMRunFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.llm_context import LLMContext
+from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
+from pipecat.processors.metrics.sentry import SentryMetrics
+from pipecat.runner.types import RunnerArguments
+from pipecat.runner.utils import create_transport
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.transports.daily.transport import DailyParams
+from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
+
+load_dotenv(override=True)
+
+# We store functions so objects (e.g. SileroVADAnalyzer) don't get
+# instantiated. The function will be called when the desired transport gets
+# selected.
+transport_params = {
+ "daily": lambda: DailyParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+ turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+ ),
+ "twilio": lambda: FastAPIWebsocketParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+ turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+ ),
+ "webrtc": lambda: TransportParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+ turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+ ),
+}
+
+
+async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
+ logger.info(f"Starting bot")
+
+ # Initialize Sentry
+ sentry_sdk.init(
+ dsn=os.getenv("SENTRY_DSN"),
+ traces_sample_rate=1.0,
+ )
+
+ stt = DeepgramSTTService(
+ api_key=os.getenv("DEEPGRAM_API_KEY"),
+ metrics=SentryMetrics(),
+ )
+
+ tts = CartesiaTTSService(
+ api_key=os.getenv("CARTESIA_API_KEY"),
+ voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
+ metrics=SentryMetrics(),
+ )
+
+ llm = OpenAILLMService(
+ api_key=os.getenv("OPENAI_API_KEY"),
+ metrics=SentryMetrics(),
+ )
+
+ messages = [
+ {
+ "role": "system",
+ "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+ },
+ ]
+
+ context = LLMContext(messages)
+ context_aggregator = LLMContextAggregatorPair(context)
+
+ pipeline = Pipeline(
+ [
+ transport.input(), # Transport user input
+ stt,
+ context_aggregator.user(), # User responses
+ llm, # LLM
+ tts, # TTS
+ transport.output(), # Transport bot output
+ context_aggregator.assistant(), # Assistant spoken responses
+ ]
+ )
+
+ task = PipelineTask(
+ pipeline,
+ params=PipelineParams(
+ enable_metrics=True,
+ enable_usage_metrics=True,
+ ),
+ idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
+ )
+
+ @transport.event_handler("on_client_connected")
+ async def on_client_connected(transport, client):
+ logger.info(f"Client connected")
+ # Kick off the conversation.
+ messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+ await task.queue_frames([LLMRunFrame()])
+
+ @transport.event_handler("on_client_disconnected")
+ async def on_client_disconnected(transport, client):
+ logger.info(f"Client disconnected")
+ await task.cancel()
+
+ runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
+
+ await runner.run(task)
+
+
+async def bot(runner_args: RunnerArguments):
+ """Main bot entry point compatible with Pipecat Cloud."""
+ transport = await create_transport(runner_args, transport_params)
+ await run_bot(transport, runner_args)
+
+
+if __name__ == "__main__":
+ from pipecat.runner.run import main
+
+ main()
From abc569b3d231e4ff1eccf835773e3aa9226a5acc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?=
Date: Tue, 14 Oct 2025 14:10:12 -0700
Subject: [PATCH 44/51] examples(foundational/07): use CartesiaSTTService
---
examples/foundational/07-interruptible.py | 4 ++--
examples/foundational/13f-cartesia-transcription.py | 5 +----
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py
index 81ba692c7..1e7bd5718 100644
--- a/examples/foundational/07-interruptible.py
+++ b/examples/foundational/07-interruptible.py
@@ -21,8 +21,8 @@ from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
+from pipecat.services.cartesia.stt import CartesiaSTTService
from pipecat.services.cartesia.tts import CartesiaTTSService
-from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -58,7 +58,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
- stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+ stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY"))
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
diff --git a/examples/foundational/13f-cartesia-transcription.py b/examples/foundational/13f-cartesia-transcription.py
index 913dce797..d685d0ba8 100644
--- a/examples/foundational/13f-cartesia-transcription.py
+++ b/examples/foundational/13f-cartesia-transcription.py
@@ -48,10 +48,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
- stt = CartesiaSTTService(
- api_key=os.getenv("CARTESIA_API_KEY"),
- base_url=os.getenv("CARTESIA_BASE_URL"),
- )
+ stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY"))
tl = TranscriptionLogger()
From 73877218e948efe5ba3a3ff4c6221d6aae7358f8 Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Wed, 15 Oct 2025 10:28:03 -0400
Subject: [PATCH 45/51] Add room_properties to the Daily runner configure()
method
---
CHANGELOG.md | 3 ++
src/pipecat/runner/daily.py | 75 +++++++++++++++++++++++++++----------
2 files changed, 59 insertions(+), 19 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76c35a7de..5a51132af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added an `aggregate_sentences` arg in `ElevenLabsHttpTTSService`, where the
default value is True.
+- Added a `room_properties` arg to the Daily runner's `configure()` method,
+ allowing `DailyRoomProperties` to be provided.
+
- The runner `--folder` argument now supports downloading files from
subdirectories.
diff --git a/src/pipecat/runner/daily.py b/src/pipecat/runner/daily.py
index f52ccbc2b..5bff7d060 100644
--- a/src/pipecat/runner/daily.py
+++ b/src/pipecat/runner/daily.py
@@ -82,6 +82,7 @@ async def configure(
sip_enable_video: Optional[bool] = False,
sip_num_endpoints: Optional[int] = 1,
sip_codecs: Optional[Dict[str, List[str]]] = None,
+ room_properties: Optional[DailyRoomProperties] = None,
) -> DailyRoomConfig:
"""Configure Daily room URL and token with optional SIP capabilities.
@@ -99,6 +100,10 @@ async def configure(
sip_num_endpoints: Number of allowed SIP endpoints.
sip_codecs: Codecs to support for audio and video. If None, uses Daily defaults.
Example: {"audio": ["OPUS"], "video": ["H264"]}
+ room_properties: Optional DailyRoomProperties to use instead of building from
+ individual parameters. When provided, this overrides room_exp_duration and
+ SIP-related parameters. If not provided, properties are built from the
+ individual parameters as before.
Returns:
DailyRoomConfig: Object with room_url, token, and optional sip_endpoint.
@@ -115,6 +120,13 @@ async def configure(
# SIP-enabled room
sip_config = await configure(session, sip_caller_phone="+15551234567")
print(f"SIP endpoint: {sip_config.sip_endpoint}")
+
+ # Custom room properties with recording enabled
+ custom_props = DailyRoomProperties(
+ enable_recording="cloud",
+ max_participants=2,
+ )
+ config = await configure(session, room_properties=custom_props)
"""
# Check for required API key
api_key = os.getenv("DAILY_API_KEY")
@@ -124,9 +136,32 @@ async def configure(
"Get your API key from https://dashboard.daily.co/developers"
)
+ # Warn if both room_properties and individual parameters are provided
+ if room_properties is not None:
+ individual_params_provided = any(
+ [
+ room_exp_duration != 2.0,
+ token_exp_duration != 2.0,
+ sip_caller_phone is not None,
+ sip_enable_video is not False,
+ sip_num_endpoints != 1,
+ sip_codecs is not None,
+ ]
+ )
+ if individual_params_provided:
+ logger.warning(
+ "Both room_properties and individual parameters (room_exp_duration, token_exp_duration, "
+ "sip_*) were provided. The room_properties will be used and individual parameters "
+ "will be ignored."
+ )
+
# Determine if SIP mode is enabled
sip_enabled = sip_caller_phone is not None
+ # If room_properties is provided, check if it has SIP configuration
+ if room_properties and room_properties.sip:
+ sip_enabled = True
+
daily_rest_helper = DailyRESTHelper(
daily_api_key=api_key,
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
@@ -150,27 +185,29 @@ async def configure(
room_name = f"{room_prefix}-{uuid.uuid4().hex[:8]}"
logger.info(f"Creating new Daily room: {room_name}")
- # Calculate expiration time
- expiration_time = time.time() + (room_exp_duration * 60 * 60)
+ # Use provided room_properties or build from parameters
+ if room_properties is None:
+ # Calculate expiration time
+ expiration_time = time.time() + (room_exp_duration * 60 * 60)
- # Create room properties
- room_properties = DailyRoomProperties(
- exp=expiration_time,
- eject_at_room_exp=True,
- )
-
- # Add SIP configuration if enabled
- if sip_enabled:
- sip_params = DailyRoomSipParams(
- display_name=sip_caller_phone,
- video=sip_enable_video,
- sip_mode="dial-in",
- num_endpoints=sip_num_endpoints,
- codecs=sip_codecs,
+ # Create room properties
+ room_properties = DailyRoomProperties(
+ exp=expiration_time,
+ eject_at_room_exp=True,
)
- room_properties.sip = sip_params
- room_properties.enable_dialout = True # Enable outbound calls if needed
- room_properties.start_video_off = not sip_enable_video # Voice-only by default
+
+ # Add SIP configuration if enabled
+ if sip_enabled:
+ sip_params = DailyRoomSipParams(
+ display_name=sip_caller_phone,
+ video=sip_enable_video,
+ sip_mode="dial-in",
+ num_endpoints=sip_num_endpoints,
+ codecs=sip_codecs,
+ )
+ room_properties.sip = sip_params
+ room_properties.enable_dialout = True # Enable outbound calls if needed
+ room_properties.start_video_off = not sip_enable_video # Voice-only by default
# Create room parameters
room_params = DailyRoomParams(name=room_name, properties=room_properties)
From 9d78402a3326eb9185d05bd7ccaf8c098d741fb4 Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Wed, 15 Oct 2025 11:32:07 -0400
Subject: [PATCH 46/51] fix: set apply_text_normalization as request parameter
in ElevenLabsHttpTTSService
---
CHANGELOG.md | 4 ++++
src/pipecat/services/elevenlabs/tts.py | 5 +++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76c35a7de..4524d8296 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Fixed an issue where audio filters' `stop()` would not be called when using
`CancelFrame`.
+- Fixed an issue in `ElevenLabsHttpTTSService`, where
+ `apply_text_normalization` was incorrectly set as a query parameter. It's now
+ being added as a request parameter.
+
- Fixed an issue where `RimeHttpTTSService` and `PiperTTSService` could generate
incorrectly 16-bit aligned audio frames, potentially leading to internal
errors or static audio.
diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py
index a9bf65376..080f54cd6 100644
--- a/src/pipecat/services/elevenlabs/tts.py
+++ b/src/pipecat/services/elevenlabs/tts.py
@@ -985,6 +985,9 @@ class ElevenLabsHttpTTSService(WordTTSService):
if self._voice_settings:
payload["voice_settings"] = self._voice_settings
+ if self._settings["apply_text_normalization"] is not None:
+ payload["apply_text_normalization"] = self._settings["apply_text_normalization"]
+
language = self._settings["language"]
if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language:
payload["language_code"] = language
@@ -1005,8 +1008,6 @@ class ElevenLabsHttpTTSService(WordTTSService):
}
if self._settings["optimize_streaming_latency"] is not None:
params["optimize_streaming_latency"] = self._settings["optimize_streaming_latency"]
- if self._settings["apply_text_normalization"] is not None:
- params["apply_text_normalization"] = self._settings["apply_text_normalization"]
try:
await self.start_ttfb_metrics()
From e11ede475b643502c1a90e840cdbdc8f6e1abceb Mon Sep 17 00:00:00 2001
From: Mark Backman
Date: Wed, 15 Oct 2025 13:22:18 -0400
Subject: [PATCH 47/51] Update moondream chatbot README link
---
README.md | 26 ++++++++++-----------
examples/foundational/assets/moondream.png | Bin 0 -> 1101056 bytes
2 files changed, 13 insertions(+), 13 deletions(-)
create mode 100644 examples/foundational/assets/moondream.png
diff --git a/README.md b/README.md
index 50cc72791..6b00832f3 100644
--- a/README.md
+++ b/README.md
@@ -63,24 +63,24 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout
-
+
## 🧩 Available services
-| Category | Services |
-| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
-| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) |
+| Category | Services |
+| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
+| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) |
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
-| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) |
-| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local |
-| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) |
-| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) |
-| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) |
-| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) |
-| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) |
-| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) |
+| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) |
+| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local |
+| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) |
+| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) |
+| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) |
+| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) |
+| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/server/utilities/audio/aic-filter) |
+| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) |
📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services)
diff --git a/examples/foundational/assets/moondream.png b/examples/foundational/assets/moondream.png
new file mode 100644
index 0000000000000000000000000000000000000000..17ba5401fb2cdfb4d1a5a2e6f9738a09bbf00d28
GIT binary patch
literal 1101056
zcmV(}K+wO5P)R%hRxONzCZo#-@UKJ>wrf9BnS{7)sUiTT2f@Ivqf2HtmK!ZD)}iD$`Sv7{FJJs
zDpje9U8PtRmmONBB+9ZR%A`mNAc!7#{d>1R_w>7OSCcX3{JypKx$glaMMQRyJP#N5
z+;jF`d#$gT-<)HPIc6{!FLIGw{|kda_?73m1VJeFh3~^$;y9Kt^gsI_=W8}iWH4RG
z-DxQQ?T3AtRXdWUp#)j3_dLM&@1D(NCXtlzD4j|y=i?dP8OTAWCd`L~Xf$IFd@WC+~fDDhInA84ZRq#nf%}TC$j~WHgye
zuU(bla4tEXy;0})Q)$#HQZ1F`tUr{4t*#8t&ZU8StCb==>PoI|HzZjuq=E6+uHz;!
zhNWsv&d)C7rR)3h)wiC=qyAEMI$b$C9bk>b98Q_fmcrG*HM3eRb^L|nlcadwP->O3
zj7DROOC%|Nu8)X6HQdVrwm#@8v&%>HpB}=^)?;VpaS%CG$o9Evc+yFnTmN9>r-w6An
zg8QG&XR_65;M##)U~M$36|A8E<(8qiZz6+6bjlEn&`P(rmS5I2y?R;jeuHjH
z>#y9E-tJYoI62Y%)@(H7`21LUowiJI%_Y`h9oJ2TuPaZEk1)4&%wr@+{eiTxZc3G^oDcib
zYt>~mp5V2iRBKh~4@da?GR7y>vCS|hyIUB4JWIJ8%M4?d;a>Qe72IbP`-E$KGM`I|
zv9DJuQo?f$Clf9lj6+?fv$>904f|w{y%oiQRIzrZ7>_1CbGcec&i#rxUtx_@OSydJ
zm6rUqKlifKf`!fr{zo&V^Q-fN`OQ{kdHlghP9M!AOv~6$kbF(37G?zFs
zn+Jo3|M7dQ75EM2li6ljje5ct+>D4cLDYj
z{**AL8SZrf`w*x0d9;IO>+SF2XYTrTwOl{PY`0
z>lpta)qCP|q`4H&ed#N(asT+QkDfJI;Kl^kqrEL*%`LFD{P^+rd`Xzw8TQey+>NQ|
z9O`;-*Q4zXavR4mD9goJD(!Afq6*hvrXw74AGtN5SJDp)Y}%gwBpV)^dt5uq?X$U#
zxJLr9^_Jzin*-4?=X%lk;IMKX!bYW8rmxR}n*8p!$MWs>ai+yJypLmqwJEVAId@nl
zJFSXr;T(8;I+X?XlW^`~8ynb;q<5pN%scxpjx9!B1E6%yv)i9uIU}@u|2UTt7;1-X=0!Vz2Yu
zgB_2s4#(4_o^c!(90fX#oQHNj*7L7af!$ePUUBcVeY5FY$GKjOWrgu5mlFB(>(_C1
zjO6n_vL`?OdRO=AGL7Zk_xn-~OY-PEm0$nX6WPCgQ|>)F*Uy|Sc|HZOldxIym7GnM
z5|=~i;JPV3?|iU?&Bl1Z{*6ZyT@NLkGo$fHV%W
z3D3cFsRy0z(LA#q1p5)_0iWo72J|s7$FwFq*McB-ujj(X+~Zv4R}@8RWBnfGzf%sF
z`{KXzjh#B1N&T(+Fo~5QqAG
z%)c?$ylce&UZ8;9~_j8OBmvRSZV7$7JFTQvnU%q=GO`LrZ
z7U&2*O1D{;d@+-2y(ZS_T%PGvrH*-M$C=z{@O;7dIA=za8TNNcj{8%LQH-+?XV!{7
zlHj~b<#d|Mga$@!1dlksPc!{H*x2~~Uz~IO9o~awCNZ9|iVJq56ng`&%gm;Gzniv~
z``=8f**~=pcwaeR%3cHS53ok<7wVU1A$+@8EYmd9c~&=!bLh?;-fQOUk=Jahe~SU+_9kjlPiC9{NPIp^ZivHU;M}-m{no8fWk|75FQB9{T?rzDis+
z{}%sMs-gOSj0=|7_cgqBjV0@`ns?KSjV|7
z7Ftz1Q#tOib3AWoFVrmP_m>YEj=i2`+>>f^uuial7*FVyj6xd|tmi7${IyPq^^N-v
zD)_j$yz+^zoF5J0>m~F;@CW{Br-VOMU2~%ADHHFLa4m8!xc)M1>eu@jG`m^A`c|Uf>WSQQ7p610Ki=v$v5_-*Nwyus?U=DfU~U
z=MR4sn)A>Rw?~?J1g83Y)?l#S0mZK
zc@1Yq=$;At>}(}(|KI-y`1zT<^jAJDckkYlD=*%V?(^52i=BPs9>3o+7JL3rzCQLV
z_x}ge{o(f>z=o%C?b)k1^8?u&HLh3A)n-iLzkbm8`;EfuuHYYWJ?HA5Bkmc^m$wwy7r*#_`cHq!f4(=Q{)YJfG@LPvtUA~di2MQe!@ud~`N-|&
zfd+eHfE_^)sRK2^fe<5rEMb$BV4O!;DDU+bvdBsZl*Qi#%OTk
zoUtYYd>D2LJ(^N058S8
zZ+BWU>W}2k)g28iS`~tL8J;_r78Y$iUP-S?hZc?r#6_jovw(+Pl&2!}I*h$=t^
z9_;E);$9V?0B}R6GiBgGCs+ePbGm7~=;ZU+Xz0rt#LU#{6YQD6!Q|_!)mYtd0)e#>
zfS=I7fo>PT4d%QIz^8#{olGYH4|yPF3U)N=3`FMYd~>Wb2gNcdDC0U@H*w71KUAQJ
z_rjku7>e+7ItUmw`LjRq0s_N(a(I&H`bscf-F63SXrX(8?;*&>FM=!F6AZN4jV8u6
zQ3t1m`x^sfngaYH$h3gt#LrLpOjt*pR|29{{B8umhYlivI68};es)U_*sp&uhO<19
zR;!7%IEPa`R-lex8o{eFopxTAr%0G-FiapTfm79Mx3NC@3R3Y9A)wZ2*A@6#z=7XC
z*ptCK;HcfWa8g-R*7ZW4+F?CYzPA0(o=@>1ZWx-su=KShjgc
zbUp(Gd_tMQp=)5R6U^$5$1+{cuzzMcUL33Ge5}r13qeJ~fR7Fwoe9PFQs>~pdC@Vf
z;a-;-WYK&^*u$mrQvURh?Z{vHsb>MW&Yh95dlK*0<|UZ{w|noK7cw}-+6$`s39%)fXdxK(LbmJ)Bss
zF|JcOU7TCKhxUeD-i6{Ud;_Q&CWv>oKrl78jYx`h2i6B@lV)T)FHT=CTSeT*(l&wF(;=R~alh
z@G%S=80F447hD*K=)ubMfgiYl#NP+)lmTG2z9RR%f4|O678HRNL4Ey9h8O?^uWTcN
z#2h|3n&{fw=~Q4dmIa79o9EJpgRsJSYXGp?+v&+@NHA-ufDhM=26(uqax#?bTRDJY
zTcaA(VGnm20Ek}Nl9#bhn(2vr@wF|0pG!GGptI88!{0N<+(6?qoE7wLdyNv#1HfuX
zK8#if2oe19GbgYU#Cmz2sw0bQEU-TmgA=4XNZ0g1Zpge5LoFF;h
zg8*Z@9l~}$3veJoU|Eu1{^uXaZ@)Q~!wZpbemIu1d0CE*CkRTX^4xW<=}bm*{#ohx
zFxXq-ep=B|Uc0?5#}_jNFDJ0Obtay0FLT;2%u$SUVa4QE?i^!`A=g_OV>ZYBB*@Ks
zUg6pb;3H9!CaYqPhi?7T-f38rSpr2DtnPoe7P8dgx@!kK*hnn;fK^>Fb^dIP=^2?CM>XLBlRN!tdFDsbDFi1
z-VfJGgCHo@V+{#{HWFNig9{`pLwRn$B@g;DBoRgZAbuFn(QSaBlle%t08~U6Ut(l!
zoU2tPKlnKdOAIZr{s?-laAq&DSZDCDmizaLGCRw0iUthe?Hw6_SBWASL?2
zv^P;;0GC?E+@&`B5fOyzvxZ;{uDjoaFBlWN$}D*STSGu|3D9A%viaS{*?DcNhUZ_v
zP7oL`Ie;ZsUzs*0)8v+&b?PVb-}I$L{k+wY3l)cxGfC)!I2oc*h
z0aC_`m%sA~!Rd}&$N3BWlXO1uUy
zThsjm04`aHC7d05NsUyb&!lT0SqG^E%M=9AImH|Rz>K4X{N$&1q;Sp@LZ
zS~-Ik68QW>1gWPAoNWtu67#%wi%syxby&$7UJ{r&oD4eS(-
z{8ItFe;xl-QiiPOjAgo(jg=DVoMQW9Zxc*zOaEkq*H&aQNTh$fz~4*QGCX5~Ya|S`
zas`pXv?ODIKQZ=8g@?PF7dm4;c;)!$UJ@x7%K64=?w?cVI&|Y-I0VKhP$my11A(%;@*lhn@&RNU<7a_k2D~C9)ZNw
zRw_HynY?f{lh5LRi@av9Oi6a8^i9CC&t{J$0`kNm}XZxy5X8@j==|}30
zfi(dW0%8?@GX91+gbf(WGY7lK>>H@n#@^yuSuB0>LloFo77{_hS%Lf}0{VaVi_gf-9oVU9ApiU;kLB$LIQRE&$amg3L*QM}{Td<$
zI@qhr688K4;ZUBxxvT7}4ENQjFxisGC$3cVJbW_96g01uYmUKdXFb;sf%tN%s=zzh
zBTQV(kgyeHm+P-|TqOaw7W!YNZm9#?9fy1-i0_pmh-
z6!y~gV_?8QXlusUIi6h=nD55dYET$xk|c~ABa5FCxSgsIgzYarTd3!d7`SDTpMmP!
z%gzofGf3BKWOkR}g#%ics1kQ=+F8FBJ^0GRA^lZaU9Kgb6{XnLHhnAl26~=g3<)Z(
z(ri68c*Z8qw;}v0?mM1AL!76*c0*Zu^e?snZq<;K+iO<y8D!uUFSAl|PJ9n&I{UKx8aPL&bjZ9Sq5
z86whKW|2PY()lG3z}yBf?kfN#uI|)icdM@K!^J8P&v*&kXOLL}uS~8sa4wcsMFK!q
zS43_n#$v&B5K9jr_udxS-!lbYErF6MU`4yOQYvNc?%T#h&q*40by<)+axC5)bFQrW
zStcg{uZDg4PEnB5(z=Jyk$KNmf>@EaApvs>UakssV3I
z_5s8X$=t&Am$+sk<{KHCKexD}p32r%Nvjcpuv7pg_RP7Czk;TY1#bR=oqP0c{g+@cx;)oKgu
z8-i|89vNH6C2azCuX+Zic24bXmytv*BR)vv=y0yAEhaxS4iNJtau>&DY3e@L`B0vr
zMVnHJY`hfIvHRy*8eWX#{A4KSI0ug(4&}+iK3)?$z&8UU_ZA?%D}{h5E{&+A+^nU+2|wds$1H0JgolN6xWrr%=3ale-A+8%S`_{9Z4
zmdTp`{&AldtnJ37n+Pub?SaOR-pkXk-S7z=c&zUT%=QVZhBC7YNiMF+z|JF$he>+D
z`B15ue@X_pRo5WV-)h5$m>DU7l^_0rxh{}-IQ#N_$??1P%LlOUBiVoUs!{5lzxr`s
ze*$3r!N0f`j_;i!X*-Z>x35XLQE@ia=AQOHc8q6!dGgZwHGu@10IvQY0d3J(k%S}u
zj1V-}5Rb0&`-d`ke5Uw>kAFOfl}mnkKLy^-ZX#p;YyI=+7k}|T{Uy148wKmiGBkg<
zuRqk{V!((4fMrbi276F9J89-(i2ASHpUS_yzZ7EUWIQlf-UevPpkuUPnOsYz0H!*~
z4hPsg&%h!0o1eWZaK~Y+5`ZxkE!P+h0pN7I8ZO^`LearKIv@?dmjb!4j5F(#I6
zDT&8Jd1ytk%Vsh#f}yL`8I-|E?R9bZ4RpN8z#_8Gvpq|CKM6f8&m8C?0aiPpJ
z1BmF@*DA;u!J%nj4rxUB`);QPXJes(NF2=M&wT!leCvA;<-s8{b}W0Fs$*}g9D;8Z
zW%PAA9d&4EsC(@Wo@u3-_A0Xu0Ny(7w(hG1Df9$p@N*h#4T1u$%cX4hwsbA>g$(GH
za7^B~)6s+SH@|&nI&ih#3C3Ws603KO#
zNv0=(vlvm^Ed-GLi!%jdPtVR|4|CN&K9uV_yVy_XSlexN#>nhy*XmlXH=0k89pg0t
zW{xJ(!#X1{Mo)JO2Ty-E)-}e%oop)_7zHDlS_|!{XLfP{Kv+R2Iy_oVXABuSkYx68
zZj2ce>Y?b%1!B#-iw0W!Y9iamQqu~8?4SMoRr&K@yn~>5VGfDXwp_qXcEmiEdv6Wp
z@E)ANtgJs%TM-(&gxQ<=RxI@n4$5q4lyZzZ$CHdHuB})zn3?#vgagPGP~pvIrpY8-3*io
zPG=5m;b*7W$^l@Jj!iPb^%evcLPy`Vuqy`0ZRI1POe!sJk`3o3$O`6_Pj+YtIh|w4
ziU7s#tB86ukZ8gQPLicLhZrj@cbz6OJe|qK*$^ONpbRgoOz_~9tTQ+cVAEhc|M>G`
z^JjruTd6B`2ozAnU9Pa6LhOx;rOYo_6Yz+iyx8c4`W+
zCvaoz59~Y*WjZ9tQ_`{#v1(v*r2QOklm%?kto?c-$YnrRzVQqjCHnq74hUBeFbIlr
zLIW!@ThoeBEal*!sUXk?9}e_<&~k@7m$Yz+En}Q(k)H)qq
z8|}OTo_@MRSFS-eB0l^5!(84!t;>Vsp`JCIyqgG&uXGu-0qDoNy>+moHh8fjvxaQp
zv1YV);27~7n7AYb%hwi(1!z?ONcBpf=fM;}5Z80Bj^D$^8USG+ysc>iPFcQO3KsJ7
zU%V-weR&taLoUDk-@lDBH_=kB$H!w~SRNVzreiXk%rv-c(;tA{PZo&=-z#|Im0&7g
zdSyr6dhaoSib&ansmp$cxdk_5Ua=&epceN#ZCVEV2Wur$Bsw(cWrBliP=Tmif2Ov?
z0gyJ&=9H;jQ8r0h*r6b|vKb4s+UF#TA1*U
zZSmi^d%a%Qa1Nb!V?!BRhi4JztgcCyRL>6j7-RTf++TrXe9duAJcoUJ0Z`-&f2vqV
z^;(RxuP--n_D$g5kj>D=noDq2SFuLBm6DFcVr6WZ>B`uPT_z6C#=??+0#}2%7DGFR
zx7o!!%`VxY&@*eh%U$_%7#PD?+4stxOx(F-PePwq!PqD+
z%3#+(1-?g8cYKk_@eu*`*s;W%ugC6NG98)c0Hc+ob_3;vuSSiRm*
zi3hUJdCvsFwThuqQJX<~mJ`f{pRsiN%%2H)I+rhg;ii1*6#(nA9P4hbK$$+fXBX2h
z5Y$dHcjn}lG}8oIQk)|ueUNL(SsbOZ+h($(DPR5iA?##H_jYE7mv}!QK{9an>W(BB
zKgAmz2%#XjV?)^>rf#lHhJod6ycT1PBuV_St-q@w$v_Zmhy=z6;ML)y8O8y&wp?*D
z>4UkTNJ17&@{Qx(v)_`;Y^vJy%-C5vdyJcw81GrC$tmwMm_0H3nY;Ps+TuKOw6ai&
zq_Z-BBZ@SnvW>mQed_@-8`X~jC;p!b(r?7bdA@$G%Spk_zc+jrwY3{1$m$bY%)zrz
z?;uE>Em-?!&ZHK9ZwB|@H{5mrqxktOUbs$_TzbZh8+X=9&;Fe27q%gDUSnJ?>6(aH
zDS=ol)lybV*f7D)y`kqN{v-2lJRV!J7-Q8&e33h3jv&VpzS+g^JdzyttMl4{^lofR
zSdI&O`28O4PXMgn-og`aLoH`=Hr
z;@QQ@&M9?;SmK9^q@#r#zJCvX_{f3Y&em@*u=M)D=3e@Ew%yafYmqC`e}twj;8|CH
zn6FLR`@iWI9rW>Nsw^A3R<_Pe(n-X9ej8cSUwIeKM!5+vN|33_<)BlMBLpaCIJj#t
zx+mu&1wN{9I)CbBRsO~&w`D(^!4aY!QB{LvW>Kg)bTJxe{XBz6f>$ifngE0#FwpL_
z<)S~*EHvrEG_1UzdX1UcL<8aeGLUaQnaG>xsr=6Up$w```4E8e;{=XW4E-Bs~DdE2j@1xti=K#
z9vsF_uO)Y}pu?yxBh9iSGfLeej0vwb?)PyRZp+2_xjK{8a){?S!_W5Q=ot6eZK=)!
z%lxhb(0cgfv6in)VX&BSqTbOG*}FOp`K69^O?AR}5EGD#U0D>zl>kz&-BRt5R=chl
zT^>qwCSn|dWXiDwbbq^{Ko-&eCgzDwvw{gZ=c26dn=Rpx0(epu6xL*`(E{Kz#@siI
zl@r>b&iSR&PbYu}HGv$}Gg9`^3im<1s*xJ$GOj(Gj1?T?X4Q-W-aEv4(ejtF0gel=
z4dwzve%3k0j9N52_gc-S9?S%3YY6;VF81)@1As&gE&Y20z)259I0yt-Dl{s%#?i^C
zbXpB96J-e>vmIolWdLSw?^NVUGm>aIkQcA)$PP>TkV$J}a$dfj^#3Bnxv>lQY#bkgqg9MCgOurPg0Y^<&;0mR`MJ+s
zhjTH9Bkwh+0u30o6loI6TVIDkenNv;F%5ytL^mu@UBY?9egk-t0o*G)fY(WrDPSET
zJGpc$GCJmF!w7~DtcG*Dl)~A
z&H+#O`4VtpDhO`dWIIOd4+%DMKY0Dgec0erbKHvzwj0ZW
z0T{S)!7y}d*@7U|DRK^iGBKh^eCoN-G1Ba??QaD>9Drs)ZQ!&cJg7$}elDMRy^X-9
zi3G)|JbR;q19Pr(lIZ?qujqe`I$q0J`j;m*DK0F=C+xJgoALqtFy9RrIECcKV0+_J^tBLHl>Z&Q3;ohm8
zH-$Z*E!DM*@t9!WUEn<9xfLtuni>O&3n#oQ0oy`5J(=70&UdT)(oV_VUg_RkoTq
z>*rSPppKQZUo3gzeUO1neH&2_G1uqgI+)-*s1n$ralm_;jjDJ(5A&fch~n97YgLT3
z6k53+buL*pZ)Hc3X-xTWC9twR4`La4m7r5eSyd{fkcY19FHknKf?A=ip)_|>>X=q$
z1i}p;%GP9oTITD0<;z;R=e;fTK$Hg~LSHf%F8N%yZ?S?YE
zhp>TNoTlJ(fCL_k1x8NPRk4Yd+`p_k)$lk?Hg@?D%`VU{Y>8L5+MGAL->-JFHsBDfGF(tP7U^;ypW)P
zy;;(W`N0?r=7A=XI*mv{kQLB=vI^tKfJiO>r`{6F$r1}%R1P4^l=YoE;8uYwF&kZ+
zjreaq&gAq6;BQtcY`VB4X6|*A+`7vgn3Fjj6`%Ov8C5C=*w0lh;pIN0tOn0x)sVuP
zU_yz0`O;~ThHgzJPQ#AtkHD7hS-f^Und!Z#fFGd&-*3>rQk@#ad*y+71>^ga=N
zM*5XZaB`0^j+%_dHZ}q0y-ep=V*H+`OsdRNJ1Ypj;A0VcOx;2-Alt7E?rAMw3@uTm
zHpP_-+OVa1ZmaxUDNboHfp~{3GCrp2?B1)Xk=vW|%Y6QW#bD$Frl>TB5-^mMv5ab7SxZnR_xTQx6MI
zWBu7{w$;;ty8q0qND3Bz7e1sEd+y`H^GiU2erwEI>R?8Qd8h+Nc9j95qUQ>Alvnt}
zq&nYHuhn(kE|vtS69bo7Zd)m9{1g|g-Qq(fU0Br#^;#R-P5A2-@in7=2pHi{zWkSC
zhfgo2Nc2CE-d<0-y8y2w*#NIB@wzT|T@QQ#Ty6X$RiABvxG~W)GIPx5Tr+=}Ts%CM
zM{j>9nI?zLch)#0&*Yq>1Uf;+L*Zl_tG
zd%>Rd3Z{3?}~Ks`86D}uXb~J1rA5Zjm-lHkFe+D=E#j*jF-f){x1EA1+>-jQaKSZ+t{qZAI#V3xpPW5&?R{~9IJm10R{vs_XZ
z6iXKO_sAd`sk5>T0OJB6M~9#koX91fiGbVb#kutGcORMeR)Zyu3&*0WA#*yBU4U#9
z$&EtNjC%mWa1GW&N1_IgT1><7KsxQF1`t~MKqsJ5(SwiGJbQR9iv7)C^jJDa0dcZ{
zQuU-wkBp0kB`0(+${0T$bYxAbgUj)!IBK+#pZeT0@~t-?B72`H&6!#MK*irvWpnX5
z)J&-{*rVfyXNei+F@PX|gZrwKORikf8XoXE%qFuWj?4s~6(U=UnnwJ7vM?~`Q+K*>
zykhyS?;dG3l8(4BK=7<(a}HKns>Yw0!deZ$5uQou<1hr&>za)xSnCV92Pot0eM;
zSMMm}s@IC-&h;IM=3{wnzapQ0{)T+#!$axzx{^$X>g+B=edY7N~K}v#f@5Lts+bJ6if-hq?-xGwbRFV*}AZl;bg14J^?$td3Br
ztjBg3mK9L0!nytnf9AIQ?2lc?+FdFGMUU*<9pD(N;dw6aey0zIxU6M-nG;Ud*E)}l
z&RwI6z`7QyBgz+Wj7^tgt!to+Be4LJF+CHDE0%B4<#bFbOFZNnlv;+@XDnTr7maEZ
zI5B&%aWr{dYXl&U6&9__g!tJ!-~%lYF~=_QpXC@|(T4M?r3(s{0)mm_Wpy
zFrY;ov%$a4g2mL8C@9;Xj&ZOC%GScFxdN>kT;~>OX0F*pFoB@HZ5RRS^z&-uBi4Ci3D%wbC+F}jUcaJ
zM2UF0C;h*Or9KFT&K{0rHe&fqSB7T*vMv(M^0PF>mm%iG5#t?F*zaX#01gIxI#}uJ
z*N)}-{{z^_*_e8K0LE)k0RU>yUaWU1v~m2osB;tea>xzmv&h`KQW*za1>x9LZ0--j!eY`IitFt>jmJ{e79F6#!wG9G_XaSC~c0BGFRz(o%IRv0GL5>i*-t
zj_G8w)H1Ohir9Z<9|^9RynB=Y{K&LyMp;maWx%yOYLzI+k>deV*mC|^z%R(~b2{x>
zGLr}D$XS}x_Lv4ZP8*L;Ni9VsW#OKvnMj~I@J_r7Nb>cHDAtg7sjL*9zX{dW8SrDQ
zDV}dh<^;S63I$ef6goy$QCjHA0L3w;%-#j6PhzieLheD}0QJb2JE1Qxbl=CJ1LUk!
zD|eexb+6oK5iX~lk1J)%%#+2T=Rt1e(qycPsJz3pPzPb
z4xobCS=6eUVhpcsw`GN7$L-y!oJ>>c!kI-D+9a
zJ=YuiX^duM&0(J>^sSg!EXnzlV0_@p-$S#ZUe{XkH8A8#`^Y+MS40`R~dE&$d}Nb7Uiyifep$fgWsjhVwl*3I(?@r_>+2CTIsj*k2sE@E3LLnkJ-*bb5Vn7
zvbh0@*GhV3(kG)`QZR}#792a(f;GQG6wHdP^bx<)gp*L-prSg7kt@%Rf~Yua7zd>e
zuqJrJpo_@>le}2Cm_gQOy1Q>4;fxpNrZ`{p5RYD4#@
zHgpLMu*tHiJoC)9#0}23h?fkw#0He0&}GZAtguI6um+gSnaWfS!|Q};5}AohKerO-
ze9+g*ieyjdd_uGL>!e{&G>6KaUZnb0F6j}uM4_HlVJMqPuFJ9qp#vf-R#Hjd(o0tq
zOhnszEQjyK@UOE~p+#lqo9%Jkuii!%PQe(|=v*EbUs@wGWi_BvYVg!T^!0|Hnq
zRVM3-l`Pt1z}-frR&xyxGED+7QQbg%$aA||Z)h_n1+K~!Eyu4S(N#x+D=*fX$n5c0
zhF?FD7T#P+-M`Q4hd#!YL$
z`nk_dvp3wVLs)wwn$Q-bJoCH7!UWo_eAN7Q#b$6;Pfmw2czhz0vw@NhjBp@@^S%j4
zT1sc_Gk@G3vfnv5D_Q>uc~i_5sE*L@Q8M^@m2>={pZ)&7OvIMr<4V7l4bfbe8D>Ih
zj3D>#VX;40)%0h(tysle7p?;kY1BgD>trFH=w$Lgd!Z#?x&fzVK0*erW(N+83OfoC
z@Z#q3ft}XEQ->%`ih&~_#V$t7A5*_7^ZG9|l5tt@l^gn67XiU)ssiaD9IY#zSn4cp
z98cv-x0>=&k7a{110U2N;F`#){U#2uC$mts4Zi)>6WQI`mM`P*X|!6(wxCci^_Un?
z)1l(Mb^-cKW()1+W2Jj@WZ{~cdE~zvWU&mV$j)CyhWu3Zm}V*OeA|AsjytdO*L0-xMzvN`=u)c%NID*<~`JdYvBTbg65kS*>daMy2dxT~TP4!P8D}Th7lf
z0AkhT*=zffkNWaSWYtRJ6Zso|`jhh0FCEAmR~zzkpSmWm-ss8>fQH+<4cW*3IOx>@
zc7?LrtjJcQrsKNg{os(|TH=sUUYCI37hZiq$AW<~0X`ZnHX`8oYXGvcjuLcSsByvP
z<$g_E*B%q`jnt85kgGz~N(ImL`2rWL8|x{7bD7A`{={|p>Cax15rRiXa>nvnB#d9voa(urp
zt8r>#4bI6a=GPC(Q-56`S4DmvK;`6)mqo3JR4VCYF3m`ak
z(h+=r{o8i|4kB1QKa=HnB=_&0V0~@Ny~k5|JgCVB4;Knj=L{^k2gwFnCJM|jqe^=;
z!CArv9o*lk3t%hxLY@a0UP~u(wTt8o3`9gdK@B=m1+_15?f~gimT;@p#agnlQBVlu
zadW$+%oOWBmB9uqR6CI4VC6faOxj!-eoP3=p<@KwJjP+kO6qhC
zMFukIzV)SkWo2F&IlzRWQWktS5UB2(R*Yfj)@v&
zi?OLoIZ{x$if5@%1baSL;dg=vEP3zI&zR2D2C>WFzhfgQfQ@{?8-S-`(IMc4pC4zG(9;j$qwWkK3D1-c5x1P?&DVR?16P4$&qc9$>uOy
znHA-jR>)h6gF>@0cY(CG@qs3y;1j{_w*dB6A_J|)tC9>;%x_SU;mSmHZ$GmoTiX@a
z$ukt+6_qxFKmiUsx%Y64=gVaeXM8P8wWQ4gwNCFt5YR7AC98S|Lzis)SFx{8g_=M#Z?jldEB);ugIn)GnoOHMD(
zgGJt-vhz1DY#!GM{eQ|CM6L^-mf^ea)m|xspP({cgY(7oyd1FO0VWA9hth>v_u!R0
z^_5?Gu`%)2%+<};DdURIDaMozJQFJjp8ix>17=VE`|{7%v*2$vmtf;J&c$;l%jKIl
z+qMR;=oD%PSaw`b&5tKInXIhZC~-b8XN`?a%53c9&IS7Rl&0~C3$5f}*ZJ`nzHGgi
z|IIJ@9y*V4&eRiv2p}ty>44ZWSUTmXKpMgf>`YM#=~o=Wlu&?
zWxcLaf$_WXYzpuymIuhvpJY|}aFEI(A{BuT3roVXhu?hfT;A^o^4^C7WV3O2hF$sUTW8Ykbmi=v`W>0F!2qLUj^}v!&Q%H6x-6rfRYNloD-}?+lC`BWqu8}7(154v=<7?R*^{C^(Liy<
z?tQrl_s^(X28X)0wWSni0*7Q-otz%2ZWy0uI-ls_Pdzqjc2NIocV}OY&(5%>n;5G|
z9zOU$e&&rz<^otic+B&6lsV+CxpeyAl(%DS435!3}v+}?!Xm#RJ1D1@q2DMFStz
zd7+rA@93E(g(F5m*2Ns)_Gksb14i}&!4K?9%&wbs(2BznS#G4?ne8`+-+?Hpu_&~f
zQ-$l}!hsB3W>iafWL^H1I~%Crhi7jU#!*DP(RM=5#-NIHk8STXThn#pyeSJ7f~JR^O^MP|;4|04#Tlu5;Q85YyG5@kQKBv8zRx%(AschB;d0|4>e
zK$_cCQPst8o%+aKDF*|T?j9WBUX0hV956A^!3B7ddsoU`T&z5k#SC>aP_XWqttrHd
zGjrZ7747jyiruw&}1xFh;71
z8C&qBnlzXj`Vt-p_2T}XDd2rYO`_=h2P%Xcx?~o?s8YFZ{}w0Q?if+xx(o786jzkJ
z5$aB2Kvyo#DpeMA`i;4zC~dFu+t^<>7L!>vFwORBSb)U}GzRW>;!0q%{E`D{Hqey;
zIaEi>2gL!p4(&0O!N6eYC-u-8<%()V&hc;U@QZVw!4ymN
zDay_Ylt_INvTv#?%&!4H)^%5tN`WT#_76HTrO0+AMv&jc_lYKy*i40z1OqIP1miWv
z`xi;BV>_HKu@(bmJQ4i#^SE}n3zJYdX?9
z;L|D$q5wWaJ(W&*r7X0-l@nSFZse1Avt-re?1xz|6BgO$-F0Mjeeo
zWZj+ly_OQXvOV8LEz5id&PaW8HlC<(d=
z?_DZ*Wo#|_JC?kTWjdN!*W}4U$0nk7FZ_+rt6m2z*h~vxg3T@}@Z)Pp)W+;!>Ymjv
zE@Sap^C7Q+7`PHj*#JX)@{Q1ahV?OnZSFM7^8Br?932nkv>!TMt3<(^O#QOC+`Q3{
z4t$J0n?HrIvk?Y*tKD)97`V3$FbW;x%Ro$v$$Zy9aRI8?00Mr2yMG!U9e?M0qz>FP
zD`J1^T;$^P{6gnnm~WIxb}8WHbDJKS6y?b38#n;WY*fC1DQ87F^tvR_=EIltvrM-h
z8{^T|qC1#3_Im-L1egDoOTWGRr5)T#9;{2(0H3#&VvMU;K{QL`JPm&NdsxfyJpCyj
zjoEbvxFIMsH-mK&B{!OQF%Eg~^z{smpqI8Hpc?5$dnd8#19ov{r)Q`4OKoUs
z`!a4yT*Cy~wR*j&{&WwqCW#BIB%=M-PTfyEi=F}WP0P^+xStia;o{-BEHCC-U66n5
z(@QaCfAVEtexGHqy$|1hpy%1nja>;#VetxI8Dk}}PkrS!p4=Nu>T@kHrp_GLsUZQb
zIXF3&;}76(on9agS!t3w$b9)+=KgH56`ul6#Y%C#oXX{gczPeY1TKB$U&5FCUW4#4
zALxJnL-|slQ{5&7Q3xvd&E}yiUk^UV0r;!;rt%Non@Ct`!C?qA7~xx(eY?_Y$S`>1?e#rrAydq{b^K4AQkFY$<5jfmA$2Wqs~QZ@F!B(dU;=sw;+HHjf-^Nr
zWBK0kLcVvN%7Y1<1Y}umbO3CWa_fkfrPiH;^kp0vS8$MYuo3H+#8>y5(xkS4YYWR~
zqeC}=f%xj50%l@
zK!i*1WRloHLd(SxL)G4LlzaZ6rWwm^wO}W<(lf8Api5qi*%-SrlrXo9FdaZrj`_uL
z$z>#;`(#`G>YsZ>MW;QTKUXjyRPgHldn0-9;X*sfSt(xT%25IXb9B#GhLjmV6`XZi
z3KW?S8-cX8s_IlK4PV<{=URi$zkOS0ufJd-?dGY~YGntySjND!>U8gu?WN-7j&T#Z
zZZ_QeLGFa*b1fs)`-#@szTB~INFG_j`JE$(it>^xUGa{{W|q|c-r5Er3kN!zr4CAQ
zz?n}|y5P#pXW*{kE+<}_$@!QYfi_@OxrHq5us^}!7Zjkof30>T!e^3^xqGFfVzBxg
zfd}v0JZOirOhrDGHbz^5eZ_LC(19AdF0kiSt7LAV)~0vj-Kqiza=Hk!F#@Ma$pmF{
z1GR#D6OH?5meH7G0F1k3*}HRK>^!WKy(_zFtAfat$hzh3y+MDKvUapW;0!y`-C~Ml
zQJjy~c0Cl|y-vqn9LRXd5zQ|-l)5*3mOBcIHDz4~L)p{?P{C{;%xEX0*qn?gk=r+V
zG9FCi-u*GwZdo<^=y)CR=umB*TQJ0JH0s***
zU>mGxrNvfYA_dSfUm7@_XPL{O8{>s#lAn8}CI87^|AO3qd?EkMKYLr2c^NhcK>u)p
z^RO(t-3GF~D=mfhooiBd*g~R-9T-)|Ei`$7{mMlC{m&oBr>>XfgC|S*)!#XiX;RaY
zJgz<3d$PI|cno7@;&M+b{P46|9eyd9C^^1f-&pj#~-$mWnIWBh!jNl`^ilE^D?fbTPOR8jh%znTKl=
zLfs$$;|SZO4V&M_dT+%D-XrUBr}M+L%=i1CHn6fmWy!iug?<}q;OVp@#fh}@l$rm*)r?7b_*_P+2ShCREKow%4eyS#_^0*|4Ry|)<8W200^*$ZX6N~^btT|
zeqY+J$T1QWN8`dn<67X8uT4_E5oG%>_JOR!7o4z*a1c8in<=0b;r!mhr
zU?p>$uS-(jF)`P&SvLF$RF)#r(%k+9{TQApB@Kprr#Kx`2ilmHTA7|)AHolB?jyw?FZjEKP*p
zv=FoFp|cORtem~fpJsEegG^)j+NnRo8GSKKj6I4q@Z?rLt>4V$UA=Mm!tOt7oKZ56Rci&&wS+re-K!3%6(P
zT;-n?BdqH%b
zY#pQrL6NLlf1U%-!!?k#zNP%qeQhWcSQ%!%5vK=hNe1!r!#!3|^9{&OST@fXiXFrg
zW5Su9FYFo4^abo_jR{Eja4cu$TrtjIg%G7#cwW*5D1aB3Oa+tO%_gN=0QTY3DwWs;*#u%Db++3u
z4CjM^>>}G(B|RR2)~)?*8C+Z_ND`*zY%pF(aJcXmm$?5e1T++#WQ~704wQ*RRK28!
z6`buXwu1=9hu~fpLBsLciR^4yM?99#wW!C%u6wxl)xB-)OsJjYn61)7CBXePrSHI|
z3V^>p?yJ+WZiD4Y&1G3EOUeUkIG$<=BK6P+Vlfy{jV@e=g8E$mmK6GLD1bE8Fg!`USJfTHV_ssatx#Ki)I*?ffiK9CE5c(d6`0n91hPmr()FfNB9
zcITO!^5MfL2+AU*h*S5803QYa>s34}>*ys{fRUYQhRTwwsZ%~wmQzz1eT(HpyB!TL
zP6;RCd1~6FWi&BCP!$O;TZWTei&6kz)U7%^J=Joj6h+WiJWf%|^$P&gxRTu`eaO
z_ZZJ`{mOxYa1S3p!E?9Nfm4zE9uHy1|9qy0$A1k1f!pqoZIF#(6v)
z>vK@Ii$GK2vgY2ow9XcApcWI?Y04ksVWBTVri3!0axq;wfL~m+WXQ4nlfay6qpU{q
ziRUZwH~;*nrA$F;1`m-qU;tBup!~zP#`5Gb_H9se0>TCgN3u~0$MJF?Lgt*EI5glk
z!h&yZA$W<`Ac>Mr7UujCM4?en^A^1LGdKPHdiSrjAx<9SsRq@m>jN
zabH&IB^}E~rzKAw^>wY0`N`iANLMFE2v%B+6*E^HGj_=%5JGbu`k-8aHLEwbOePMH
zv!SxdG+w}KDNygl=mY1x@Su0KD^e&4(4JDGtQjba>J&kbh?ST+05ozyKXBP
zt2MV|h5$8G(}azjYF}FSCYOzMueU=FK{UItB6y{v#ZD~A!ocBVej#PVxwl_?Uf%lh
zd$O9!)6ka`B?i3TaUjQcCvtEHHYs1}!5X`{@pNem1jN{xDwrJ?-lql)p=tei=7eh3
zWU13Sa!fvV{}#)-QVV6eth3@eWCT#7sr>BETtgs!U%vW{iJV-Ref`9XyYk+9C(`Wp
z02-`RFOiK+-h6W)pMK_=;wl$IB!+(No+gC;!cSe3^WzJ7e45D>*ztQ0kL1eMo?IKu
zW!z6yCy2nyey^%b-Kh#!;XDEeQX|U}08kI(d^nuSl;Bkmx$H5{GA1uD_9K7@3>oHh
zdR7MRHsOG>i*ByRCbbyH@*^+xf|l7
zWT>nRtH_n_D65*yCtySCuzlySkL=c)C#JWkI+F^H?af30}J2@VDR3l1Cr>zuiNWFW3^N(wBsvRZW^fml>(Z&=NvCEaINFv
zqqG22J-^eE7k8U-ty`C^Mn${yMf6Vy8j)#+`=8?V=i?Q|ZYtlrdnWhKri6Qqp_7MR
z;>(z$yl0*Tw6(cu+VBRbH$$Bu29c^o2ylrFWpX6fdNu0gW!gA{<#6-~=6a^s-e-9N
z*ULn86=Q%tp)#?o0S4?ceKX1lz^{qR6iR2)m!aJ5<#N3m%QvycS~%;bOakC6y}C^u
z&6R>eb(Xc@y4L|{_2KJWZ8uc@fS*SvUCYhj5cEe2Wo2uJSEs#VyKysk3^x9eDd(Xu|*E-d^hYNtbqKt0srXQ(X&JuGFx;
z*(hjgC2n|bYRYiUdeMC?D<$K*T+<|8Vnk;>LkL)H*Af7gOJ_gV0XmB;Ybr8^wI-ZP_S{@=>>x;g0_e5rVTLQcPR8!d
zWDk*P2me`U}4*XxShxFH5WLg9$g8liQOS6x=x0j$TJZX_?id`%uAah>GhI=;%Cw1K;}_qW!G
zGAX)C=I&nTOQk-+LRks;Ia+?sQfAj7Q7`8bVhS3%B%OA@UMv;7(vnan#uuwX_ex)@
ziQv$QLOWfx+`r%MB}tQl4bNs)7JaAAKX9e3JbMZBI}AT^dX?&Hz$TUv3)TQUN9%+8
z637t-my7wlUeCwH#hbvDP3!tKn$XhqE|$P8(sFs>XFSdLDkeuZjF0?a0LDP;%b?o&
zcOUcm+~z~i3MS+E{IDOQdc;dtuAV4!ig6r4EBctK9j)5Xq55KaCUBn;ICm(BCTG}`
zsZ!Co(k2LG*5Vos#Jl-Z;6=VNpi1=}oog8RBrN|2n_ij)a`!jCD`DAuifm;%wEV}x
zr=OMUpMBQFB=RR;A_@|p+&h*@A2IvhmNd3o243asl08if7jqG4;>@$IwJC)yK5_jH
zB!m5)A$d4MjG4M_t+XcGw*>G#fc=B9rTi&SYx6hmw0-(lfvb360u=XLyLzx@V0|35
z>)#Z-`h(V>alF#-V`*qPnw0<$Y#^Ahat^2~o!0Dxzi%E|1X7rd&ym&q-jj)Z`AI5oBWv{N
zbRwU4ZcC~-lzQb%ZnhDa;$Y|D8EY4z$kqSnnw5x(n|FfIj+N4%XY$ntGx^3zE?<7@
zLiTre<-AW}*&HBHS=l+~1AsDcDyq2V*=Gu()
z*yN!`W?juPZq)6{aw{XO@UzpE&J(+jeEySHdO|3&F*=SQ7g56d}9hem0
z)$=zZ`QQDGFCko-=z5~ln7amf*{Uq>A>clK%(4Np;yla2oB-A5&@xSzwV}9n964He
zWEpS7s?8m|H#R_ldy;E#HCySy>9ebz=AC+djKHzLoIPT8nOT3;!EmKZ71v>qpV#SH
zC!|0FiKSzZCE5kfn1*H;2udpBD^H5dVy-24>x@{z@}eU}>TYUIoR&o~(3o0jY7|)Z
zR>f1D(5v3Dql