From 503e5e910662f289427bad2749d023be84fd104e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 25 Mar 2026 18:00:13 -0400 Subject: [PATCH 1/2] Fix Gemini Live local VAD by sending correct activity events to server When Gemini Live was configured with local VAD (server-side VAD disabled), the service was listening for the wrong frame types and not sending ActivityStart/ActivityEnd events to the server. Now it listens for VADUserStartedSpeakingFrame/VADUserStoppedSpeakingFrame and sends the appropriate activity signals when local VAD is in use. Also removes the unnecessary local SileroVADAnalyzer from server-side VAD examples and adds a new 26a example demonstrating local VAD configuration. --- examples/foundational/26-gemini-live.py | 15 +- .../foundational/26a-gemini-live-local-vad.py | 136 ++++++++++++++++++ .../26b-gemini-live-function-calling.py | 19 +-- .../foundational/26c-gemini-live-video.py | 19 +-- examples/foundational/26d-gemini-live-text.py | 19 +-- .../26e-gemini-live-google-search.py | 19 +-- .../foundational/26f-gemini-live-files-api.py | 20 +-- .../26g-gemini-live-groundingMetadata.py | 19 +-- ...26h-gemini-live-vertex-function-calling.py | 19 +-- .../26i-gemini-live-graceful-end.py | 19 +-- scripts/evals/run-release-evals.py | 1 + .../services/google/gemini_live/llm.py | 25 +++- 12 files changed, 181 insertions(+), 149 deletions(-) create mode 100644 examples/foundational/26a-gemini-live-local-vad.py diff --git a/examples/foundational/26-gemini-live.py b/examples/foundational/26-gemini-live.py index 90b5f0d13..dc9e8e55a 100644 --- a/examples/foundational/26-gemini-live.py +++ b/examples/foundational/26-gemini-live.py @@ -10,8 +10,6 @@ import os from dotenv import load_dotenv from loguru import logger -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -20,7 +18,6 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( AssistantTurnStoppedMessage, LLMContextAggregatorPair, - LLMUserAggregatorParams, UserTurnStoppedMessage, ) from pipecat.runner.types import RunnerArguments @@ -71,16 +68,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ], ) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26a-gemini-live-local-vad.py b/examples/foundational/26a-gemini-live-local-vad.py new file mode 100644 index 000000000..7a73742e9 --- /dev/null +++ b/examples/foundational/26a-gemini-live-local-vad.py @@ -0,0 +1,136 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + AssistantTurnStoppedMessage, + LLMContextAggregatorPair, + LLMUserAggregatorParams, + UserTurnStoppedMessage, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiVADParams +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +# We use lambdas to defer transport parameter creation until the transport +# type is selected at runtime. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + llm = GeminiLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + settings=GeminiLiveLLMService.Settings( + voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede + vad=GeminiVADParams(disabled=True), + ), + # inference_on_context_initialization=False, + ) + + context = LLMContext( + [ + { + "role": "user", + "content": "Say hello. Then ask if I want to hear a joke.", + }, + ], + ) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + @user_aggregator.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(aggregator, strategy, message: UserTurnStoppedMessage): + timestamp = f"[{message.timestamp}] " if message.timestamp else "" + line = f"{timestamp}user: {message.content}" + logger.info(f"Transcript: {line}") + + @assistant_aggregator.event_handler("on_assistant_turn_stopped") + async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage): + timestamp = f"[{message.timestamp}] " if message.timestamp else "" + line = f"{timestamp}assistant: {message.content}" + logger.info(f"Transcript: {line}") + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/26b-gemini-live-function-calling.py b/examples/foundational/26b-gemini-live-function-calling.py index edc5f30b2..a370fdf57 100644 --- a/examples/foundational/26b-gemini-live-function-calling.py +++ b/examples/foundational/26b-gemini-live-function-calling.py @@ -13,17 +13,12 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService @@ -134,16 +129,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # trigger a (fast) reconnection when the GeminiLiveLLMService first # receives the context (i.e. when we send the LLMRunFrame below). context = LLMContext() - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26c-gemini-live-video.py b/examples/foundational/26c-gemini-live-video.py index 38626cb6d..269f22718 100644 --- a/examples/foundational/26c-gemini-live-video.py +++ b/examples/foundational/26c-gemini-live-video.py @@ -11,17 +11,12 @@ import os from dotenv import load_dotenv from loguru import logger -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -68,16 +63,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ], ) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26d-gemini-live-text.py b/examples/foundational/26d-gemini-live-text.py index 645b53b56..05b3bdce9 100644 --- a/examples/foundational/26d-gemini-live-text.py +++ b/examples/foundational/26d-gemini-live-text.py @@ -10,17 +10,12 @@ import os from dotenv import load_dotenv from loguru import logger -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -96,16 +91,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set up conversation context and management # The context_aggregator will automatically collect conversation context context = LLMContext(messages) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26e-gemini-live-google-search.py b/examples/foundational/26e-gemini-live-google-search.py index 0d124fa8d..7822bb085 100644 --- a/examples/foundational/26e-gemini-live-google-search.py +++ b/examples/foundational/26e-gemini-live-google-search.py @@ -10,17 +10,12 @@ import os from dotenv import load_dotenv from loguru import logger -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService @@ -88,16 +83,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): } ], ) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26f-gemini-live-files-api.py b/examples/foundational/26f-gemini-live-files-api.py index fd019fc2a..1871bb5bf 100644 --- a/examples/foundational/26f-gemini-live-files-api.py +++ b/examples/foundational/26f-gemini-live-files-api.py @@ -10,17 +10,12 @@ import tempfile from dotenv import load_dotenv from loguru import logger -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService @@ -162,17 +157,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ] ) - # Create context aggregator - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) # Build the pipeline pipeline = Pipeline( diff --git a/examples/foundational/26g-gemini-live-groundingMetadata.py b/examples/foundational/26g-gemini-live-groundingMetadata.py index 0285483e5..31fd7d832 100644 --- a/examples/foundational/26g-gemini-live-groundingMetadata.py +++ b/examples/foundational/26g-gemini-live-groundingMetadata.py @@ -4,17 +4,12 @@ from dotenv import load_dotenv from loguru import logger from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import Frame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -126,16 +121,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set up conversation context and management context = LLMContext(messages) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26h-gemini-live-vertex-function-calling.py b/examples/foundational/26h-gemini-live-vertex-function-calling.py index 3c9862d39..68add5bc9 100644 --- a/examples/foundational/26h-gemini-live-vertex-function-calling.py +++ b/examples/foundational/26h-gemini-live-vertex-function-calling.py @@ -13,17 +13,12 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.vertex.llm import GeminiLiveVertexLLMService @@ -128,16 +123,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) context = LLMContext([{"role": "developer", "content": "Say hello."}]) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/examples/foundational/26i-gemini-live-graceful-end.py b/examples/foundational/26i-gemini-live-graceful-end.py index 6ee89ef7f..21abcc496 100644 --- a/examples/foundational/26i-gemini-live-graceful-end.py +++ b/examples/foundational/26i-gemini-live-graceful-end.py @@ -12,17 +12,12 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import EndTaskFrame, LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport @@ -145,16 +140,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context = LLMContext( [{"role": "developer", "content": "Say hello."}], ) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - # Set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't - # really matter because we can only use the Multimodal Live API's - # phrase endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)) - ), - ) + # Server-side VAD is enabled by default; no local VAD is added. + user_aggregator, assistant_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 829f03538..86fcd8fd6 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -230,6 +230,7 @@ TESTS_22 = [ TESTS_26 = [ ("26-gemini-live.py", EVAL_SIMPLE_MATH), + ("26a-gemini-live-local-vad.py", EVAL_SIMPLE_MATH), ("26b-gemini-live-function-calling.py", EVAL_WEATHER), ("26c-gemini-live-video.py", EVAL_VISION_CAMERA), ("26e-gemini-live-google-search.py", EVAL_ONLINE_SEARCH), diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 2a0b9409d..7564d341b 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -54,8 +54,8 @@ from pipecat.frames.frames import ( TTSStoppedFrame, TTSTextFrame, UserImageRawFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, ) from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.llm_context import LLMContext @@ -88,6 +88,8 @@ try: from google.genai import Client from google.genai.live import AsyncSession from google.genai.types import ( + ActivityEnd, + ActivityStart, AudioTranscriptionConfig, AutomaticActivityDetection, Blob, @@ -522,7 +524,7 @@ class GeminiVADParams(BaseModel): """Voice Activity Detection parameters for Gemini Live. Parameters: - disabled: Whether to disable VAD. Defaults to None. + disabled: Whether to disable VAD. Defaults to None (server-side VAD is enabled). start_sensitivity: Sensitivity for speech start detection. Defaults to None. end_sensitivity: Sensitivity for speech end detection. Defaults to None. prefix_padding_ms: Prefix padding in milliseconds. Defaults to None. @@ -828,7 +830,7 @@ class GeminiLiveLLMService(LLMService): if self._settings.language else "en-US" ) - self._vad_params = self._settings.vad + self._vad_disabled = bool(self._settings.vad and self._settings.vad.disabled) # Reconnection tracking self._consecutive_failures = 0 @@ -994,12 +996,21 @@ class GeminiLiveLLMService(LLMService): async def _handle_user_started_speaking(self, frame): self._user_is_speaking = True - pass + if self._vad_disabled and self._session: + try: + await self._session.send_realtime_input(activity_start=ActivityStart()) + except Exception as e: + await self._handle_send_error(e) async def _handle_user_stopped_speaking(self, frame): self._user_is_speaking = False self._user_audio_buffer = bytearray() await self.start_ttfb_metrics() + if self._vad_disabled and self._session: + try: + await self._session.send_realtime_input(activity_end=ActivityEnd()) + except Exception as e: + await self._handle_send_error(e) if self._needs_initial_turn_complete_message: self._needs_initial_turn_complete_message = False # NOTE: without this, the model ignores the context it's been @@ -1049,10 +1060,10 @@ class GeminiLiveLLMService(LLMService): elif isinstance(frame, InterruptionFrame): await self._handle_interruption() await self.push_frame(frame, direction) - elif isinstance(frame, UserStartedSpeakingFrame): + elif isinstance(frame, VADUserStartedSpeakingFrame): await self._handle_user_started_speaking(frame) await self.push_frame(frame, direction) - elif isinstance(frame, UserStoppedSpeakingFrame): + elif isinstance(frame, VADUserStoppedSpeakingFrame): await self._handle_user_stopped_speaking(frame) await self.push_frame(frame, direction) elif isinstance(frame, BotStartedSpeakingFrame): From 9db15e7942703b759bf397fc6acb70cf0726cf70 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 25 Mar 2026 18:07:05 -0400 Subject: [PATCH 2/2] Add changelog entry for #4146 --- changelog/4146.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4146.fixed.md diff --git a/changelog/4146.fixed.md b/changelog/4146.fixed.md new file mode 100644 index 000000000..db58dad06 --- /dev/null +++ b/changelog/4146.fixed.md @@ -0,0 +1 @@ +- Fixed Gemini Live local VAD mode (`GeminiVADParams(disabled=True)` with external VAD) not working. The bot now correctly detects user speech and signals turn boundaries to the Gemini API.