From 38d7882f0fd851d15ef25194a5d3c301d9d15bc9 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Wed, 18 Mar 2026 21:28:17 +0000 Subject: [PATCH 1/8] updated context seeding to allow gemini 3.1 to greet the user --- pyproject.toml | 2 +- src/pipecat/services/google/gemini_live/llm.py | 8 +++++--- uv.lock | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 91afcc794..447041417 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ fal = [] fireworks = [] fish = [ "ormsgpack>=1.7.0,<2", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] -google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.57.0,<2", "pipecat-ai[websockets-base]" ] +google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.68.0,<2", "pipecat-ai[websockets-base]" ] gradium = [ "pipecat-ai[websockets-base]" ] grok = [] groq = [ "groq>=0.23.0,<2" ] diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 5c9e5f8b3..121abe468 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -98,6 +98,7 @@ try: FunctionResponse, GenerationConfig, GroundingMetadata, + HistoryConfig, HttpOptions, LiveConnectConfig, LiveServerMessage, @@ -1204,6 +1205,7 @@ class GeminiLiveLLMService(LLMService): input_audio_transcription=AudioTranscriptionConfig(), output_audio_transcription=AudioTranscriptionConfig(), session_resumption=SessionResumptionConfig(handle=session_resumption_handle), + history_config=HistoryConfig(initial_history_in_client_content=True), ) # Add context window compression to configuration, if enabled @@ -1505,9 +1507,9 @@ class GeminiLiveLLMService(LLMService): await self.start_ttfb_metrics() try: - await self._session.send_client_content( - turns=messages, turn_complete=self._inference_on_context_initialization - ) + await self._session.send_client_content(turns=messages, turn_complete=False) + if self._inference_on_context_initialization: + await self._session.send_realtime_input(text=" ") except Exception as e: await self._handle_send_error(e) diff --git a/uv.lock b/uv.lock index 1263097a8..070cd38e1 100644 --- a/uv.lock +++ b/uv.lock @@ -2086,7 +2086,7 @@ wheels = [ [[package]] name = "google-genai" -version = "1.67.0" +version = "1.68.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2100,9 +2100,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/07/59a498f81f2c7b0649eacda2ea470b7fd8bd7149f20caba22962081bdd51/google_genai-1.67.0.tar.gz", hash = "sha256:897195a6a9742deb6de240b99227189ada8b2d901d61bdfba836c3092021eab6", size = 506972, upload-time = "2026-03-12T20:39:16.241Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/2c/f059982dbcb658cc535c81bbcbe7e2c040d675f4b563b03cdb01018a4bc3/google_genai-1.68.0.tar.gz", hash = "sha256:ac30c0b8bc630f9372993a97e4a11dae0e36f2e10d7c55eacdca95a9fa14ca96", size = 511285, upload-time = "2026-03-18T01:03:18.243Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/c2/562aa1f086e53529ffbeb5b43d5d8bc42c1b968102b5e2163fad005ce298/google_genai-1.67.0-py3-none-any.whl", hash = "sha256:58b0484ff2d4335fa53c724b489e9f807fcca8115d9cdbd8fdf341121fbd6d2d", size = 733542, upload-time = "2026-03-12T20:39:14.615Z" }, + { url = "https://files.pythonhosted.org/packages/84/de/7d3ee9c94b74c3578ea4f88d45e8de9405902f857932334d81e89bce3dfa/google_genai-1.68.0-py3-none-any.whl", hash = "sha256:a1bc9919c0e2ea2907d1e319b65471d3d6d58c54822039a249fe1323e4178d15", size = 750912, upload-time = "2026-03-18T01:03:15.983Z" }, ] [[package]] @@ -4819,7 +4819,7 @@ requires-dist = [ { name = "faster-whisper", marker = "extra == 'whisper'", specifier = "~=1.2.1" }, { name = "google-cloud-speech", marker = "extra == 'google'", specifier = ">=2.33.0,<3" }, { name = "google-cloud-texttospeech", marker = "extra == 'google'", specifier = ">=2.31.0,<3" }, - { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.57.0,<2" }, + { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.68.0,<2" }, { name = "groq", marker = "extra == 'groq'", specifier = ">=0.23.0,<2" }, { name = "hume", marker = "extra == 'hume'", specifier = ">=0.11.2,<1" }, { name = "kokoro-onnx", marker = "extra == 'kokoro'", specifier = ">=0.5.0,<1" }, From 6cd39b8b42fd49a754ff03011d7e765d280f9282 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Wed, 18 Mar 2026 23:04:22 +0000 Subject: [PATCH 2/8] updates --- src/pipecat/services/google/gemini_live/llm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 121abe468..bcc45f433 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1507,7 +1507,10 @@ class GeminiLiveLLMService(LLMService): await self.start_ttfb_metrics() try: - await self._session.send_client_content(turns=messages, turn_complete=False) + await self._session.send_client_content( + turns=messages, turn_complete=self._inference_on_context_initialization + ) + # Gemini 3.1 wants turn_complete=True, but also won't run inference without a realtime input if self._inference_on_context_initialization: await self._session.send_realtime_input(text=" ") except Exception as e: From 4a0411cbc4c3b1a43796f7e0d22da0f18fe8f6f3 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Wed, 18 Mar 2026 23:23:45 +0000 Subject: [PATCH 3/8] disabled single responses for gemini 3 live models --- src/pipecat/services/google/gemini_live/llm.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index bcc45f433..97ad8a10d 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1527,6 +1527,14 @@ class GeminiLiveLLMService(LLMService): if self._disconnecting or not self._session: return + model = self._settings.model or "" + if "gemini-3" in model: + await self.push_error( + f"LLMMessagesAppendFrame is not supported with model '{model}'. " + "This model does not support send_client_content." + ) + return + # Create a throwaway context just for the purpose of getting messages # in the right format context = LLMContext(messages=messages_list) From 05157129e298431be2a053a325ef88835c45e924 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Wed, 18 Mar 2026 23:31:18 +0000 Subject: [PATCH 4/8] added changelog --- changelog/4078.changed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4078.changed.md diff --git a/changelog/4078.changed.md b/changelog/4078.changed.md new file mode 100644 index 000000000..1c5882753 --- /dev/null +++ b/changelog/4078.changed.md @@ -0,0 +1 @@ +- Upcoming Gemini Live models don't support replacing the context during a session, so creating a single response (by using an `LLMMessagesAppendFrame` without a context manager, for example) will return an error with those models. From 844555c520346f331620f2d544510ec3f3d9c49f Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Mon, 23 Mar 2026 18:31:36 +0000 Subject: [PATCH 5/8] removed old Gemini Live example --- ...ion.py => 26-gemini-live-transcription.py} | 0 examples/foundational/26-gemini-live.py | 133 ------------------ 2 files changed, 133 deletions(-) rename examples/foundational/{26a-gemini-live-transcription.py => 26-gemini-live-transcription.py} (100%) delete mode 100644 examples/foundational/26-gemini-live.py diff --git a/examples/foundational/26a-gemini-live-transcription.py b/examples/foundational/26-gemini-live-transcription.py similarity index 100% rename from examples/foundational/26a-gemini-live-transcription.py rename to examples/foundational/26-gemini-live-transcription.py diff --git a/examples/foundational/26-gemini-live.py b/examples/foundational/26-gemini-live.py deleted file mode 100644 index e8ca05407..000000000 --- a/examples/foundational/26-gemini-live.py +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright (c) 2024-2026, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMMessagesAppendFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.audio.vad_processor import VADProcessor -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -# Load environment variables -load_dotenv(override=True) - - -# We use lambdas to defer transport parameter creation until the transport -# type is selected at runtime. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - # Create the Gemini Multimodal Live LLM service - system_instruction = f""" - You are a helpful AI assistant. - Your goal is to demonstrate your capabilities in a helpful and engaging way. - Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. - Respond to what the user said in a creative and helpful way. - """ - - llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - settings=GeminiLiveLLMService.Settings( - system_instruction=system_instruction, - voice="Puck", # Aoede, Charon, Fenrir, Kore, Puck - ), - ) - - vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5))) - - # Build the pipeline - pipeline = Pipeline( - [ - transport.input(), - vad_processor, - llm, - transport.output(), - ] - ) - - # Configure the pipeline task - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - # Handle client connection event - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - await task.queue_frames( - [ - LLMMessagesAppendFrame( - messages=[ - { - "role": "user", - "content": f"Greet the user and introduce yourself.", - } - ] - ) - ] - ) - - # Handle client disconnection events - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - # Run the pipeline - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() From d314e2831a4b5c0beb54aea1407f31679e1c2b04 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 23 Mar 2026 15:46:13 -0400 Subject: [PATCH 6/8] Simplify 26 name, update evals --- .../{26-gemini-live-transcription.py => 26-gemini-live.py} | 0 scripts/evals/run-release-evals.py | 1 - 2 files changed, 1 deletion(-) rename examples/foundational/{26-gemini-live-transcription.py => 26-gemini-live.py} (100%) diff --git a/examples/foundational/26-gemini-live-transcription.py b/examples/foundational/26-gemini-live.py similarity index 100% rename from examples/foundational/26-gemini-live-transcription.py rename to examples/foundational/26-gemini-live.py diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 625f33564..4e88453f7 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -221,7 +221,6 @@ TESTS_22 = [ TESTS_26 = [ ("26-gemini-live.py", EVAL_SIMPLE_MATH), - ("26a-gemini-live-transcription.py", EVAL_SIMPLE_MATH), ("26b-gemini-live-function-calling.py", EVAL_WEATHER), ("26c-gemini-live-video.py", EVAL_VISION_CAMERA), ("26e-gemini-live-google-search.py", EVAL_ONLINE_SEARCH), From 8109ab613552ff19edfc7fded15b3879dcb7cb4a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 10:45:41 -0400 Subject: [PATCH 7/8] Further tweaks and improvements to Gemini 3 support in Gemini Live Gets Gemini 3 support to the point where it works with: - The "legacy" pattern from the previous (removed) 26- example - inference_on_context_initialization=True (the default) - inference_on_context_initialization=False --- changelog/4078.changed.md | 2 +- .../services/google/gemini_live/llm.py | 49 +++++++++++-------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/changelog/4078.changed.md b/changelog/4078.changed.md index 1c5882753..c0f4fb0a9 100644 --- a/changelog/4078.changed.md +++ b/changelog/4078.changed.md @@ -1 +1 @@ -- Upcoming Gemini Live models don't support replacing the context during a session, so creating a single response (by using an `LLMMessagesAppendFrame` without a context manager, for example) will return an error with those models. +- Added Gemini 3 support to the Gemini Live service. diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 97ad8a10d..18b2c2e8a 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -649,6 +649,11 @@ class GeminiLiveLLMService(LLMService): # Overriding the default adapter to use the Gemini one. adapter_class = GeminiLLMAdapter + @property + def _is_gemini_3(self) -> bool: + """Check if the current model is a Gemini 3.x model.""" + return "gemini-3" in (self._settings.model or "") + def __init__( self, *, @@ -792,7 +797,7 @@ class GeminiLiveLLMService(LLMService): self._system_instruction_from_init = system_instruction self._tools_from_init = tools self._inference_on_context_initialization = inference_on_context_initialization - self._needs_turn_complete_message = False + self._needs_initial_turn_complete_message = False self._audio_input_paused = start_audio_paused self._video_input_paused = start_video_paused @@ -994,8 +999,8 @@ class GeminiLiveLLMService(LLMService): self._user_is_speaking = False self._user_audio_buffer = bytearray() await self.start_ttfb_metrics() - if self._needs_turn_complete_message: - self._needs_turn_complete_message = False + if self._needs_initial_turn_complete_message: + self._needs_initial_turn_complete_message = False # NOTE: without this, the model ignores the context it's been # seeded with before the user started speaking await self._session.send_client_content(turn_complete=True) @@ -1057,9 +1062,10 @@ class GeminiLiveLLMService(LLMService): elif isinstance(frame, LLMMessagesAppendFrame): # NOTE: handling LLMMessagesAppendFrame here in the LLMService is # unusual - typically this would be handled in the user context - # aggregator. Leaving this handling here so that user code that - # uses this frame *without* a user context aggregator still works - # (we have an example that does just that, actually). + # aggregator. Leaving this handling here so that legacy user code + # that uses this frame *without* a user context aggregator to kick + # off a conversation still works (we used to have an example that + # did that). await self._create_single_response(frame.messages) elif isinstance(frame, LLMSetToolsFrame): # TODO: implement runtime tool updates for Gemini Live. @@ -1510,29 +1516,27 @@ class GeminiLiveLLMService(LLMService): await self._session.send_client_content( turns=messages, turn_complete=self._inference_on_context_initialization ) - # Gemini 3.1 wants turn_complete=True, but also won't run inference without a realtime input - if self._inference_on_context_initialization: + # Gemini 3.x wants turn_complete=True, but also won't run inference without a realtime input + if self._is_gemini_3 and self._inference_on_context_initialization: await self._session.send_realtime_input(text=" ") except Exception as e: await self._handle_send_error(e) # If we're generating a response right away upon initializing - # conversation history, set a flag saying that we need a turn complete - # message when the user stops speaking. - if not self._inference_on_context_initialization: - self._needs_turn_complete_message = True + # conversation history, set a flag saying that we'll need a turn + # complete message when the user stops speaking. + # This is a quirky workaround, and not one that Gemini 3 needs. + if not self._inference_on_context_initialization and not self._is_gemini_3: + self._needs_initial_turn_complete_message = True async def _create_single_response(self, messages_list): - """Create a single response from a list of messages.""" - if self._disconnecting or not self._session: - return + """Create a single response from a list of messages. - model = self._settings.model or "" - if "gemini-3" in model: - await self.push_error( - f"LLMMessagesAppendFrame is not supported with model '{model}'. " - "This model does not support send_client_content." - ) + This is only here to support the very specific 'legacy' scenario of + kicking off a conversation using LLMMessagesAppendFrame when there's no + context aggregators in the pipeline (see process_frame for more details). + """ + if self._disconnecting or not self._session: return # Create a throwaway context just for the purpose of getting messages @@ -1550,6 +1554,9 @@ class GeminiLiveLLMService(LLMService): try: await self._session.send_client_content(turns=messages, turn_complete=True) + # Gemini 3.x wants turn_complete=True, but also won't run inference without a realtime input + if self._is_gemini_3: + await self._session.send_realtime_input(text=" ") except Exception as e: await self._handle_send_error(e) From 68a440ae2e28fde6e46b1c2122344bb67ccf21f6 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 10:49:45 -0400 Subject: [PATCH 8/8] Move inference_on_context_initialization comment to constructor level --- examples/foundational/26-gemini-live.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/26-gemini-live.py b/examples/foundational/26-gemini-live.py index 5d9adbc7f..90b5f0d13 100644 --- a/examples/foundational/26-gemini-live.py +++ b/examples/foundational/26-gemini-live.py @@ -59,8 +59,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): settings=GeminiLiveLLMService.Settings( voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." - # inference_on_context_initialization=False, ), + # inference_on_context_initialization=False, ) context = LLMContext(