Merge pull request #3944 from pipecat-ai/mb/gemini-live-settings-examples-fixes

examples/foundational/26-gemini-live.py

@@ -18,7 +18,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.audio.vad_processor import VADProcessor
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
 from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -64,8 +64,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
 
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        system_instruction=system_instruction,
-        voice_id="Puck",  # Aoede, Charon, Fenrir, Kore, Puck
+        settings=GeminiLiveLLMSettings(
+            system_instruction=system_instruction,
+            voice="Puck",  # Aoede, Charon, Fenrir, Kore, Puck
+        ),
     )
 
     vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)))

examples/foundational/26a-gemini-live-transcription.py

@@ -67,16 +67,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
                 "role": "user",
                 "content": "Say hello. Then ask if I want to hear a joke.",
             },
-            #     {"role": "assistant", "content": "Hello! Why don't scientists trust atoms?"},
-            #     {
-            #         "role": "user",
-            #         "content": [
-            #             {
-            #                 "type": "text",
-            #                 "text": "Oh, I know this one: because they make up everything.",
-            #             }
-            #         ],
-            #     },
         ],
     )
     user_aggregator, assistant_aggregator = LLMContextAggregatorPair(

examples/foundational/26b-gemini-live-function-calling.py

@@ -26,7 +26,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
 )
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.services.llm_service import FunctionCallParams
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
@@ -120,7 +120,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
 
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        system_instruction=system_instruction,
+        settings=GeminiLiveLLMSettings(
+            system_instruction=system_instruction,
+        ),
         tools=tools,
     )
 

examples/foundational/26c-gemini-live-video.py

@@ -28,7 +28,7 @@ from pipecat.runner.utils import (
     maybe_capture_participant_camera,
     maybe_capture_participant_screen,
 )
-from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
 
@@ -53,8 +53,10 @@ transport_params = {
 async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        voice_id="Aoede",  # Puck, Charon, Kore, Fenrir, Aoede
-        # system_instruction="Talk like a pirate."
+        settings=GeminiLiveLLMSettings(
+            voice="Aoede",  # Puck, Charon, Kore, Fenrir, Aoede
+            # system_instruction="Talk like a pirate."
+        ),
         # inference_on_context_initialization=False,
     )
 

examples/foundational/26d-gemini-live-text.py

@@ -26,6 +26,7 @@ from pipecat.runner.utils import create_transport
 from pipecat.services.cartesia.tts import CartesiaTTSService, CartesiaTTSSettings
 from pipecat.services.google.gemini_live.llm import (
     GeminiLiveLLMService,
+    GeminiLiveLLMSettings,
     GeminiModalities,
     InputParams,
 )
@@ -74,9 +75,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     # https://cloud.google.com/vertex-ai/generative-ai/docs/live-api/tools#native-audio).
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        system_instruction=SYSTEM_INSTRUCTION,
+        settings=GeminiLiveLLMSettings(
+            system_instruction=SYSTEM_INSTRUCTION,
+            modalities=GeminiModalities.TEXT,
+        ),
         tools=[{"google_search": {}}, {"code_execution": {}}],
-        params=InputParams(modalities=GeminiModalities.TEXT),
     )
 
     # Optionally, you can set the response modalities via a function

examples/foundational/26e-gemini-live-google-search.py

@@ -23,7 +23,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
 )
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
 from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -73,8 +73,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     # Initialize the Gemini Multimodal Live model
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        voice_id="Puck",  # Aoede, Charon, Fenrir, Kore, Puck
-        system_instruction=system_instruction,
+        settings=GeminiLiveLLMSettings(
+            voice="Puck",  # Aoede, Charon, Fenrir, Kore, Puck
+            system_instruction=system_instruction,
+        ),
         tools=tools,
     )
 

examples/foundational/26f-gemini-live-files-api.py

@@ -23,7 +23,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
 )
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
 from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -110,9 +110,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     # Initialize Gemini service with File API support
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        system_instruction=system_instruction,
-        voice_id="Charon",  # Aoede, Charon, Fenrir, Kore, Puck
-        transcribe_user_audio=True,
+        settings=GeminiLiveLLMSettings(
+            system_instruction=system_instruction,
+            voice="Charon",  # Aoede, Charon, Fenrir, Kore, Puck
+        ),
     )
 
     # Upload the sample file to Gemini File API

examples/foundational/26g-gemini-live-groundingMetadata.py

@@ -19,7 +19,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
 from pipecat.services.google.frames import LLMSearchResponseFrame
-from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
 from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -107,9 +107,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
 
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        system_instruction=SYSTEM_INSTRUCTION,
-        voice_id="Charon",  # Aoede, Charon, Fenrir, Kore, Puck
-        transcribe_user_audio=True,
+        settings=GeminiLiveLLMSettings(
+            system_instruction=SYSTEM_INSTRUCTION,
+            voice="Charon",  # Aoede, Charon, Fenrir, Kore, Puck
+        ),
         tools=tools,
     )
 

examples/foundational/26h-gemini-live-vertex-function-calling.py

@@ -26,6 +26,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
 )
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMSettings
 from pipecat.services.google.gemini_live.llm_vertex import GeminiLiveVertexLLMService
 from pipecat.services.llm_service import FunctionCallParams
 from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -117,8 +118,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
         credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"),
         project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
         location=os.getenv("GOOGLE_CLOUD_LOCATION"),
-        system_instruction=system_instruction,
-        voice_id="Puck",  # Aoede, Charon, Fenrir, Kore, Puck
+        settings=GeminiLiveLLMSettings(
+            system_instruction=system_instruction,
+            voice="Puck",  # Aoede, Charon, Fenrir, Kore, Puck
+        ),
         tools=tools,
     )
 

examples/foundational/26i-gemini-live-graceful-end.py

@@ -26,7 +26,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.services.llm_service import FunctionCallParams
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
@@ -132,7 +132,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
 
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        system_instruction=system_instruction,
+        settings=GeminiLiveLLMSettings(
+            system_instruction=system_instruction,
+        ),
         tools=tools,
     )
 

examples/foundational/55zm-update-settings-gemini-live-vertex.py

@@ -53,7 +53,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
         credentials=os.getenv("GOOGLE_VERTEX_TEST_CREDENTIALS"),
         project_id=os.getenv("GOOGLE_CLOUD_PROJECT_ID"),
         location=os.getenv("GOOGLE_CLOUD_LOCATION"),
-        system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
+        settings=GeminiLiveLLMSettings(
+            system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
+        ),
     )
 
     context = LLMContext()
@@ -81,6 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     @transport.event_handler("on_client_connected")
     async def on_client_connected(transport, client):
         logger.info(f"Client connected")
+        context.add_message({"role": "user", "content": "Please introduce yourself to the user."})
         await task.queue_frames([LLMRunFrame()])
 
         await asyncio.sleep(10)

examples/foundational/55zm-update-settings-gemini-live.py

@@ -19,10 +19,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext
 from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
 from pipecat.runner.types import RunnerArguments
 from pipecat.runner.utils import create_transport
-from pipecat.services.google.gemini_live.llm import (
-    GeminiLiveLLMService,
-    GeminiLiveLLMSettings,
-)
+from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiLiveLLMSettings
 from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.transports.daily.transport import DailyParams
 from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -53,7 +50,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
 
     llm = GeminiLiveLLMService(
         api_key=os.getenv("GOOGLE_API_KEY"),
-        system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
+        settings=GeminiLiveLLMSettings(
+            system_instruction="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
+        ),
     )
 
     context = LLMContext()
@@ -81,6 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     @transport.event_handler("on_client_connected")
     async def on_client_connected(transport, client):
         logger.info(f"Client connected")
+        context.add_message({"role": "user", "content": "Please introduce yourself to the user."})
         await task.queue_frames([LLMRunFrame()])
 
         await asyncio.sleep(10)

src/pipecat/services/google/gemini_live/llm.py

@@ -607,7 +607,7 @@ class InputParams(BaseModel):
 
 @dataclass
 class GeminiLiveLLMSettings(LLMSettings):
-    """Settings for Gemini Live LLM services.
+    """Settings for GeminiLiveLLMService and GeminiLiveVertexLLMService.
 
     Parameters:
         voice: TTS voice identifier (e.g. ``"Charon"``).
@@ -717,6 +717,7 @@ class GeminiLiveLLMService(LLMService):
         # 1. Initialize default_settings with hardcoded defaults
         default_settings = GeminiLiveLLMSettings(
             model="models/gemini-2.5-flash-native-audio-preview-12-2025",
+            system_instruction=system_instruction,
             voice="Charon",
             frequency_penalty=None,
             max_tokens=4096,
@@ -785,7 +786,6 @@ class GeminiLiveLLMService(LLMService):
 
         self._last_sent_time = 0
         self._base_url = base_url
-        self._language_code = params.language if params is not None else Language.EN_US
 
         self._system_instruction_from_init = system_instruction
         self._tools_from_init = tools
@@ -815,11 +815,13 @@ class GeminiLiveLLMService(LLMService):
 
         self._sample_rate = 24000
 
-        self._language = _params.language
+        self._language = self._settings.language
         self._language_code = (
-            language_to_gemini_language(_params.language) if _params.language else "en-US"
+            language_to_gemini_language(self._settings.language)
+            if self._settings.language
+            else "en-US"
         )
-        self._vad_params = _params.vad
+        self._vad_params = self._settings.vad
 
         # Reconnection tracking
         self._consecutive_failures = 0