From 737e4fa3bdcb80597c95667e0620caf05c514375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Sun, 26 Jan 2025 21:11:17 -0800
Subject: [PATCH] gemini(multimodal live): connect on StartFrame

---
 .../foundational/26-gemini-multimodal-live.py    | 16 ++++++++++++++++
 .../services/gemini_multimodal_live/gemini.py    | 16 ++++++++--------
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py
index 21c4b66f7..5f41833c8 100644
--- a/examples/foundational/26-gemini-multimodal-live.py
+++ b/examples/foundational/26-gemini-multimodal-live.py
@@ -15,6 +15,7 @@ from runner import configure
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import LLMMessagesAppendFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -71,6 +72,21 @@ async def main():
             ),
         )
 
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            await task.queue_frames(
+                [
+                    LLMMessagesAppendFrame(
+                        messages=[
+                            {
+                                "role": "assistant",
+                                "content": "Greet the user.",
+                            }
+                        ]
+                    )
+                ]
+            )
+
         runner = PipelineRunner()
 
         await runner.run(task)
diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py
index 770b64711..5fee17478 100644
--- a/src/pipecat/services/gemini_multimodal_live/gemini.py
+++ b/src/pipecat/services/gemini_multimodal_live/gemini.py
@@ -248,6 +248,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
+        await self._connect()
 
     async def stop(self, frame: EndFrame):
         await super().stop(frame)
@@ -385,13 +386,13 @@ class GeminiMultimodalLiveLLMService(LLMService):
         await self._ws_send(event.model_dump(exclude_none=True))
 
     async def _connect(self):
+        if self._websocket:
+            # Here we assume that if we have a websocket, we are connected. We
+            # handle disconnections in the send/recv code paths.
+            return
+
         logger.info("Connecting to Gemini service")
         try:
-            if self._websocket:
-                # Here we assume that if we have a websocket, we are connected. We
-                # handle disconnections in the send/recv code paths.
-                return
-
             uri = f"wss://{self.base_url}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={self.api_key}"
             logger.info(f"Connecting to {uri}")
             self._websocket = await websockets.connect(uri=uri)
@@ -464,9 +465,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
     async def _ws_send(self, message):
         # logger.debug(f"Sending message to websocket: {message}")
         try:
-            if not self._websocket:
-                await self._connect()
-            await self._websocket.send(json.dumps(message))
+            if self._websocket:
+                await self._websocket.send(json.dumps(message))
         except Exception as e:
             if self._disconnecting:
                 return