From bd7a0f27ccb3023547c0b8cce35d7bd2dfbf4985 Mon Sep 17 00:00:00 2001 From: Pete Date: Sun, 20 Jul 2025 17:39:31 -0400 Subject: [PATCH 1/5] Add text input handling to Gemini multimodal service - Updated `RealtimeInput` to include an optional `text` parameter. - Introduced `TextInputMessage` class for encapsulating text input data. - Implemented `_send_user_text` method to send text input to the Gemini Live API. - Enhanced message processing to support text input alongside media chunks. --- .../services/gemini_multimodal_live/events.py | 24 +++++++++++++++++-- .../services/gemini_multimodal_live/gemini.py | 13 ++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index 8fea91666..3fb04b147 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -114,13 +114,15 @@ class RealtimeInputConfig(BaseModel): class RealtimeInput(BaseModel): - """Contains realtime input media chunks. + """Contains realtime input media chunks and text. Parameters: mediaChunks: List of media chunks for realtime processing. + text: Text for realtime processing. """ - mediaChunks: List[MediaChunk] + mediaChunks: Optional[List[MediaChunk]] = None + text: Optional[str] = None class ClientContent(BaseModel): @@ -190,6 +192,24 @@ class VideoInputMessage(BaseModel): ) +class TextInputMessage(BaseModel): + """Message containing text input data.""" + + realtimeInput: RealtimeInput + + @classmethod + def from_text(cls, text: str) -> "TextInputMessage": + """Create a text input message from a string. + + Args: + text: The text to send. + + Returns: + A TextInputMessage instance. + """ + return cls(realtimeInput=RealtimeInput(text=text)) + + class ClientContentMessage(BaseModel): """Message containing client content for the API. diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 30bb3c529..9d9bbb0d5 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -733,6 +733,8 @@ class GeminiMultimodalLiveLLMService(LLMService): # Support just one tool call per context frame for now tool_result_message = context.messages[-1] await self._tool_result(tool_result_message) + elif isinstance(frame, LLMTextFrame): + await self._send_user_text(frame.text) elif isinstance(frame, InputAudioRawFrame): await self._send_user_audio(frame) await self.push_frame(frame, direction) @@ -964,6 +966,17 @@ class GeminiMultimodalLiveLLMService(LLMService): length = int((frame.sample_rate * frame.num_channels * 2) * 0.5) self._user_audio_buffer = self._user_audio_buffer[-length:] + async def _send_user_text(self, text: str): + """Send user text to Gemini Live API.""" + logger.debug(f"Sending text to Gemini: {text}") + evt = events.TextInputMessage.from_text(text) + await self.send_client_event(evt) + # After sending text, we need to signal that the turn is complete. + evt = events.ClientContentMessage.model_validate( + {"clientContent": {"turnComplete": True}} + ) + await self.send_client_event(evt) + async def _send_user_video(self, frame): """Send user video frame to Gemini Live API.""" if self._video_input_paused: From 1d69cd1a5ea42352d4279262f725897877a0f285 Mon Sep 17 00:00:00 2001 From: Pete Date: Sun, 20 Jul 2025 18:04:57 -0400 Subject: [PATCH 2/5] Remove debug logging from `_send_user_text` method in Gemini multimodal service --- src/pipecat/services/gemini_multimodal_live/gemini.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 9d9bbb0d5..1a47755bd 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -968,7 +968,6 @@ class GeminiMultimodalLiveLLMService(LLMService): async def _send_user_text(self, text: str): """Send user text to Gemini Live API.""" - logger.debug(f"Sending text to Gemini: {text}") evt = events.TextInputMessage.from_text(text) await self.send_client_event(evt) # After sending text, we need to signal that the turn is complete. From 8f74b97591951368db2d4996c93ce7d0a2a4e3d9 Mon Sep 17 00:00:00 2001 From: Pete Date: Sun, 20 Jul 2025 18:08:45 -0400 Subject: [PATCH 3/5] Refactor `_send_user_text` method in Gemini multimodal service to streamline event creation for turn completion --- src/pipecat/services/gemini_multimodal_live/gemini.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 1a47755bd..2e713c8c4 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -971,9 +971,7 @@ class GeminiMultimodalLiveLLMService(LLMService): evt = events.TextInputMessage.from_text(text) await self.send_client_event(evt) # After sending text, we need to signal that the turn is complete. - evt = events.ClientContentMessage.model_validate( - {"clientContent": {"turnComplete": True}} - ) + evt = events.ClientContentMessage.model_validate({"clientContent": {"turnComplete": True}}) await self.send_client_event(evt) async def _send_user_video(self, frame): From 8ca06e5887787848b069047d2d32314c12cb3cee Mon Sep 17 00:00:00 2001 From: Pete Date: Tue, 29 Jul 2025 17:43:14 -0400 Subject: [PATCH 4/5] Add `InputTextRawFrame` class for handling raw text input in frames - Introduced `InputTextRawFrame` to represent raw text input from users or programs. - Updated `GeminiMultimodalLiveLLMService` to process `InputTextRawFrame` and send user text via the Gemini Live API's realtime input stream. - Enhanced `_send_user_text` method documentation for clarity on its functionality and usage. --- src/pipecat/frames/frames.py | 14 ++++++++++++++ .../services/gemini_multimodal_live/gemini.py | 18 ++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 9d73b44ff..b92196a31 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1074,6 +1074,20 @@ class InputImageRawFrame(SystemFrame, ImageRawFrame): return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {self.size}, format: {self.format})" +@dataclass +class InputTextRawFrame(SystemFrame, TextFrame): + """Raw text input frame from transport. + + Text input usually coming from user typing or programmatic text injection + that should be sent to LLM services as input, similar to how InputAudioRawFrame + and InputImageRawFrame represent user audio and video input. + """ + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, source: {self.transport_source}, text: [{self.text}])" + + @dataclass class UserAudioRawFrame(InputAudioRawFrame): """Raw audio input frame associated with a specific user. diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 2e713c8c4..ea71cd10d 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -32,6 +32,7 @@ from pipecat.frames.frames import ( Frame, InputAudioRawFrame, InputImageRawFrame, + InputTextRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesAppendFrame, @@ -733,8 +734,9 @@ class GeminiMultimodalLiveLLMService(LLMService): # Support just one tool call per context frame for now tool_result_message = context.messages[-1] await self._tool_result(tool_result_message) - elif isinstance(frame, LLMTextFrame): + elif isinstance(frame, InputTextRawFrame): await self._send_user_text(frame.text) + await self.push_frame(frame, direction) elif isinstance(frame, InputAudioRawFrame): await self._send_user_audio(frame) await self.push_frame(frame, direction) @@ -967,7 +969,19 @@ class GeminiMultimodalLiveLLMService(LLMService): self._user_audio_buffer = self._user_audio_buffer[-length:] async def _send_user_text(self, text: str): - """Send user text to Gemini Live API.""" + """Send user text via Gemini Live API's realtime input stream. + + This method sends text through the realtimeInput stream (via TextInputMessage) + rather than the clientContent stream. This ensures text input is synchronized + with audio and video inputs, preventing temporal misalignment that can occur + when different modalities are processed through separate API pathways. + + After sending the text, we signal turn completion to trigger a model response + for text-only interactions. + + Args: + text: The text to send as user input. + """ evt = events.TextInputMessage.from_text(text) await self.send_client_event(evt) # After sending text, we need to signal that the turn is complete. From 2af3b6329d2038fa92d65a1ac2ec8c635331a1c9 Mon Sep 17 00:00:00 2001 From: Pete Date: Tue, 29 Jul 2025 17:48:11 -0400 Subject: [PATCH 5/5] Ruff format debug --- src/pipecat/frames/frames.py | 6 +++--- src/pipecat/services/gemini_multimodal_live/gemini.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index b92196a31..4815dae86 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1074,12 +1074,12 @@ class InputImageRawFrame(SystemFrame, ImageRawFrame): return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {self.size}, format: {self.format})" -@dataclass +@dataclass class InputTextRawFrame(SystemFrame, TextFrame): """Raw text input frame from transport. - + Text input usually coming from user typing or programmatic text injection - that should be sent to LLM services as input, similar to how InputAudioRawFrame + that should be sent to LLM services as input, similar to how InputAudioRawFrame and InputImageRawFrame represent user audio and video input. """ diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index ea71cd10d..b7adb8c49 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -970,15 +970,15 @@ class GeminiMultimodalLiveLLMService(LLMService): async def _send_user_text(self, text: str): """Send user text via Gemini Live API's realtime input stream. - + This method sends text through the realtimeInput stream (via TextInputMessage) rather than the clientContent stream. This ensures text input is synchronized with audio and video inputs, preventing temporal misalignment that can occur when different modalities are processed through separate API pathways. - + After sending the text, we signal turn completion to trigger a model response for text-only interactions. - + Args: text: The text to send as user input. """