Add text input handling to Gemini multimodal service

- Updated `RealtimeInput` to include an optional `text` parameter.
- Introduced `TextInputMessage` class for encapsulating text input data.
- Implemented `_send_user_text` method to send text input to the Gemini Live API.
- Enhanced message processing to support text input alongside media chunks.
This commit is contained in:
Pete
2025-07-20 17:39:31 -04:00
parent d4e33663b2
commit bd7a0f27cc
2 changed files with 35 additions and 2 deletions

View File

@@ -733,6 +733,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
# Support just one tool call per context frame for now
tool_result_message = context.messages[-1]
await self._tool_result(tool_result_message)
elif isinstance(frame, LLMTextFrame):
await self._send_user_text(frame.text)
elif isinstance(frame, InputAudioRawFrame):
await self._send_user_audio(frame)
await self.push_frame(frame, direction)
@@ -964,6 +966,17 @@ class GeminiMultimodalLiveLLMService(LLMService):
length = int((frame.sample_rate * frame.num_channels * 2) * 0.5)
self._user_audio_buffer = self._user_audio_buffer[-length:]
async def _send_user_text(self, text: str):
"""Send user text to Gemini Live API."""
logger.debug(f"Sending text to Gemini: {text}")
evt = events.TextInputMessage.from_text(text)
await self.send_client_event(evt)
# After sending text, we need to signal that the turn is complete.
evt = events.ClientContentMessage.model_validate(
{"clientContent": {"turnComplete": True}}
)
await self.send_client_event(evt)
async def _send_user_video(self, frame):
"""Send user video frame to Gemini Live API."""
if self._video_input_paused: