Add InputTextRawFrame class for handling raw text input in frames
- Introduced `InputTextRawFrame` to represent raw text input from users or programs. - Updated `GeminiMultimodalLiveLLMService` to process `InputTextRawFrame` and send user text via the Gemini Live API's realtime input stream. - Enhanced `_send_user_text` method documentation for clarity on its functionality and usage.
This commit is contained in:
@@ -1074,6 +1074,20 @@ class InputImageRawFrame(SystemFrame, ImageRawFrame):
|
|||||||
return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {self.size}, format: {self.format})"
|
return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {self.size}, format: {self.format})"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class InputTextRawFrame(SystemFrame, TextFrame):
|
||||||
|
"""Raw text input frame from transport.
|
||||||
|
|
||||||
|
Text input usually coming from user typing or programmatic text injection
|
||||||
|
that should be sent to LLM services as input, similar to how InputAudioRawFrame
|
||||||
|
and InputImageRawFrame represent user audio and video input.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
pts = format_pts(self.pts)
|
||||||
|
return f"{self.name}(pts: {pts}, source: {self.transport_source}, text: [{self.text}])"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class UserAudioRawFrame(InputAudioRawFrame):
|
class UserAudioRawFrame(InputAudioRawFrame):
|
||||||
"""Raw audio input frame associated with a specific user.
|
"""Raw audio input frame associated with a specific user.
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from pipecat.frames.frames import (
|
|||||||
Frame,
|
Frame,
|
||||||
InputAudioRawFrame,
|
InputAudioRawFrame,
|
||||||
InputImageRawFrame,
|
InputImageRawFrame,
|
||||||
|
InputTextRawFrame,
|
||||||
LLMFullResponseEndFrame,
|
LLMFullResponseEndFrame,
|
||||||
LLMFullResponseStartFrame,
|
LLMFullResponseStartFrame,
|
||||||
LLMMessagesAppendFrame,
|
LLMMessagesAppendFrame,
|
||||||
@@ -733,8 +734,9 @@ class GeminiMultimodalLiveLLMService(LLMService):
|
|||||||
# Support just one tool call per context frame for now
|
# Support just one tool call per context frame for now
|
||||||
tool_result_message = context.messages[-1]
|
tool_result_message = context.messages[-1]
|
||||||
await self._tool_result(tool_result_message)
|
await self._tool_result(tool_result_message)
|
||||||
elif isinstance(frame, LLMTextFrame):
|
elif isinstance(frame, InputTextRawFrame):
|
||||||
await self._send_user_text(frame.text)
|
await self._send_user_text(frame.text)
|
||||||
|
await self.push_frame(frame, direction)
|
||||||
elif isinstance(frame, InputAudioRawFrame):
|
elif isinstance(frame, InputAudioRawFrame):
|
||||||
await self._send_user_audio(frame)
|
await self._send_user_audio(frame)
|
||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
@@ -967,7 +969,19 @@ class GeminiMultimodalLiveLLMService(LLMService):
|
|||||||
self._user_audio_buffer = self._user_audio_buffer[-length:]
|
self._user_audio_buffer = self._user_audio_buffer[-length:]
|
||||||
|
|
||||||
async def _send_user_text(self, text: str):
|
async def _send_user_text(self, text: str):
|
||||||
"""Send user text to Gemini Live API."""
|
"""Send user text via Gemini Live API's realtime input stream.
|
||||||
|
|
||||||
|
This method sends text through the realtimeInput stream (via TextInputMessage)
|
||||||
|
rather than the clientContent stream. This ensures text input is synchronized
|
||||||
|
with audio and video inputs, preventing temporal misalignment that can occur
|
||||||
|
when different modalities are processed through separate API pathways.
|
||||||
|
|
||||||
|
After sending the text, we signal turn completion to trigger a model response
|
||||||
|
for text-only interactions.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: The text to send as user input.
|
||||||
|
"""
|
||||||
evt = events.TextInputMessage.from_text(text)
|
evt = events.TextInputMessage.from_text(text)
|
||||||
await self.send_client_event(evt)
|
await self.send_client_event(evt)
|
||||||
# After sending text, we need to signal that the turn is complete.
|
# After sending text, we need to signal that the turn is complete.
|
||||||
|
|||||||
Reference in New Issue
Block a user