Add support for universal LLMContext to RTVIObserver

This commit is contained in:
Paul Kompfner
2025-08-19 16:24:21 -04:00
parent 37be8805f4
commit ecc4cc4a79

View File

@@ -42,6 +42,7 @@ from pipecat.frames.frames import (
FunctionCallResultFrame,
InputAudioRawFrame,
InterimTranscriptionFrame,
LLMContextFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesAppendFrame,
@@ -916,7 +917,10 @@ class RTVIObserver(BaseObserver):
and self._params.user_transcription_enabled
):
await self._handle_user_transcriptions(frame)
elif isinstance(frame, OpenAILLMContextFrame) and self._params.user_llm_enabled:
elif (
isinstance(frame, (OpenAILLMContextFrame, LLMContextFrame))
and self._params.user_llm_enabled
):
await self._handle_context(frame)
elif isinstance(frame, LLMFullResponseStartFrame) and self._params.bot_llm_enabled:
await self.push_transport_message_urgent(RTVIBotLLMStartedMessage())
@@ -1017,16 +1021,20 @@ class RTVIObserver(BaseObserver):
if message:
await self.push_transport_message_urgent(message)
async def _handle_context(self, frame: OpenAILLMContextFrame):
async def _handle_context(self, frame: OpenAILLMContextFrame | LLMContextFrame):
"""Process LLM context frames to extract user messages for the RTVI client."""
try:
messages = frame.context.messages
if isinstance(frame, OpenAILLMContextFrame):
messages = frame.context.messages
else:
messages = frame.context.get_messages()
if not messages:
return
message = messages[-1]
# Handle Google LLM format (protobuf objects with attributes)
# Note: not possible if frame is a universal LLMContextFrame
if hasattr(message, "role") and message.role == "user" and hasattr(message, "parts"):
text = "".join(part.text for part in message.parts if hasattr(part, "text"))
if text: