wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for universal LLMContext to RTVIObserver

Browse Source
This commit is contained in:
Paul Kompfner
2025-08-19 16:24:21 -04:00
parent 37be8805f4
commit ecc4cc4a79
1 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -42,6 +42,7 @@ from pipecat.frames.frames import (
FunctionCallResultFrame, FunctionCallResultFrame,
InputAudioRawFrame, InputAudioRawFrame,
InterimTranscriptionFrame, InterimTranscriptionFrame,
LLMContextFrame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
LLMFullResponseStartFrame, LLMFullResponseStartFrame,
LLMMessagesAppendFrame, LLMMessagesAppendFrame,
@@ -916,7 +917,10 @@ class RTVIObserver(BaseObserver):
and self._params.user_transcription_enabled and self._params.user_transcription_enabled
): ):
await self._handle_user_transcriptions(frame) await self._handle_user_transcriptions(frame)
elif isinstance(frame, OpenAILLMContextFrame) and self._params.user_llm_enabled: elif (
isinstance(frame, (OpenAILLMContextFrame, LLMContextFrame))
and self._params.user_llm_enabled
):
await self._handle_context(frame) await self._handle_context(frame)
elif isinstance(frame, LLMFullResponseStartFrame) and self._params.bot_llm_enabled: elif isinstance(frame, LLMFullResponseStartFrame) and self._params.bot_llm_enabled:
await self.push_transport_message_urgent(RTVIBotLLMStartedMessage()) await self.push_transport_message_urgent(RTVIBotLLMStartedMessage())
@@ -1017,16 +1021,20 @@ class RTVIObserver(BaseObserver):
if message: if message:
await self.push_transport_message_urgent(message) await self.push_transport_message_urgent(message)
async def _handle_context(self, frame: OpenAILLMContextFrame): async def _handle_context(self, frame: OpenAILLMContextFrame | LLMContextFrame):
"""Process LLM context frames to extract user messages for the RTVI client.""" """Process LLM context frames to extract user messages for the RTVI client."""
try: try:
messages = frame.context.messages if isinstance(frame, OpenAILLMContextFrame):
messages = frame.context.messages
else:
messages = frame.context.get_messages()
if not messages: if not messages:
return return
message = messages[-1] message = messages[-1]
# Handle Google LLM format (protobuf objects with attributes) # Handle Google LLM format (protobuf objects with attributes)
# Note: not possible if frame is a universal LLMContextFrame
if hasattr(message, "role") and message.role == "user" and hasattr(message, "parts"): if hasattr(message, "role") and message.role == "user" and hasattr(message, "parts"):
text = "".join(part.text for part in message.parts if hasattr(part, "text")) text = "".join(part.text for part in message.parts if hasattr(part, "text"))
if text: if text: