wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tiny bit more cleanup

Browse Source
This commit is contained in:
Kwindla Hultman Kramer
2024-09-07 12:38:17 -07:00
parent 37bbb687de
commit 6c58a5b024
1 changed files with 17 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/pipecat/services/together.py
View File

@@ -18,7 +18,7 @@ from pipecat.frames.frames import (
Frame, Frame,
LLMModelUpdateFrame, LLMModelUpdateFrame,
TextFrame, TextFrame,
UserImageRawFrame, UserImageRequestFrame,
LLMMessagesFrame, LLMMessagesFrame,
LLMFullResponseStartFrame, LLMFullResponseStartFrame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
@@ -249,17 +249,9 @@ class TogetherUserContextAggregator(LLMUserContextAggregator):
if isinstance(context, OpenAILLMContext): if isinstance(context, OpenAILLMContext):
self._context = TogetherLLMContext.from_openai_context(context) self._context = TogetherLLMContext.from_openai_context(context)
def get_messages_frame(self):
return OpenAILLMContextFrame(self._context)
async def push_messages_frame(self): async def push_messages_frame(self):
await self.push_frame(self.get_messages_frame()) frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)
def append_image_description_tool_message(self, description):
self._context.add_message({
"role": "tool",
"content": json.dumps({"image_description": description})
})
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
@@ -268,10 +260,20 @@ class TogetherUserContextAggregator(LLMUserContextAggregator):
# to talk through (tagging @aleix). At some point we might need to refactor these # to talk through (tagging @aleix). At some point we might need to refactor these
# context aggregators. # context aggregators.
try: try:
if isinstance(frame, UserImageRawFrame): if isinstance(frame, UserImageRequestFrame):
if frame.description: # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
self.append_image_description_tool_message(frame.description) # that frame so we can use it when we assemble the image message in the assistant
await self.push_messages_frame() # context aggregator.
if (frame.context):
if isinstance(frame.context, str):
self._context._user_image_request_context[frame.user_id] = frame.context
else:
logger.error(
f"Unexpected UserImageRequestFrame context type: {type(frame.context)}")
del self._context._user_image_request_context[frame.user_id]
else:
if frame.user_id in self._context._user_image_request_context:
del self._context._user_image_request_context[frame.user_id]
except Exception as e: except Exception as e:
logger.error(f"Error processing frame: {e}") logger.error(f"Error processing frame: {e}")