wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix for out-of-order image messages in anthropic context

Browse Source
This commit is contained in:
Kwindla Hultman Kramer
2024-08-17 14:47:29 -07:00
parent 5b4061b0d5
commit 17f8c93e44
1 changed files with 13 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/pipecat/services/anthropic.py
View File

@@ -322,6 +322,7 @@ class AnthropicLLMContext(OpenAILLMContext):
buffer = io.BytesIO() buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG") Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
# Anthropic docs say that the image should be the first content block in the message. # Anthropic docs say that the image should be the first content block in the message.
content = [{"type": "image", content = [{"type": "image",
"source": { "source": {
@@ -457,6 +458,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
self._user_context_aggregator = user_context_aggregator self._user_context_aggregator = user_context_aggregator
self._function_call_in_progress = None self._function_call_in_progress = None
self._function_call_result = None self._function_call_result = None
self._pending_image_frame_message = None
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
@@ -477,15 +479,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
self._function_call_in_progress = None self._function_call_in_progress = None
self._function_call_result = None self._function_call_result = None
elif isinstance(frame, AnthropicImageMessageFrame): elif isinstance(frame, AnthropicImageMessageFrame):
try: self._pending_image_frame_message = frame
self._context.add_image_frame_message(
format=frame.user_image_raw_frame.format,
size=frame.user_image_raw_frame.size,
image=frame.user_image_raw_frame.image,
text=frame.text)
await self._user_context_aggregator.push_context_frame()
except Exception as e:
logger.error(f"Error processing AnthropicImageMessageFrame: {e}")
def add_message(self, message): def add_message(self, message):
self._user_context_aggregator.add_message(message) self._user_context_aggregator.add_message(message)
@@ -532,6 +526,16 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
else: else:
self._context.add_message({"role": "assistant", "content": aggregation}) self._context.add_message({"role": "assistant", "content": aggregation})
if self._pending_image_frame_message:
frame = self._pending_image_frame_message
self._pending_image_frame_message = None
self._context.add_image_frame_message(
format=frame.user_image_raw_frame.format,
size=frame.user_image_raw_frame.size,
image=frame.user_image_raw_frame.image,
text=frame.text)
run_llm = True
if run_llm: if run_llm:
await self._user_context_aggregator.push_context_frame() await self._user_context_aggregator.push_context_frame()