From 9c5690d670e71503564e697e1399265321c51288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 15:54:28 -0700 Subject: [PATCH] LLMContext: added support for image messages with URLs --- CHANGELOG.md | 5 +-- .../adapters/services/anthropic_adapter.py | 26 ++++++++++---- .../adapters/services/bedrock_adapter.py | 23 +++++++----- .../adapters/services/gemini_adapter.py | 5 ++- .../processors/aggregators/llm_context.py | 36 ++++++++++++------- 5 files changed, 65 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 056c548ca..07596b3b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 guides the vision service on how to analyze the image. - Added support for including images or audio to LLM context messages using - `LLMContext.create_image_message()` and `LLMContext.create_audio_message()`. - For example, when creating `LLMMessagesAppendFrame`: + `LLMContext.create_image_message()` or `LLMContext.create_image_url_message()` + (not all LLMs support URLs) and `LLMContext.create_audio_message()`. For + example, when creating `LLMMessagesAppendFrame`: ```python message = LLMContext.create_image_message(image=..., size= ...) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index a106b4de4..75fa5899d 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -245,13 +245,25 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): item["text"] = "(empty)" # handle image_url -> image conversion if item["type"] == "image_url": - item["type"] = "image" - item["source"] = { - "type": "base64", - "media_type": "image/jpeg", - "data": item["image_url"]["url"].split(",")[1], - } - del item["image_url"] + if item["image_url"]["url"].startswith("data:"): + item["type"] = "image" + item["source"] = { + "type": "base64", + "media_type": "image/jpeg", + "data": item["image_url"]["url"].split(",")[1], + } + del item["image_url"] + elif item["image_url"]["url"].startswith("http"): + item["type"] = "image" + item["source"] = { + "type": "url", + "url": item["image_url"]["url"], + } + del item["image_url"] + else: + url = item["image_url"]["url"] + logger.warning(f"Unsupported 'image_url': {url}") + # In the case where there's a single image in the list (like what # would result from a UserImageRawFrame), ensure that the image # comes before text, as recommended by Anthropic docs diff --git a/src/pipecat/adapters/services/bedrock_adapter.py b/src/pipecat/adapters/services/bedrock_adapter.py index 852ea17a4..213e3b01c 100644 --- a/src/pipecat/adapters/services/bedrock_adapter.py +++ b/src/pipecat/adapters/services/bedrock_adapter.py @@ -256,15 +256,22 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): new_content.append({"text": text_content}) # handle image_url -> image conversion if item["type"] == "image_url": - new_item = { - "image": { - "format": "jpeg", - "source": { - "bytes": base64.b64decode(item["image_url"]["url"].split(",")[1]) - }, + if item["image_url"]["url"].startswith("data:"): + new_item = { + "image": { + "format": "jpeg", + "source": { + "bytes": base64.b64decode( + item["image_url"]["url"].split(",")[1] + ) + }, + } } - } - new_content.append(new_item) + new_content.append(new_item) + else: + url = item["image_url"]["url"] + logger.warning(f"Unsupported 'image_url': {url}") + # In the case where there's a single image in the list (like what # would result from a UserImageRawFrame), ensure that the image # comes before text diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 1fa8d9e6f..dc5bae559 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -343,7 +343,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): for c in content: if c["type"] == "text": parts.append(Part(text=c["text"])) - elif c["type"] == "image_url": + elif c["type"] == "image_url" and c["image_url"]["url"].startswith("data:"): parts.append( Part( inline_data=Blob( @@ -352,6 +352,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): ) ) ) + elif c["type"] == "image_url": + url = c["image_url"]["url"] + logger.warning(f"Unsupported 'image_url': {url}") elif c["type"] == "input_audio": input_audio = c["input_audio"] audio_bytes = base64.b64decode(input_audio["data"]) diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 768df0e5e..d9280f9c0 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -114,6 +114,28 @@ class LLMContext: self._tools: ToolsSchema | NotGiven = LLMContext._normalize_and_validate_tools(tools) self._tool_choice: LLMContextToolChoice | NotGiven = tool_choice + @staticmethod + def create_image_url_message( + *, + role: str = "user", + url: str, + text: Optional[str] = None, + ) -> LLMContextMessage: + """Create a context message containing an image URL. + + Args: + role: The role of this message (defaults to "user"). + url: The URL of the image. + text: Optional text to include with the image. + """ + content = [] + if text: + content.append({"type": "text", "text": text}) + + content.append({"type": "image_url", "image_url": {"url": url}}) + + return {"role": role, "content": content} + @staticmethod def create_image_message( *, @@ -135,19 +157,9 @@ class LLMContext: buffer = io.BytesIO() Image.frombytes(format, size, image).save(buffer, format="JPEG") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + url = f"data:image/jpeg;base64,{encoded_image}" - content = [] - if text: - content.append({"type": "text", "text": text}) - - content.append( - { - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}, - }, - ) - - return {"role": role, "content": content} + return LLMContext.create_image_url_message(role=role, url=url, text=text) @staticmethod def create_audio_message(