From 71197fbc2c31a10a449b78179ac31971e0a8d329 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Thu, 15 Jan 2026 11:28:47 -0500 Subject: [PATCH] Support files provided via url --- src/pipecat/frames/frames.py | 8 ++-- .../processors/aggregators/llm_context.py | 41 ++++++++++++++----- .../aggregators/llm_response_universal.py | 1 + .../processors/frameworks/rtvi/models.py | 1 + .../processors/frameworks/rtvi/processor.py | 3 -- 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index bbab49cb2..2ab8b0919 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -25,7 +25,6 @@ from typing import ( Optional, Sequence, Tuple, - get_args, ) from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -251,10 +250,7 @@ DocFileFormat = Literal[ ] MediaFileFormat = Literal["mp3", "wav", "ogg", "aac", "mp4", "webm", "ogg", "avi"] -_all_formats = list( - set(get_args(ImageFileFormat) + get_args(DocFileFormat) + get_args(MediaFileFormat)) -) -FileFormat = Literal[tuple(_all_formats)] +FileFormat = Literal[ImageFileFormat, DocFileFormat, MediaFileFormat] FileSourceType = Literal["bytes", "url"] # TODO: Add support for "id" @@ -266,11 +262,13 @@ class FileRawFrame: Parameters: file: Raw file bytes. type: Type of the file ('bytes' or 'url'), + name: Optional name of the file. format: File format (e.g., 'pdf', 'docx'). """ file: bytes | str type: FileSourceType + name: Optional[str] format: Optional[FileFormat] diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 8a6f335ac..8cca1155e 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -19,8 +19,9 @@ import base64 import io import wave from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union +from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union, get_args +import aiohttp from loguru import logger from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN from openai._types import NotGiven as OpenAINotGiven @@ -31,7 +32,7 @@ from openai.types.chat import ( from PIL import Image from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema -from pipecat.frames.frames import AudioRawFrame +from pipecat.frames.frames import AudioRawFrame, ImageFileFormat if TYPE_CHECKING: from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -202,16 +203,16 @@ class LLMContext: async def create_file_message( *, role: str = "user", - format: str, file: bytes, + name: Optional[str] = None, text: Optional[str] = None, ) -> LLMContextMessage: """Create a context message containing a file. Args: role: The role of this message (defaults to "user"). - format: File format (the MIME type like 'image/jpeg'). file: Raw file bytes. + name: Optional name of the file. text: Optional text to include with the file. """ # Right now: assumes file is already encoded properly as a data URL: @@ -221,7 +222,7 @@ class LLMContext: if text: content.append({"type": "text", "text": text}) - file = {"file_data": file, "filename": "test"} + file = {"file_data": file, "filename": name if name else ""} content.append({"type": "file", "file": file}) return {"role": role, "content": content} @@ -393,22 +394,42 @@ class LLMContext: async def add_file_frame_message( self, *, + type: str, format: str, - file: bytes, + file: bytes | str, text: Optional[str] = None, + name: Optional[str] = None, role: str = "user", ): """Add a message containing a file frame. Args: + type: File type (e.g., 'bytes' or 'url'). format: File format (the MIME type like 'image/jpeg'). - file: Raw file bytes. + file: Raw file bytes or URL string. text: Optional text to include with the file. + name: Optional name of the file. role: The role of this message (defaults to "user"). """ - message = await LLMContext.create_file_message( - role=role, format=format, file=file, text=text - ) + bytes = file + is_image = format in get_args(ImageFileFormat) + if not is_image and format != "pdf": + # TODO how to send error back? + logger.warning(f"Unsupported file format for LLM context: {format}") + return + mime_type = f"image/{format}" if is_image else "application/pdf" + if type == "url": + async with aiohttp.ClientSession() as session: + async with session.get(file) as response: + content = io.BytesIO(await response.content.read()) + base64_string = base64.b64encode(content.getvalue()).decode("utf-8") + bytes = f"data:{mime_type};base64,{base64_string}" + if format in get_args(ImageFileFormat): + message = LLMContext.create_image_url_message(role=role, url=bytes, text=text) + self.add_message(message) + return + + message = await LLMContext.create_file_message(role=role, file=bytes, name=name, text=text) self.add_message(message) async def add_image_frame_message( diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 364bd3a72..779af65af 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -1144,6 +1144,7 @@ class LLMAssistantAggregator(LLMContextAggregator): logger.debug(f"{self} Appending UserFileRawFrame to LLM context (format: {frame.format})") await self._context.add_file_frame_message( + type=frame.type, format=frame.format, text=frame.text, file=frame.file, diff --git a/src/pipecat/processors/frameworks/rtvi/models.py b/src/pipecat/processors/frameworks/rtvi/models.py index 5255fef48..922a93256 100644 --- a/src/pipecat/processors/frameworks/rtvi/models.py +++ b/src/pipecat/processors/frameworks/rtvi/models.py @@ -257,6 +257,7 @@ class File(BaseModel): """File data structure for RTVI file sending.""" format: FileFormat + name: Optional[str] = None source: FileBytes | FileUrl customOpts: Optional[dict] = None # ex. 'detail' in openAI or 'citations' in Bedrock diff --git a/src/pipecat/processors/frameworks/rtvi/processor.py b/src/pipecat/processors/frameworks/rtvi/processor.py index 8457d0fa3..44224e27b 100644 --- a/src/pipecat/processors/frameworks/rtvi/processor.py +++ b/src/pipecat/processors/frameworks/rtvi/processor.py @@ -575,9 +575,6 @@ class RTVIProcessor(FrameProcessor): source = file.source.bytes elif file.source.type == "url": source = file.source.url - elif file.source.type == "id": - logger.warning("File source type 'id' is not supported yet.") - return else: logger.warning(f"Unsupported file source type: {file.source.type}") return