Support files provided via url

src/pipecat/frames/frames.py

@@ -25,7 +25,6 @@ from typing import (
     Optional,
     Sequence,
     Tuple,
-    get_args,
 )
 
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -251,10 +250,7 @@ DocFileFormat = Literal[
 ]
 MediaFileFormat = Literal["mp3", "wav", "ogg", "aac", "mp4", "webm", "ogg", "avi"]
 
-_all_formats = list(
+FileFormat = Literal[ImageFileFormat, DocFileFormat, MediaFileFormat]
-    set(get_args(ImageFileFormat) + get_args(DocFileFormat) + get_args(MediaFileFormat))
-)
-FileFormat = Literal[tuple(_all_formats)]
 
 FileSourceType = Literal["bytes", "url"]  # TODO: Add support for "id"
 
@@ -266,11 +262,13 @@ class FileRawFrame:
     Parameters:
         file: Raw file bytes.
         type: Type of the file ('bytes' or 'url'),
+        name: Optional name of the file.
         format: File format (e.g., 'pdf', 'docx').
     """
 
     file: bytes | str
     type: FileSourceType
+    name: Optional[str]
     format: Optional[FileFormat]
 
 

src/pipecat/processors/aggregators/llm_context.py

@@ -19,8 +19,9 @@ import base64
 import io
 import wave
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union
+from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union, get_args
 
+import aiohttp
 from loguru import logger
 from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN
 from openai._types import NotGiven as OpenAINotGiven
@@ -31,7 +32,7 @@ from openai.types.chat import (
 from PIL import Image
 
 from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
-from pipecat.frames.frames import AudioRawFrame
+from pipecat.frames.frames import AudioRawFrame, ImageFileFormat
 
 if TYPE_CHECKING:
     from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
@@ -202,16 +203,16 @@ class LLMContext:
     async def create_file_message(
         *,
         role: str = "user",
-        format: str,
         file: bytes,
+        name: Optional[str] = None,
         text: Optional[str] = None,
     ) -> LLMContextMessage:
         """Create a context message containing a file.
 
         Args:
             role: The role of this message (defaults to "user").
-            format: File format (the MIME type like 'image/jpeg').
             file: Raw file bytes.
+            name: Optional name of the file.
             text: Optional text to include with the file.
         """
         # Right now: assumes file is already encoded properly as a data URL:
@@ -221,7 +222,7 @@ class LLMContext:
         if text:
             content.append({"type": "text", "text": text})
 
-        file = {"file_data": file, "filename": "test"}
+        file = {"file_data": file, "filename": name if name else ""}
         content.append({"type": "file", "file": file})
 
         return {"role": role, "content": content}
@@ -393,22 +394,42 @@ class LLMContext:
     async def add_file_frame_message(
         self,
         *,
+        type: str,
         format: str,
-        file: bytes,
+        file: bytes | str,
         text: Optional[str] = None,
+        name: Optional[str] = None,
         role: str = "user",
     ):
         """Add a message containing a file frame.
 
         Args:
+            type: File type (e.g., 'bytes' or 'url').
             format: File format (the MIME type like 'image/jpeg').
-            file: Raw file bytes.
+            file: Raw file bytes or URL string.
             text: Optional text to include with the file.
+            name: Optional name of the file.
             role: The role of this message (defaults to "user").
         """
-        message = await LLMContext.create_file_message(
+        bytes = file
-            role=role, format=format, file=file, text=text
+        is_image = format in get_args(ImageFileFormat)
-        )
+        if not is_image and format != "pdf":
+            # TODO how to send error back?
+            logger.warning(f"Unsupported file format for LLM context: {format}")
+            return
+        mime_type = f"image/{format}" if is_image else "application/pdf"
+        if type == "url":
+            async with aiohttp.ClientSession() as session:
+                async with session.get(file) as response:
+                    content = io.BytesIO(await response.content.read())
+                    base64_string = base64.b64encode(content.getvalue()).decode("utf-8")
+                    bytes = f"data:{mime_type};base64,{base64_string}"
+            if format in get_args(ImageFileFormat):
+                message = LLMContext.create_image_url_message(role=role, url=bytes, text=text)
+                self.add_message(message)
+                return
+
+        message = await LLMContext.create_file_message(role=role, file=bytes, name=name, text=text)
         self.add_message(message)
 
     async def add_image_frame_message(

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -1144,6 +1144,7 @@ class LLMAssistantAggregator(LLMContextAggregator):
 
         logger.debug(f"{self} Appending UserFileRawFrame to LLM context (format: {frame.format})")
         await self._context.add_file_frame_message(
+            type=frame.type,
             format=frame.format,
             text=frame.text,
             file=frame.file,

src/pipecat/processors/frameworks/rtvi/models.py

@@ -257,6 +257,7 @@ class File(BaseModel):
     """File data structure for RTVI file sending."""
 
     format: FileFormat
+    name: Optional[str] = None
     source: FileBytes | FileUrl
     customOpts: Optional[dict] = None  # ex. 'detail' in openAI or 'citations' in Bedrock
 

src/pipecat/processors/frameworks/rtvi/processor.py

@@ -575,9 +575,6 @@ class RTVIProcessor(FrameProcessor):
             source = file.source.bytes
         elif file.source.type == "url":
             source = file.source.url
-        elif file.source.type == "id":
-            logger.warning("File source type 'id' is not supported yet.")
-            return
         else:
             logger.warning(f"Unsupported file source type: {file.source.type}")
             return