Update File upload RTVI messages and frames to use mime-type as the format

src/pipecat/frames/frames.py

@@ -244,14 +244,6 @@ class ImageRawFrame:
     format: Optional[str]
 
 
-ImageFileFormat = Literal["png", "jpg", "jpeg", "webp", "gif", "heic", "hief"]
-DocFileFormat = Literal[
-    "pdf", "csv", "txt", "md", "doc", "docx", "xls", "xlsx", "json", "html", "css", "javascript"
-]
-MediaFileFormat = Literal["mp3", "wav", "ogg", "aac", "mp4", "webm", "ogg", "avi"]
-
-FileFormat = Literal[ImageFileFormat, DocFileFormat, MediaFileFormat]
-
 FileSourceType = Literal["bytes", "url"]  # TODO: Add support for "id"
 
 
@@ -263,13 +255,13 @@ class FileRawFrame:
         file: Raw file bytes.
         type: Type of the file ('bytes' or 'url'),
         name: Optional name of the file.
-        format: File format (e.g., 'pdf', 'docx').
+        format: File format (expected in Mime Format).
     """
 
     file: bytes | str
     type: FileSourceType
     name: Optional[str]
-    format: Optional[FileFormat]
+    format: Optional[str]
 
 
 #

src/pipecat/processors/aggregators/llm_context.py

@@ -32,7 +32,7 @@ from openai.types.chat import (
 from PIL import Image
 
 from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
-from pipecat.frames.frames import AudioRawFrame, ImageFileFormat
+from pipecat.frames.frames import AudioRawFrame
 
 if TYPE_CHECKING:
     from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
@@ -412,19 +412,18 @@ class LLMContext:
             role: The role of this message (defaults to "user").
         """
         bytes = file
-        is_image = format in get_args(ImageFileFormat)
+        is_image = format.startswith("image/")
-        if not is_image and format != "pdf":
+        if not is_image and format != "application/pdf":
             # TODO how to send error back?
             logger.warning(f"Unsupported file format for LLM context: {format}")
             return
-        mime_type = f"image/{format}" if is_image else "application/pdf"
         if type == "url":
             async with aiohttp.ClientSession() as session:
                 async with session.get(file) as response:
                     content = io.BytesIO(await response.content.read())
                     base64_string = base64.b64encode(content.getvalue()).decode("utf-8")
-                    bytes = f"data:{mime_type};base64,{base64_string}"
+                    bytes = f"data:{format};base64,{base64_string}"
-            if format in get_args(ImageFileFormat):
+            if is_image:
                 message = LLMContext.create_image_url_message(role=role, url=bytes, text=text)
                 self.add_message(message)
                 return

src/pipecat/processors/frameworks/rtvi/models.py

@@ -26,7 +26,6 @@ from pydantic import BaseModel
 
 from pipecat.frames.frames import (
     AggregationType,
-    FileFormat,
     FileSourceType,
 )
 
@@ -256,7 +255,7 @@ class FileUrl(FileSource):
 class File(BaseModel):
     """File data structure for RTVI file sending."""
 
-    format: FileFormat
+    format: str  # Mime format of the file, e.g., 'application/pdf'
     name: Optional[str] = None
     source: FileBytes | FileUrl
     customOpts: Optional[dict] = None  # ex. 'detail' in openAI or 'citations' in Bedrock

src/pipecat/processors/frameworks/rtvi/processor.py

@@ -8,7 +8,7 @@
 
 import asyncio
 import base64
-from typing import Any, Dict, Mapping, Optional, get_args
+from typing import Any, Dict, Mapping, Optional
 
 from loguru import logger
 from pydantic import BaseModel, ValidationError
@@ -20,10 +20,8 @@ from pipecat.frames.frames import (
     EndFrame,
     EndTaskFrame,
     ErrorFrame,
-    FileFormat,
     Frame,
     FunctionCallResultFrame,
-    ImageFileFormat,
     InputAudioRawFrame,
     InputTransportMessageFrame,
     LLMConfigureOutputFrame,
@@ -566,9 +564,6 @@ class RTVIProcessor(FrameProcessor):
     async def _handle_send_file(self, data: RTVI.SendFileData):
         """Handle a send-file message from the client."""
         file = data.file
-        if file.format not in get_args(FileFormat):
-            logger.warning(f"Unsupported file format: {file.format}")
-            return
 
         source = None
         if file.source.type == "bytes":
@@ -579,7 +574,7 @@ class RTVIProcessor(FrameProcessor):
             logger.warning(f"Unsupported file source type: {file.source.type}")
             return
 
-        if file.source.type == "bytes" and file.format in get_args(ImageFileFormat):
+        if file.source.type == "bytes" and file.format.startswith("image/"):
             size = [file.source.width or 0, file.source.height or 0]
             file_frame = UserImageRawFrame(
                 text=data.content,