diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 2ab8b0919..6773eb9fc 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -244,14 +244,6 @@ class ImageRawFrame: format: Optional[str] -ImageFileFormat = Literal["png", "jpg", "jpeg", "webp", "gif", "heic", "hief"] -DocFileFormat = Literal[ - "pdf", "csv", "txt", "md", "doc", "docx", "xls", "xlsx", "json", "html", "css", "javascript" -] -MediaFileFormat = Literal["mp3", "wav", "ogg", "aac", "mp4", "webm", "ogg", "avi"] - -FileFormat = Literal[ImageFileFormat, DocFileFormat, MediaFileFormat] - FileSourceType = Literal["bytes", "url"] # TODO: Add support for "id" @@ -263,13 +255,13 @@ class FileRawFrame: file: Raw file bytes. type: Type of the file ('bytes' or 'url'), name: Optional name of the file. - format: File format (e.g., 'pdf', 'docx'). + format: File format (expected in Mime Format). """ file: bytes | str type: FileSourceType name: Optional[str] - format: Optional[FileFormat] + format: Optional[str] # diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 8cca1155e..b45c20e1e 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -32,7 +32,7 @@ from openai.types.chat import ( from PIL import Image from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema -from pipecat.frames.frames import AudioRawFrame, ImageFileFormat +from pipecat.frames.frames import AudioRawFrame if TYPE_CHECKING: from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -412,19 +412,18 @@ class LLMContext: role: The role of this message (defaults to "user"). """ bytes = file - is_image = format in get_args(ImageFileFormat) - if not is_image and format != "pdf": + is_image = format.startswith("image/") + if not is_image and format != "application/pdf": # TODO how to send error back? logger.warning(f"Unsupported file format for LLM context: {format}") return - mime_type = f"image/{format}" if is_image else "application/pdf" if type == "url": async with aiohttp.ClientSession() as session: async with session.get(file) as response: content = io.BytesIO(await response.content.read()) base64_string = base64.b64encode(content.getvalue()).decode("utf-8") - bytes = f"data:{mime_type};base64,{base64_string}" - if format in get_args(ImageFileFormat): + bytes = f"data:{format};base64,{base64_string}" + if is_image: message = LLMContext.create_image_url_message(role=role, url=bytes, text=text) self.add_message(message) return diff --git a/src/pipecat/processors/frameworks/rtvi/models.py b/src/pipecat/processors/frameworks/rtvi/models.py index 922a93256..95a97638f 100644 --- a/src/pipecat/processors/frameworks/rtvi/models.py +++ b/src/pipecat/processors/frameworks/rtvi/models.py @@ -26,7 +26,6 @@ from pydantic import BaseModel from pipecat.frames.frames import ( AggregationType, - FileFormat, FileSourceType, ) @@ -256,7 +255,7 @@ class FileUrl(FileSource): class File(BaseModel): """File data structure for RTVI file sending.""" - format: FileFormat + format: str # Mime format of the file, e.g., 'application/pdf' name: Optional[str] = None source: FileBytes | FileUrl customOpts: Optional[dict] = None # ex. 'detail' in openAI or 'citations' in Bedrock diff --git a/src/pipecat/processors/frameworks/rtvi/processor.py b/src/pipecat/processors/frameworks/rtvi/processor.py index 44224e27b..d885b955a 100644 --- a/src/pipecat/processors/frameworks/rtvi/processor.py +++ b/src/pipecat/processors/frameworks/rtvi/processor.py @@ -8,7 +8,7 @@ import asyncio import base64 -from typing import Any, Dict, Mapping, Optional, get_args +from typing import Any, Dict, Mapping, Optional from loguru import logger from pydantic import BaseModel, ValidationError @@ -20,10 +20,8 @@ from pipecat.frames.frames import ( EndFrame, EndTaskFrame, ErrorFrame, - FileFormat, Frame, FunctionCallResultFrame, - ImageFileFormat, InputAudioRawFrame, InputTransportMessageFrame, LLMConfigureOutputFrame, @@ -566,9 +564,6 @@ class RTVIProcessor(FrameProcessor): async def _handle_send_file(self, data: RTVI.SendFileData): """Handle a send-file message from the client.""" file = data.file - if file.format not in get_args(FileFormat): - logger.warning(f"Unsupported file format: {file.format}") - return source = None if file.source.type == "bytes": @@ -579,7 +574,7 @@ class RTVIProcessor(FrameProcessor): logger.warning(f"Unsupported file source type: {file.source.type}") return - if file.source.type == "bytes" and file.format in get_args(ImageFileFormat): + if file.source.type == "bytes" and file.format.startswith("image/"): size = [file.source.width or 0, file.source.height or 0] file_frame = UserImageRawFrame( text=data.content,