From 246c825a82a1a5ecf50f7cc6ecad803d8dff6470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 10 Dec 2024 17:07:17 -0800 Subject: [PATCH 1/3] examples: rename 07p-interruptible-google-audio-in to 07s --- ...le-google-audio-in.py => 07s-interruptible-google-audio-in.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/foundational/{07p-interruptible-google-audio-in.py => 07s-interruptible-google-audio-in.py} (100%) diff --git a/examples/foundational/07p-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py similarity index 100% rename from examples/foundational/07p-interruptible-google-audio-in.py rename to examples/foundational/07s-interruptible-google-audio-in.py From a618bd3fa6b93fd39b52b493240212c89b6a4a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 10 Dec 2024 17:08:12 -0800 Subject: [PATCH 2/3] openai: remove from_image_frame() and use add_image_frame_message() --- .../aggregators/openai_llm_context.py | 31 +++---------------- src/pipecat/services/openai.py | 5 ++- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index f647936c9..6e7474c17 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -19,7 +19,6 @@ from pipecat.frames.frames import ( Frame, FunctionCallInProgressFrame, FunctionCallResultFrame, - VisionImageRawFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -71,28 +70,6 @@ class OpenAILLMContext: context.add_message(message) return context - # todo: deprecate from_image_frame. It's only used to create a single-use - # context, which isn't useful for most real-world applications. - @staticmethod - def from_image_frame(frame: VisionImageRawFrame) -> "OpenAILLMContext": - """ - For images, we are deviating from the OpenAI messages shape. OpenAI - expects images to be base64 encoded, but other vision models may not. - So we'll store the image as bytes and do the base64 encoding as needed - in the LLM service. - - NOTE: the above only applies to the deprecated use of this method. The - add_image_frame_message() below does the base64 encoding as expected - in the OpenAI format. - """ - context = OpenAILLMContext() - buffer = io.BytesIO() - Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") - context.add_message( - {"content": frame.text, "role": "user", "data": buffer, "mime_type": "image/jpeg"} - ) - return context - @property def messages(self) -> List[ChatCompletionMessageParam]: return self._messages @@ -167,12 +144,12 @@ class OpenAILLMContext: Image.frombytes(format, size, image).save(buffer, format="JPEG") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") - content = [ - {"type": "text", "text": text}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}, - ] + content = [] if text: content.append({"type": "text", "text": text}) + content.append( + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}, + ) self.add_message({"role": "user", "content": content}) def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None): diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 4e11edc22..43ad16536 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -294,7 +294,10 @@ class BaseOpenAILLMService(LLMService): elif isinstance(frame, LLMMessagesFrame): context = OpenAILLMContext.from_messages(frame.messages) elif isinstance(frame, VisionImageRawFrame): - context = OpenAILLMContext.from_image_frame(frame) + context = OpenAILLMContext() + context.add_image_frame_message( + format=frame.format, size=frame.size, image=frame.image, text=frame.text + ) elif isinstance(frame, LLMUpdateSettingsFrame): await self._update_settings(frame.settings) else: From 9e7ecccf1efda09d987556d741ef2a630c4b717b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 10 Dec 2024 17:08:49 -0800 Subject: [PATCH 3/3] google: fix VisionImageRawFrame context --- src/pipecat/services/google.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 715724152..5442ee91c 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -320,6 +320,15 @@ class GoogleContextAggregatorPair: class GoogleLLMContext(OpenAILLMContext): + def __init__( + self, + messages: list[dict] | None = None, + tools: list[dict] | None = None, + tool_choice: dict | None = None, + ): + super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) + self.system_message = None + @staticmethod def upgrade_to_google(obj: OpenAILLMContext) -> "GoogleLLMContext": if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GoogleLLMContext): @@ -371,9 +380,8 @@ class GoogleLLMContext(OpenAILLMContext): parts = [] if text: parts.append(glm.Part(text=text)) - parts.append( - glm.Part(inline_data=glm.Blob(mime_type="image/jpeg", data=buffer.getvalue())), - ) + parts.append(glm.Part(inline_data=glm.Blob(mime_type="image/jpeg", data=buffer.getvalue()))) + self.add_message(glm.Content(role="user", parts=parts)) def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None): @@ -649,12 +657,14 @@ class GoogleLLMService(LLMService): context = None if isinstance(frame, OpenAILLMContextFrame): - context: GoogleLLMContext = GoogleLLMContext.upgrade_to_google(frame.context) + context = GoogleLLMContext.upgrade_to_google(frame.context) elif isinstance(frame, LLMMessagesFrame): context = GoogleLLMContext(frame.messages) elif isinstance(frame, VisionImageRawFrame): - # todo: fix this - context = OpenAILLMContext.from_image_frame(frame) + context = GoogleLLMContext() + context.add_image_frame_message( + format=frame.format, size=frame.size, image=frame.image, text=frame.text + ) elif isinstance(frame, LLMUpdateSettingsFrame): await self._update_settings(frame.settings) else: