From ceaf53fdb0cd9f9f1fe7c58c2ae62e454b8915fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 12:08:09 -0800 Subject: [PATCH] LLMContext: async create_image_message/create_audio_message fixes --- CHANGELOG.md | 7 ++++--- .../22d-natural-conversation-gemini-audio.py | 2 +- src/pipecat/processors/aggregators/llm_context.py | 14 ++++++++------ .../aggregators/llm_response_universal.py | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47be7d2bf..5ace3aa14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- ⚠️ Breaking change: `LLMContext.create_image_message()` and - `LLMContext.create_audio_message()` are now async methods. This fixes and - issue where the asyncio event loop would be blocked while encoding audio or +- ⚠️ Breaking change: `LLMContext.create_image_message()`, + `LLMContext.create_audio_message()`, `LLMContext.add_image_frame_message()` + and `LLMContext.add_audio_frames_message()` are now async methods. This fixes + an issue where the asyncio event loop would be blocked while encoding audio or images. - `ConsumerProcessor` now queues frames from the producer internally instead of diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 7a7155297..a7837ce60 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -391,7 +391,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = LLMContext() - context.add_audio_frames_message(audio_frames=self._audio_frames) + await context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(LLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index b9216103a..99b9aeaa9 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -180,7 +180,7 @@ class LLMContext: text: Optional text to include with the audio. """ - def encode_audio(): + async def encode_audio(): sample_rate = audio_frames[0].sample_rate num_channels = audio_frames[0].num_channels @@ -198,7 +198,7 @@ class LLMContext: encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") return encoded_audio - encoded_audio = asyncio.to_thread(encode_audio) + encoded_audio = await asyncio.to_thread(encode_audio) content.append( { @@ -333,7 +333,7 @@ class LLMContext: """ self._tool_choice = tool_choice - def add_image_frame_message( + async def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None ): """Add a message containing an image frame. @@ -344,10 +344,12 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - message = LLMContext.create_image_message(format=format, size=size, image=image, text=text) + message = await LLMContext.create_image_message( + format=format, size=size, image=image, text=text + ) self.add_message(message) - def add_audio_frames_message( + async def add_audio_frames_message( self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" ): """Add a message containing audio frames. @@ -356,7 +358,7 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - message = LLMContext.create_audio_message(audio_frames=audio_frames, text=text) + message = await LLMContext.create_audio_message(audio_frames=audio_frames, text=text) self.add_message(message) @staticmethod diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index d0ac67257..87974bed2 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -793,7 +793,7 @@ class LLMAssistantAggregator(LLMContextAggregator): logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") - self._context.add_image_frame_message( + await self._context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image,