From 685d4402063200de34630e285cc7119c9f0aeb6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 30 Oct 2025 15:18:27 -0700 Subject: [PATCH] UserImageRawFrame: rename add_to_context to append_to_context --- CHANGELOG.md | 10 +++++----- .../14d-function-calling-anthropic-video.py | 2 +- .../foundational/14d-function-calling-aws-video.py | 2 +- .../14d-function-calling-gemini-flash-video.py | 2 +- .../14d-function-calling-moondream-video.py | 2 +- .../14d-function-calling-openai-video.py | 2 +- src/pipecat/frames/frames.py | 12 ++++++------ .../processors/aggregators/llm_response_universal.py | 4 ++-- src/pipecat/transports/daily/transport.py | 2 +- src/pipecat/transports/smallwebrtc/transport.py | 8 ++++---- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ec6ad556..0755769c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,12 +164,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- `UserImageRawFrame` new fields `add_to_context` and `text`. The - `add_to_context` field indicates if this image and text should be added to the - LLM context (by the LLM assistant aggregator). The `text` field, if set, might - also guide the LLM or the vision service on how to analyze the image. +- `UserImageRawFrame` new fields `append_to_context` and `text`. The + `append_to_context` field indicates if this image and text should be added to + the LLM context (by the LLM assistant aggregator). The `text` field, if set, + might also guide the LLM or the vision service on how to analyze the image. -- `UserImageRequestFrame` new fiels `add_to_context` and `text`. Both fields +- `UserImageRequestFrame` new fiels `append_to_context` and `text`. Both fields will be used to set the same fields on the captured `UserImageRawFrame`. - `UserImageRequestFrame` don't require function call name and ID anymore. diff --git a/examples/foundational/14d-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py index b21b9fdba..c933779bb 100644 --- a/examples/foundational/14d-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -53,7 +53,7 @@ async def fetch_user_image(params: FunctionCallParams): # Request a user image frame and indicate that it should be added to the # context. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), FrameDirection.UPSTREAM, ) diff --git a/examples/foundational/14d-function-calling-aws-video.py b/examples/foundational/14d-function-calling-aws-video.py index dffee193c..392aefca7 100644 --- a/examples/foundational/14d-function-calling-aws-video.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -53,7 +53,7 @@ async def fetch_user_image(params: FunctionCallParams): # Request a user image frame and indicate that it should be added to the # context. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), FrameDirection.UPSTREAM, ) diff --git a/examples/foundational/14d-function-calling-gemini-flash-video.py b/examples/foundational/14d-function-calling-gemini-flash-video.py index acb977c68..c11a4de2e 100644 --- a/examples/foundational/14d-function-calling-gemini-flash-video.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -53,7 +53,7 @@ async def fetch_user_image(params: FunctionCallParams): # Request a user image frame and indicate that it should be added to the # context. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), FrameDirection.UPSTREAM, ) diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index 3a7889c00..83d6ffd66 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -55,7 +55,7 @@ async def fetch_user_image(params: FunctionCallParams): # image to be added to the context because we will process it with # Moondream. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, add_to_context=False), + UserImageRequestFrame(user_id=user_id, text=question, append_to_context=False), FrameDirection.UPSTREAM, ) diff --git a/examples/foundational/14d-function-calling-openai-video.py b/examples/foundational/14d-function-calling-openai-video.py index ec6fb008b..d68ba7f7c 100644 --- a/examples/foundational/14d-function-calling-openai-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -54,7 +54,7 @@ async def fetch_user_image(params: FunctionCallParams): # Request a user image frame and indicate that it should be added to the # context. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), FrameDirection.UPSTREAM, ) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 70975d262..6577ae7a4 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1207,17 +1207,17 @@ class UserImageRequestFrame(SystemFrame): Parameters: user_id: Identifier of the user to request image from. text: An optional text associated to the image request. - add_to_context: Whether the requested image should be added to an LLM context. + append_to_context: Whether the requested image should be appended to the LLM context. video_source: Specific video source to capture from. """ user_id: str text: Optional[str] = None - add_to_context: Optional[bool] = None + append_to_context: Optional[bool] = None video_source: Optional[str] = None def __str__(self): - return f"{self.name}(user: {self.user_id}, text: {self.text}, add_to_context: {self.add_to_context}, {self.video_source})" + return f"{self.name}(user: {self.user_id}, text: {self.text}, append_to_context: {self.append_to_context}, {self.video_source})" @dataclass @@ -1292,16 +1292,16 @@ class UserImageRawFrame(InputImageRawFrame): Parameters: user_id: Identifier of the user who provided this image. text: An optional text associated to this image. - add_to_context: Whether this image should be added to an LLM context. + append_to_context: Whether the requested image should be appended to the LLM context. """ user_id: str = "" text: Optional[str] = None - add_to_context: Optional[bool] = None + append_to_context: Optional[bool] = None def __str__(self): pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text}, add_to_context: {self.add_to_context})" + return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text}, append_to_context: {self.append_to_context})" @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 44c534b9b..cc5c33393 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -767,10 +767,10 @@ class LLMAssistantAggregator(LLMContextAggregator): message["content"] = result async def _handle_user_image_frame(self, frame: UserImageRawFrame): - if not frame.add_to_context: + if not frame.append_to_context: return - logger.debug(f"{self} Adding UserImageRawFrame to LLM context (size: {frame.size})") + logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") self._context.add_image_frame_message( format=frame.format, diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index c6dd06986..ce97eb4dc 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -1843,7 +1843,7 @@ class DailyInputTransport(BaseInputTransport): size=(video_frame.width, video_frame.height), format=video_frame.color_format, text=request_frame.text if request_frame else None, - add_to_context=request_frame.add_to_context if request_frame else None, + append_to_context=request_frame.append_to_context if request_frame else None, ) frame.transport_source = video_source await self.push_video_frame(frame) diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index 4bd18c9ef..6c2854e9a 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -661,6 +661,8 @@ class SmallWebRTCInputTransport(BaseInputTransport): # UserImageRawFrame. Use a shallow copy so we can remove # elements. for request_frame in self._image_requests[:]: + request_text = request_frame.text if request_frame else None + add_to_context = request_frame.append_to_context if request_frame else None if request_frame.video_source == video_source: # Create UserImageRawFrame using the current video frame image_frame = UserImageRawFrame( @@ -668,10 +670,8 @@ class SmallWebRTCInputTransport(BaseInputTransport): image=video_frame.image, size=video_frame.size, format=video_frame.format, - text=request_frame.text if request_frame else None, - add_to_context=request_frame.add_to_context - if request_frame - else None, + text=request_text, + append_to_context=add_to_context, ) image_frame.transport_source = video_source # Push the frame to the pipeline