Merge pull request #2947 from pipecat-ai/aleix/rename-add-to-context

UserImageRawFrame: rename add_to_context to append_to_context

CHANGELOG.md

@@ -160,12 +160,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- `UserImageRawFrame` new fields `add_to_context` and `text`. The
+- `UserImageRawFrame` new fields `append_to_context` and `text`. The
-  `add_to_context` field indicates if this image and text should be added to the
+  `append_to_context` field indicates if this image and text should be added to
-  LLM context (by the LLM assistant aggregator). The `text` field, if set, might
+  the LLM context (by the LLM assistant aggregator). The `text` field, if set,
-  also guide the LLM or the vision service on how to analyze the image.
+  might also guide the LLM or the vision service on how to analyze the image.
 
-- `UserImageRequestFrame` new fiels `add_to_context` and `text`. Both fields
+- `UserImageRequestFrame` new fiels `append_to_context` and `text`. Both fields
   will be used to set the same fields on the captured `UserImageRawFrame`.
 
 - `UserImageRequestFrame` don't require function call name and ID anymore.

examples/foundational/14d-function-calling-anthropic-video.py

@@ -53,7 +53,7 @@ async def fetch_user_image(params: FunctionCallParams):
     # Request a user image frame and indicate that it should be added to the
     # context.
     await params.llm.push_frame(
-        UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True),
+        UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True),
         FrameDirection.UPSTREAM,
     )
 

examples/foundational/14d-function-calling-aws-video.py

@@ -53,7 +53,7 @@ async def fetch_user_image(params: FunctionCallParams):
     # Request a user image frame and indicate that it should be added to the
     # context.
     await params.llm.push_frame(
-        UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True),
+        UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True),
         FrameDirection.UPSTREAM,
     )
 

examples/foundational/14d-function-calling-gemini-flash-video.py

@@ -53,7 +53,7 @@ async def fetch_user_image(params: FunctionCallParams):
     # Request a user image frame and indicate that it should be added to the
     # context.
     await params.llm.push_frame(
-        UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True),
+        UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True),
         FrameDirection.UPSTREAM,
     )
 

examples/foundational/14d-function-calling-moondream-video.py

@@ -55,7 +55,7 @@ async def fetch_user_image(params: FunctionCallParams):
     # image to be added to the context because we will process it with
     # Moondream.
     await params.llm.push_frame(
-        UserImageRequestFrame(user_id=user_id, text=question, add_to_context=False),
+        UserImageRequestFrame(user_id=user_id, text=question, append_to_context=False),
         FrameDirection.UPSTREAM,
     )
 

examples/foundational/14d-function-calling-openai-video.py

@@ -54,7 +54,7 @@ async def fetch_user_image(params: FunctionCallParams):
     # Request a user image frame and indicate that it should be added to the
     # context.
     await params.llm.push_frame(
-        UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True),
+        UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True),
         FrameDirection.UPSTREAM,
     )
 

src/pipecat/frames/frames.py

@@ -1207,17 +1207,17 @@ class UserImageRequestFrame(SystemFrame):
     Parameters:
         user_id: Identifier of the user to request image from.
         text: An optional text associated to the image request.
-        add_to_context: Whether the requested image should be added to an LLM context.
+        append_to_context: Whether the requested image should be appended to the LLM context.
         video_source: Specific video source to capture from.
     """
 
     user_id: str
     text: Optional[str] = None
-    add_to_context: Optional[bool] = None
+    append_to_context: Optional[bool] = None
     video_source: Optional[str] = None
 
     def __str__(self):
-        return f"{self.name}(user: {self.user_id}, text: {self.text}, add_to_context: {self.add_to_context}, {self.video_source})"
+        return f"{self.name}(user: {self.user_id}, text: {self.text}, append_to_context: {self.append_to_context}, {self.video_source})"
 
 
 @dataclass
@@ -1292,16 +1292,16 @@ class UserImageRawFrame(InputImageRawFrame):
     Parameters:
         user_id: Identifier of the user who provided this image.
         text: An optional text associated to this image.
-        add_to_context: Whether this image should be added to an LLM context.
+        append_to_context: Whether the requested image should be appended to the LLM context.
     """
 
     user_id: str = ""
     text: Optional[str] = None
-    add_to_context: Optional[bool] = None
+    append_to_context: Optional[bool] = None
 
     def __str__(self):
         pts = format_pts(self.pts)
-        return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text}, add_to_context: {self.add_to_context})"
+        return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text}, append_to_context: {self.append_to_context})"
 
 
 @dataclass

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -783,10 +783,10 @@ class LLMAssistantAggregator(LLMContextAggregator):
                 message["content"] = result
 
     async def _handle_user_image_frame(self, frame: UserImageRawFrame):
-        if not frame.add_to_context:
+        if not frame.append_to_context:
             return
 
-        logger.debug(f"{self} Adding UserImageRawFrame to LLM context (size: {frame.size})")
+        logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})")
 
         self._context.add_image_frame_message(
             format=frame.format,

src/pipecat/transports/daily/transport.py

@@ -1843,7 +1843,7 @@ class DailyInputTransport(BaseInputTransport):
                 size=(video_frame.width, video_frame.height),
                 format=video_frame.color_format,
                 text=request_frame.text if request_frame else None,
-                add_to_context=request_frame.add_to_context if request_frame else None,
+                append_to_context=request_frame.append_to_context if request_frame else None,
             )
             frame.transport_source = video_source
             await self.push_video_frame(frame)

src/pipecat/transports/smallwebrtc/transport.py

@@ -661,6 +661,8 @@ class SmallWebRTCInputTransport(BaseInputTransport):
                     # UserImageRawFrame. Use a shallow copy so we can remove
                     # elements.
                     for request_frame in self._image_requests[:]:
+                        request_text = request_frame.text if request_frame else None
+                        add_to_context = request_frame.append_to_context if request_frame else None
                         if request_frame.video_source == video_source:
                             # Create UserImageRawFrame using the current video frame
                             image_frame = UserImageRawFrame(
@@ -668,10 +670,8 @@ class SmallWebRTCInputTransport(BaseInputTransport):
                                 image=video_frame.image,
                                 size=video_frame.size,
                                 format=video_frame.format,
-                                text=request_frame.text if request_frame else None,
+                                text=request_text,
-                                add_to_context=request_frame.add_to_context
+                                append_to_context=add_to_context,
-                                if request_frame
-                                else None,
                             )
                             image_frame.transport_source = video_source
                             # Push the frame to the pipeline