user image requests can now be related to function calls

examples/foundational/14b-function-calling-anthropic-video.py

@@ -39,7 +39,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
 
 async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
     question = arguments["question"]
-    await llm.request_image_frame(user_id=video_participant_id, text_content=question)
+    await llm.request_image_frame(
+        user_id=video_participant_id,
+        function_name=function_name,
+        tool_call_id=tool_call_id,
+        text_content=question,
+    )
 
 
 async def main():

examples/foundational/14d-function-calling-video.py

@@ -39,7 +39,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
 async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
     logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}")
     question = arguments["question"]
-    await llm.request_image_frame(user_id=video_participant_id, text_content=question)
+    await llm.request_image_frame(
+        user_id=video_participant_id,
+        function_name=function_name,
+        tool_call_id=tool_call_id,
+        text_content=question,
+    )
 
 
 async def main():
@@ -141,7 +146,7 @@ indicate you should use the get_image tool are:
             await transport.capture_participant_transcription(participant["id"])
             await transport.capture_participant_video(video_participant_id, framerate=0)
             # Kick off the conversation.
-            await tts.say("Hi! Ask me about the weather in San Francisco.")
+            await task.queue_frames([context_aggregator.user().get_context_frame()])
 
         runner = PipelineRunner()
 

examples/foundational/14e-function-calling-gemini.py

@@ -42,7 +42,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
 async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
     logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}")
     question = arguments["question"]
-    await llm.request_image_frame(user_id=video_participant_id, text_content=question)
+    await llm.request_image_frame(
+        user_id=video_participant_id,
+        function_name=function_name,
+        tool_call_id=tool_call_id,
+        text_content=question,
+    )
 
 
 async def main():

examples/foundational/20d-persistent-context-gemini.py

@@ -54,7 +54,12 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
 
 async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
     question = arguments["question"]
-    await llm.request_image_frame(user_id=video_participant_id, text_content=question)
+    await llm.request_image_frame(
+        user_id=video_participant_id,
+        function_name=function_name,
+        tool_call_id=tool_call_id,
+        text_content=question,
+    )
 
 
 async def get_saved_conversation_filenames(

examples/moondream-chatbot/bot.py

@@ -23,7 +23,6 @@ from pipecat.frames.frames import (
     OutputImageRawFrame,
     SpriteFrame,
     TextFrame,
-    UserImageRawFrame,
     UserImageRequestFrame,
 )
 from pipecat.pipeline.parallel_pipeline import ParallelPipeline