diff --git a/changelog/3430.fixed.md b/changelog/3430.fixed.md new file mode 100644 index 000000000..6f689eb78 --- /dev/null +++ b/changelog/3430.fixed.md @@ -0,0 +1 @@ +- Fixed `request_image_frame` (for backwards compatibility) and restored function-call–related fields in `UserImageRequestFrame` and `UserImageRawFrame`, preventing a case where adding a non-LLM message to the context could trigger duplicate LLM inferences (on image arrival and on function-call result), potentially causing an infinite inference loop. diff --git a/examples/foundational/14d-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py index a217f11b1..d851b99fc 100644 --- a/examples/foundational/14d-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -14,7 +14,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -55,9 +55,15 @@ async def fetch_user_image(params: FunctionCallParams): logger.debug(f"Requesting image with user_id={user_id}, question={question}") # Request a user image frame and indicate that it should be added to the - # context. + # context. Also associate it to the function call. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=True, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + ), FrameDirection.UPSTREAM, ) @@ -101,6 +107,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) llm.register_function("fetch_user_image", fetch_user_image) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + fetch_image_function = FunctionSchema( name="fetch_user_image", description="Called when the user requests a description of their camera feed", diff --git a/examples/foundational/14d-function-calling-aws-video.py b/examples/foundational/14d-function-calling-aws-video.py index a216b5485..3e8c1785f 100644 --- a/examples/foundational/14d-function-calling-aws-video.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -14,7 +14,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -55,9 +55,15 @@ async def fetch_user_image(params: FunctionCallParams): logger.debug(f"Requesting image with user_id={user_id}, question={question}") # Request a user image frame and indicate that it should be added to the - # context. + # context. Also associate it to the function call. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=True, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + ), FrameDirection.UPSTREAM, ) @@ -108,6 +114,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm.register_function("fetch_user_image", fetch_user_image) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + fetch_image_function = FunctionSchema( name="fetch_user_image", description="Called when the user requests a description of their camera feed", diff --git a/examples/foundational/14d-function-calling-gemini-flash-video.py b/examples/foundational/14d-function-calling-gemini-flash-video.py index d7885b0dc..2c6a6c6ac 100644 --- a/examples/foundational/14d-function-calling-gemini-flash-video.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -14,7 +14,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -55,9 +55,15 @@ async def fetch_user_image(params: FunctionCallParams): logger.debug(f"Requesting image with user_id={user_id}, question={question}") # Request a user image frame and indicate that it should be added to the - # context. + # context. Also associate it to the function call. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=True, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + ), FrameDirection.UPSTREAM, ) @@ -101,6 +107,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) llm.register_function("fetch_user_image", fetch_user_image) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + fetch_image_function = FunctionSchema( name="fetch_user_image", description="Called when the user requests a description of their camera feed", diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index a6c2ef24b..cf244880c 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -20,6 +20,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMRunFrame, TextFrame, + TTSSpeakFrame, UserImageRequestFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline @@ -64,9 +65,15 @@ async def fetch_user_image(params: FunctionCallParams): # Request a user image frame. In this case, we don't want the requested # image to be added to the context because we will process it with - # Moondream. + # Moondream. Also associate it to the function call. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, append_to_context=False), + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=False, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + ), FrameDirection.UPSTREAM, ) @@ -130,6 +137,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) llm.register_function("fetch_user_image", fetch_user_image) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + fetch_image_function = FunctionSchema( name="fetch_user_image", description="Called when the user requests a description of their camera feed", diff --git a/examples/foundational/14d-function-calling-openai-video.py b/examples/foundational/14d-function-calling-openai-video.py index 60c8f5952..e1c3f52be 100644 --- a/examples/foundational/14d-function-calling-openai-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -15,7 +15,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -56,9 +56,15 @@ async def fetch_user_image(params: FunctionCallParams): logger.debug(f"Requesting image with user_id={user_id}, question={question}") # Request a user image frame and indicate that it should be added to the - # context. + # context. Also associate it to the function call. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, text=question, append_to_context=True), + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=True, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + ), FrameDirection.UPSTREAM, ) @@ -101,6 +107,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) llm.register_function("fetch_user_image", fetch_user_image) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + fetch_image_function = FunctionSchema( name="fetch_user_image", description="Called when the user requests a description of their camera feed", diff --git a/examples/foundational/14e-function-calling-google.py b/examples/foundational/14e-function-calling-google.py index 49a9f98c3..69791ffbc 100644 --- a/examples/foundational/14e-function-calling-google.py +++ b/examples/foundational/14e-function-calling-google.py @@ -5,7 +5,6 @@ # -import asyncio import os from dotenv import load_dotenv @@ -16,7 +15,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -25,6 +24,7 @@ from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, LLMUserAggregatorParams, ) +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -43,10 +43,6 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies load_dotenv(override=True) -# Global variable to store the client ID -client_id = "" - - async def get_weather(params: FunctionCallParams): location = params.arguments["location"] await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") @@ -57,24 +53,35 @@ async def fetch_restaurant_recommendation(params: FunctionCallParams): async def get_image(params: FunctionCallParams): + """Fetch the user image and push it to the LLM. + + When called, this function pushes a UserImageRequestFrame upstream to the + transport. As a result, the transport will request the user image and push a + UserImageRawFrame downstream which will be added to the context by the LLM + assistant aggregator. + """ + user_id = params.arguments["user_id"] question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") + logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, + # Request a user image frame and indicate that it should be added to the + # context. Also associate it to the function call. + await params.llm.push_frame( + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=True, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + ), + FrameDirection.UPSTREAM, ) - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) + await params.result_callback(None) - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) + # Instead of None, it's possible to also provide a tool call answer to + # tell the LLM that we are grabbing the image to analyze. + # await params.result_callback({"result": "Image is being captured."}) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -144,14 +151,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) get_image_function = FunctionSchema( name="get_image", - description="Get an image from the video stream.", + description="Called when the user requests a description of their camera feed", properties={ + "user_id": { + "type": "string", + "description": "The ID of the user to grab the image from", + }, "question": { "type": "string", - "description": "The question that the user is asking about the image.", - } + "description": "The question that the user is asking about the image", + }, }, - required=["question"], + required=["user_id", "question"], ) tools = ToolsSchema(standard_tools=[weather_function, get_image_function, restaurant_function]) @@ -175,7 +186,6 @@ indicate you should use the get_image tool are: """ messages = [ {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Say hello."}, ] context = LLMContext(messages, tools) @@ -215,10 +225,15 @@ indicate you should use the get_image tool are: await maybe_capture_participant_camera(transport, client) - global client_id client_id = get_transport_client_id(transport, client) # Kick off the conversation. + messages.append( + { + "role": "system", + "content": f"Please introduce yourself to the user. Use '{client_id}' as the user ID during function calls.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 35f3a22db..37a2f16a2 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -17,7 +17,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -26,6 +26,7 @@ from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, LLMUserAggregatorParams, ) +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -46,9 +47,6 @@ load_dotenv(override=True) BASE_FILENAME = "/tmp/pipecat_conversation_" -# Global variable to store the client ID -client_id = "" - async def fetch_weather_from_api(params: FunctionCallParams): temperature = 75 if params.arguments["format"] == "fahrenheit" else 24 @@ -63,17 +61,29 @@ async def fetch_weather_from_api(params: FunctionCallParams): async def get_image(params: FunctionCallParams): + user_id = params.arguments["user_id"] question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") + logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the image frame - await params.llm.request_image_frame( - user_id=client_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, + # Request a user image frame and indicate that it should be added to the + # context. Also associate it to the function call. + await params.llm.push_frame( + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=True, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + ), + FrameDirection.UPSTREAM, ) + await params.result_callback(None) + + # Instead of None, it's possible to also provide a tool call answer to + # tell the LLM that we are grabbing the image to analyze. + # await params.result_callback({"result": "Image is being captured."}) + async def get_saved_conversation_filenames(params: FunctionCallParams): # Construct the full pattern including the BASE_FILENAME @@ -207,14 +217,18 @@ load_conversation_function = FunctionSchema( get_image_function = FunctionSchema( name="get_image", - description="Get and image from the camera or video stream.", + description="Called when the user requests a description of their camera feed", properties={ + "user_id": { + "type": "string", + "description": "The ID of the user to grab the image from", + }, "question": { "type": "string", - "description": "The question to to use when running inference on the acquired image.", + "description": "The question that the user is asking about the image", }, }, - required=["question"], + required=["user_id", "question"], ) tools = ToolsSchema( @@ -257,7 +271,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) - llm = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) # you can either register a single function for all function calls, or specific functions # llm.register_function(None, fetch_weather_from_api) @@ -304,10 +318,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): await maybe_capture_participant_camera(transport, client) - global client_id client_id = get_transport_client_id(transport, client) # Kick off the conversation. + messages.append( + { + "role": "system", + "content": f"Please introduce yourself to the user. Use '{client_id}' as the user ID during function calls.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index fb0f8243b..a27bb780e 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1461,29 +1461,29 @@ class UserImageRequestFrame(SystemFrame): text: An optional text associated to the image request. append_to_context: Whether the requested image should be appended to the LLM context. video_source: Specific video source to capture from. + function_name: Name of function that generated this request (if any). + tool_call_id: Tool call ID if generated by function call (if any). context: [DEPRECATED] Optional context for the image request. - function_name: [DEPRECATED] Name of function that generated this request (if any). - tool_call_id: [DEPRECATED] Tool call ID if generated by function call. """ user_id: str text: Optional[str] = None append_to_context: Optional[bool] = None video_source: Optional[str] = None - context: Optional[Any] = None function_name: Optional[str] = None tool_call_id: Optional[str] = None + context: Optional[Any] = None def __post_init__(self): super().__post_init__() - if self.context or self.function_name or self.tool_call_id: + if self.context: import warnings with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( - "`UserImageRequestFrame` fields `context`, `function_name` and `tool_call_id` are deprecated.", + "`UserImageRequestFrame` field `context` is deprecated.", DeprecationWarning, stacklevel=2, ) @@ -1565,7 +1565,7 @@ class UserImageRawFrame(InputImageRawFrame): user_id: Identifier of the user who provided this image. text: An optional text associated to this image. append_to_context: Whether the requested image should be appended to the LLM context. - request: [DEPRECATED] The original image request frame if this is a response. + request: The original image request frame if this is a response. """ user_id: str = "" @@ -1573,20 +1573,6 @@ class UserImageRawFrame(InputImageRawFrame): append_to_context: Optional[bool] = None request: Optional[UserImageRequestFrame] = None - def __post_init__(self): - super().__post_init__() - - if self.request: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "`UserImageRawFrame` field `request` is deprecated.", - DeprecationWarning, - stacklevel=2, - ) - def __str__(self): pts = format_pts(self.pts) return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text}, append_to_context: {self.append_to_context})" diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 1baf87304..901e5c62c 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -641,6 +641,7 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {} + self._function_calls_image_results: Dict[str, UserImageRawFrame] = {} self._context_updated_tasks: Set[asyncio.Task] = set() self._assistant_turn_start_timestamp = "" @@ -820,6 +821,15 @@ class LLMAssistantAggregator(LLMContextAggregator): run_llm = False + # Append any images that were generated by function calls. + if frame.tool_call_id in self._function_calls_image_results: + image_frame = self._function_calls_image_results[frame.tool_call_id] + + del self._function_calls_image_results[frame.tool_call_id] + + # If an image frame has been added to the context, let's run inference. + run_llm = await self._maybe_append_image_to_context(image_frame) + # Run inference if the function call result requires it. if frame.result: if properties and properties.run_llm is not None: @@ -856,31 +866,24 @@ class LLMAssistantAggregator(LLMContextAggregator): self._update_function_call_result(frame.function_name, frame.tool_call_id, "CANCELLED") del self._function_calls_in_progress[frame.tool_call_id] - def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): - for message in self._context.get_messages(): - if ( - not isinstance(message, LLMSpecificMessage) - and message["role"] == "tool" - and message["tool_call_id"] - and message["tool_call_id"] == tool_call_id - ): - message["content"] = result - async def _handle_user_image_frame(self, frame: UserImageRawFrame): - if not frame.append_to_context: - return + image_appended = False - logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") + # Check if this image is a result of a function call if so, let's cache. + # TODO(aleix): The function call might have already been executed + # because FunctionCallResultFrame was just faster, in that case we just + # push the context frame now. + if ( + frame.request + and frame.request.tool_call_id + and frame.request.tool_call_id in self._function_calls_in_progress + ): + self._function_calls_image_results[frame.request.tool_call_id] = frame + else: + image_appended = await self._maybe_append_image_to_context(frame) - await self._context.add_image_frame_message( - format=frame.format, - size=frame.size, - image=frame.image, - text=frame.text, - ) - - await self._trigger_assistant_turn_stopped() - await self.push_context_frame(FrameDirection.UPSTREAM) + if image_appended: + await self.push_context_frame(FrameDirection.UPSTREAM) async def _handle_assistant_image_frame(self, frame: AssistantImageRawFrame): logger.debug(f"{self} Appending AssistantImageRawFrame to LLM context (size: {frame.size})") @@ -970,6 +973,31 @@ class LLMAssistantAggregator(LLMContextAggregator): await self._call_event_handler("on_assistant_thought", message) + async def _maybe_append_image_to_context(self, frame: UserImageRawFrame) -> bool: + if not frame.append_to_context: + return False + + logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") + + await self._context.add_image_frame_message( + format=frame.format, + size=frame.size, + image=frame.image, + text=frame.text, + ) + + return True + + def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any): + for message in self._context.get_messages(): + if ( + not isinstance(message, LLMSpecificMessage) + and message["role"] == "tool" + and message["tool_call_id"] + and message["tool_call_id"] == tool_call_id + ): + message["content"] = result + def _context_updated_task_finished(self, task: asyncio.Task): self._context_updated_tasks.discard(task) diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index a6717b5cc..0368c8feb 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -519,9 +519,10 @@ class LLMService(AIService): UserImageRequestFrame( user_id=user_id, text=text_content, - # Deprecated fields below. + append_to_context=True, function_name=function_name, tool_call_id=tool_call_id, + # Deprecated fields below. context=text_content, ), FrameDirection.UPSTREAM, diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 147a9a111..203cbb9fa 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -27,7 +27,6 @@ from pipecat.frames.frames import ( CancelFrame, ControlFrame, EndFrame, - ErrorFrame, Frame, InputAudioRawFrame, InputTransportMessageFrame, @@ -1844,7 +1843,6 @@ class DailyInputTransport(BaseInputTransport): format=video_frame.color_format, text=request_frame.text if request_frame else None, append_to_context=request_frame.append_to_context if request_frame else None, - # Deprecated fields below. request=request_frame, ) frame.transport_source = video_source diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index 6b6a14829..e9a07cf40 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -680,7 +680,6 @@ class SmallWebRTCInputTransport(BaseInputTransport): format=video_frame.format, text=request_text, append_to_context=add_to_context, - # Deprecated fields below. request=request_frame, ) image_frame.transport_source = video_source