From 17203ba3e699bd723b29a71548e1b9cb8ae7e68c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 10:50:19 -0500 Subject: [PATCH] Change `FunctionInProgressFrame.llm_specific_extra` to a more generic `FunctionInProgressFrame.append_extra_context_messages`. --- .../adapters/services/gemini_adapter.py | 5 ++--- src/pipecat/frames/frames.py | 19 +++++++++--------- .../aggregators/llm_response_universal.py | 20 +++---------------- src/pipecat/services/google/llm.py | 17 ++++++++++------ src/pipecat/services/llm_service.py | 19 +++++++++++------- 5 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index ec0afa885..52d8cd114 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -222,9 +222,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # messages if ( isinstance(message.message, dict) - and message.message.get("type") == "tool_call_extra" - and isinstance(data := message.message.get("data"), dict) - and (thought_signature := data.get("thought_signature")) + and message.message.get("type") == "fn_call_thought_signature" + and (thought_signature := message.message.get("signature")) ): self._apply_function_call_thought_signature_to_messages( thought_signature, message.message.get("tool_call_id"), messages diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 9209d603a..43cc357a2 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -38,7 +38,7 @@ from pipecat.utils.time import nanoseconds_to_str from pipecat.utils.utils import obj_count, obj_id if TYPE_CHECKING: - from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven + from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage, NotGiven from pipecat.processors.frame_processor import FrameProcessor @@ -1218,16 +1218,16 @@ class FunctionCallFromLLM: tool_call_id: A unique identifier for the function call. arguments: The arguments to pass to the function. context: The LLM context when the function call was made. - llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: - {"google": {"thought_signature": ...}} - Uses the LLM adapter's ID for LLM-specific messages as the key. + append_extra_context_messages: Optional extra messages to append to the + context after the function call message. Used to add Google + function-call-related thought signatures to the context. """ function_name: str tool_call_id: str arguments: Mapping[str, Any] context: Any - llm_specific_extra: Optional[Dict[str, Any]] = None + append_extra_context_messages: Optional[List["LLMContextMessage"]] = None @dataclass @@ -1765,18 +1765,17 @@ class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame): function_name: Name of the function being executed. tool_call_id: Unique identifier for this function call. arguments: Arguments passed to the function. - llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: - {"google": {"thought_signature": ...}} - Uses the LLM adapter's ID for LLM-specific messages as the key. cancel_on_interruption: Whether to cancel this call if interrupted. - + append_extra_context_messages: Optional extra messages to append to the + context after the function call message. Used to add Google + function-call-related thought signatures to the context. """ function_name: str tool_call_id: str arguments: Any - llm_specific_extra: Optional[Dict[str, Any]] = None cancel_on_interruption: bool = False + append_extra_context_messages: Optional[List["LLMContextMessage"]] = None @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 76641f0e0..2f7b5b97e 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -743,23 +743,9 @@ class LLMAssistantAggregator(LLMContextAggregator): } ) - # If there's LLM-specific extra data associated with this function call - # add it to the context as an adjacent LLM-specific message. The - # LLM-specific adapter can then use this extra data as needed, for - # example by merging it into the tool call message. This is how Google's - # "thought_signature" makes it into the tool call message. - if frame.llm_specific_extra: - for key, value in frame.llm_specific_extra.items(): - self._context.add_message( - LLMSpecificMessage( - llm=key, - message={ - "type": "tool_call_extra", - "data": value, - "tool_call_id": frame.tool_call_id, - }, - ) - ) + # Append to context any specified extra context messages + if frame.append_extra_context_messages: + self._context.add_messages(frame.append_extra_context_messages) self._function_calls_in_progress[frame.tool_call_id] = frame diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 408918287..393f2c814 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -43,7 +43,7 @@ from pipecat.frames.frames import ( UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, @@ -985,11 +985,16 @@ class GoogleLLMService(LLMService): tool_call_id=id, function_name=function_call.name, arguments=function_call.args or {}, - llm_specific_extra={ - self.get_llm_adapter().id_for_llm_specific_messages: { - "thought_signature": part.thought_signature - } - } + append_extra_context_messages=[ + LLMSpecificMessage( + llm=self.get_llm_adapter().id_for_llm_specific_messages, + message={ + "type": "fn_call_thought_signature", + "signature": part.thought_signature, + "tool_call_id": id, + }, + ) + ] if part.thought_signature else None, ) diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 71e31d9b6..91358dcf3 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -14,6 +14,7 @@ from typing import ( Awaitable, Callable, Dict, + List, Mapping, Optional, Protocol, @@ -44,7 +45,11 @@ from pipecat.frames.frames import ( StartFrame, UserImageRequestFrame, ) -from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage +from pipecat.processors.aggregators.llm_context import ( + LLMContext, + LLMContextMessage, + LLMSpecificMessage, +) from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, @@ -127,9 +132,9 @@ class FunctionCallRunnerItem: tool_call_id: A unique identifier for the function call. arguments: The arguments for the function. context: The LLM context. - llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: - {"google": {"thought_signature": ...}} - Uses the LLM adapter's ID for LLM-specific messages as the key. + append_extra_context_messages: Optional extra messages to append to the + context after the function call message. Used to add Google + function-call-related thought signatures to the context. run_llm: Optional flag to control LLM execution after function call. """ @@ -138,7 +143,7 @@ class FunctionCallRunnerItem: tool_call_id: str arguments: Mapping[str, Any] context: OpenAILLMContext | LLMContext - llm_specific_extra: Optional[Dict[str, Any]] = None + append_extra_context_messages: Optional[List[LLMContextMessage]] = None run_llm: Optional[bool] = None @@ -460,7 +465,7 @@ class LLMService(AIService): tool_call_id=function_call.tool_call_id, arguments=function_call.arguments, context=function_call.context, - llm_specific_extra=function_call.llm_specific_extra, + append_extra_context_messages=function_call.append_extra_context_messages, ) ) @@ -585,7 +590,7 @@ class LLMService(AIService): function_name=runner_item.function_name, tool_call_id=runner_item.tool_call_id, arguments=runner_item.arguments, - llm_specific_extra=runner_item.llm_specific_extra, + append_extra_context_messages=runner_item.append_extra_context_messages, cancel_on_interruption=item.cancel_on_interruption, )