Change FunctionInProgressFrame.llm_specific_extra to a more generic FunctionInProgressFrame.append_extra_context_messages.

src/pipecat/adapters/services/gemini_adapter.py

@@ -222,9 +222,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
                 # messages
                 if (
                     isinstance(message.message, dict)
-                    and message.message.get("type") == "tool_call_extra"
+                    and message.message.get("type") == "fn_call_thought_signature"
-                    and isinstance(data := message.message.get("data"), dict)
+                    and (thought_signature := message.message.get("signature"))
-                    and (thought_signature := data.get("thought_signature"))
                 ):
                     self._apply_function_call_thought_signature_to_messages(
                         thought_signature, message.message.get("tool_call_id"), messages

src/pipecat/frames/frames.py

@@ -38,7 +38,7 @@ from pipecat.utils.time import nanoseconds_to_str
 from pipecat.utils.utils import obj_count, obj_id
 
 if TYPE_CHECKING:
-    from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven
+    from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage, NotGiven
     from pipecat.processors.frame_processor import FrameProcessor
 
 
@@ -1218,16 +1218,16 @@ class FunctionCallFromLLM:
         tool_call_id: A unique identifier for the function call.
         arguments: The arguments to pass to the function.
         context: The LLM context when the function call was made.
-        llm_specific_extra: Optional extra data specific to particular LLMs, e.g.:
+        append_extra_context_messages: Optional extra messages to append to the
-            {"google": {"thought_signature": ...}}
+            context after the function call message. Used to add Google
-            Uses the LLM adapter's ID for LLM-specific messages as the key.
+            function-call-related thought signatures to the context.
     """
 
     function_name: str
     tool_call_id: str
     arguments: Mapping[str, Any]
     context: Any
-    llm_specific_extra: Optional[Dict[str, Any]] = None
+    append_extra_context_messages: Optional[List["LLMContextMessage"]] = None
 
 
 @dataclass
@@ -1765,18 +1765,17 @@ class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame):
         function_name: Name of the function being executed.
         tool_call_id: Unique identifier for this function call.
         arguments: Arguments passed to the function.
-        llm_specific_extra: Optional extra data specific to particular LLMs, e.g.:
-            {"google": {"thought_signature": ...}}
-            Uses the LLM adapter's ID for LLM-specific messages as the key.
         cancel_on_interruption: Whether to cancel this call if interrupted.
-
+        append_extra_context_messages: Optional extra messages to append to the
+            context after the function call message. Used to add Google
+            function-call-related thought signatures to the context.
     """
 
     function_name: str
     tool_call_id: str
     arguments: Any
-    llm_specific_extra: Optional[Dict[str, Any]] = None
     cancel_on_interruption: bool = False
+    append_extra_context_messages: Optional[List["LLMContextMessage"]] = None
 
 
 @dataclass

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -743,23 +743,9 @@ class LLMAssistantAggregator(LLMContextAggregator):
             }
         )
 
-        # If there's LLM-specific extra data associated with this function call
+        # Append to context any specified extra context messages
-        # add it to the context as an adjacent LLM-specific message. The
+        if frame.append_extra_context_messages:
-        # LLM-specific adapter can then use this extra data as needed, for
+            self._context.add_messages(frame.append_extra_context_messages)
-        # example by merging it into the tool call message. This is how Google's
-        # "thought_signature" makes it into the tool call message.
-        if frame.llm_specific_extra:
-            for key, value in frame.llm_specific_extra.items():
-                self._context.add_message(
-                    LLMSpecificMessage(
-                        llm=key,
-                        message={
-                            "type": "tool_call_extra",
-                            "data": value,
-                            "tool_call_id": frame.tool_call_id,
-                        },
-                    )
-                )
 
         self._function_calls_in_progress[frame.tool_call_id] = frame
 

src/pipecat/services/google/llm.py

@@ -43,7 +43,7 @@ from pipecat.frames.frames import (
     UserImageRawFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
-from pipecat.processors.aggregators.llm_context import LLMContext
+from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
 from pipecat.processors.aggregators.llm_response import (
     LLMAssistantAggregatorParams,
     LLMUserAggregatorParams,
@@ -985,11 +985,16 @@ class GoogleLLMService(LLMService):
                                         tool_call_id=id,
                                         function_name=function_call.name,
                                         arguments=function_call.args or {},
-                                        llm_specific_extra={
+                                        append_extra_context_messages=[
-                                            self.get_llm_adapter().id_for_llm_specific_messages: {
+                                            LLMSpecificMessage(
-                                                "thought_signature": part.thought_signature
+                                                llm=self.get_llm_adapter().id_for_llm_specific_messages,
-                                            }
+                                                message={
-                                        }
+                                                    "type": "fn_call_thought_signature",
+                                                    "signature": part.thought_signature,
+                                                    "tool_call_id": id,
+                                                },
+                                            )
+                                        ]
                                         if part.thought_signature
                                         else None,
                                     )

src/pipecat/services/llm_service.py

@@ -14,6 +14,7 @@ from typing import (
     Awaitable,
     Callable,
     Dict,
+    List,
     Mapping,
     Optional,
     Protocol,
@@ -44,7 +45,11 @@ from pipecat.frames.frames import (
     StartFrame,
     UserImageRequestFrame,
 )
-from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
+from pipecat.processors.aggregators.llm_context import (
+    LLMContext,
+    LLMContextMessage,
+    LLMSpecificMessage,
+)
 from pipecat.processors.aggregators.llm_response import (
     LLMAssistantAggregatorParams,
     LLMUserAggregatorParams,
@@ -127,9 +132,9 @@ class FunctionCallRunnerItem:
         tool_call_id: A unique identifier for the function call.
         arguments: The arguments for the function.
         context: The LLM context.
-        llm_specific_extra: Optional extra data specific to particular LLMs, e.g.:
+        append_extra_context_messages: Optional extra messages to append to the
-            {"google": {"thought_signature": ...}}
+            context after the function call message. Used to add Google
-            Uses the LLM adapter's ID for LLM-specific messages as the key.
+            function-call-related thought signatures to the context.
         run_llm: Optional flag to control LLM execution after function call.
     """
 
@@ -138,7 +143,7 @@ class FunctionCallRunnerItem:
     tool_call_id: str
     arguments: Mapping[str, Any]
     context: OpenAILLMContext | LLMContext
-    llm_specific_extra: Optional[Dict[str, Any]] = None
+    append_extra_context_messages: Optional[List[LLMContextMessage]] = None
     run_llm: Optional[bool] = None
 
 
@@ -460,7 +465,7 @@ class LLMService(AIService):
                     tool_call_id=function_call.tool_call_id,
                     arguments=function_call.arguments,
                     context=function_call.context,
-                    llm_specific_extra=function_call.llm_specific_extra,
+                    append_extra_context_messages=function_call.append_extra_context_messages,
                 )
             )
 
@@ -585,7 +590,7 @@ class LLMService(AIService):
             function_name=runner_item.function_name,
             tool_call_id=runner_item.tool_call_id,
             arguments=runner_item.arguments,
-            llm_specific_extra=runner_item.llm_specific_extra,
+            append_extra_context_messages=runner_item.append_extra_context_messages,
             cancel_on_interruption=item.cancel_on_interruption,
         )