wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change FunctionInProgressFrame.llm_specific_extra to a more generic FunctionInProgressFrame.append_extra_context_messages.

Browse Source
This commit is contained in:
Paul Kompfner
2025-12-08 10:50:19 -05:00
parent 61674d7758
commit 17203ba3e6
5 changed files with 37 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/pipecat/adapters/services/gemini_adapter.py
View File

@@ -222,9 +222,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
# messages
if (
isinstance(message.message, dict)
and message.message.get("type") == "tool_call_extra"
and isinstance(data := message.message.get("data"), dict)
and (thought_signature := data.get("thought_signature"))
and message.message.get("type") == "fn_call_thought_signature"
and (thought_signature := message.message.get("signature"))
):
self._apply_function_call_thought_signature_to_messages(
thought_signature, message.message.get("tool_call_id"), messages

19
src/pipecat/frames/frames.py
View File

@@ -38,7 +38,7 @@ from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
if TYPE_CHECKING:
from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven
from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage, NotGiven
from pipecat.processors.frame_processor import FrameProcessor
@@ -1218,16 +1218,16 @@ class FunctionCallFromLLM:
tool_call_id: A unique identifier for the function call.
arguments: The arguments to pass to the function.
context: The LLM context when the function call was made.
llm_specific_extra: Optional extra data specific to particular LLMs, e.g.:
{"google": {"thought_signature": ...}}
Uses the LLM adapter's ID for LLM-specific messages as the key.
append_extra_context_messages: Optional extra messages to append to the
context after the function call message. Used to add Google
function-call-related thought signatures to the context.
"""
function_name: str
tool_call_id: str
arguments: Mapping[str, Any]
context: Any
llm_specific_extra: Optional[Dict[str, Any]] = None
append_extra_context_messages: Optional[List["LLMContextMessage"]] = None
@dataclass
@@ -1765,18 +1765,17 @@ class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame):
function_name: Name of the function being executed.
tool_call_id: Unique identifier for this function call.
arguments: Arguments passed to the function.
llm_specific_extra: Optional extra data specific to particular LLMs, e.g.:
{"google": {"thought_signature": ...}}
Uses the LLM adapter's ID for LLM-specific messages as the key.
cancel_on_interruption: Whether to cancel this call if interrupted.
append_extra_context_messages: Optional extra messages to append to the
context after the function call message. Used to add Google
function-call-related thought signatures to the context.
"""
function_name: str
tool_call_id: str
arguments: Any
llm_specific_extra: Optional[Dict[str, Any]] = None
cancel_on_interruption: bool = False
append_extra_context_messages: Optional[List["LLMContextMessage"]] = None
@dataclass

20
src/pipecat/processors/aggregators/llm_response_universal.py
View File

@@ -743,23 +743,9 @@ class LLMAssistantAggregator(LLMContextAggregator):
}
)
# If there's LLM-specific extra data associated with this function call
# add it to the context as an adjacent LLM-specific message. The
# LLM-specific adapter can then use this extra data as needed, for
# example by merging it into the tool call message. This is how Google's
# "thought_signature" makes it into the tool call message.
if frame.llm_specific_extra:
for key, value in frame.llm_specific_extra.items():
self._context.add_message(
LLMSpecificMessage(
llm=key,
message={
"type": "tool_call_extra",
"data": value,
"tool_call_id": frame.tool_call_id,
},
)
)
# Append to context any specified extra context messages
if frame.append_extra_context_messages:
self._context.add_messages(frame.append_extra_context_messages)
self._function_calls_in_progress[frame.tool_call_id] = frame

17
src/pipecat/services/google/llm.py
View File

@@ -43,7 +43,7 @@ from pipecat.frames.frames import (
UserImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMUserAggregatorParams,
@@ -985,11 +985,16 @@ class GoogleLLMService(LLMService):
tool_call_id=id,
function_name=function_call.name,
arguments=function_call.args or {},
llm_specific_extra={
self.get_llm_adapter().id_for_llm_specific_messages: {
"thought_signature": part.thought_signature
}
}
append_extra_context_messages=[
LLMSpecificMessage(
llm=self.get_llm_adapter().id_for_llm_specific_messages,
message={
"type": "fn_call_thought_signature",
"signature": part.thought_signature,
"tool_call_id": id,
},
)
]
if part.thought_signature
else None,
)

19
src/pipecat/services/llm_service.py
View File

@@ -14,6 +14,7 @@ from typing import (
Awaitable,
Callable,
Dict,
List,
Mapping,
Optional,
Protocol,
@@ -44,7 +45,11 @@ from pipecat.frames.frames import (
StartFrame,
UserImageRequestFrame,
)
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
from pipecat.processors.aggregators.llm_context import (
LLMContext,
LLMContextMessage,
LLMSpecificMessage,
)
from pipecat.processors.aggregators.llm_response import (
LLMAssistantAggregatorParams,
LLMUserAggregatorParams,
@@ -127,9 +132,9 @@ class FunctionCallRunnerItem:
tool_call_id: A unique identifier for the function call.
arguments: The arguments for the function.
context: The LLM context.
llm_specific_extra: Optional extra data specific to particular LLMs, e.g.:
{"google": {"thought_signature": ...}}
Uses the LLM adapter's ID for LLM-specific messages as the key.
append_extra_context_messages: Optional extra messages to append to the
context after the function call message. Used to add Google
function-call-related thought signatures to the context.
run_llm: Optional flag to control LLM execution after function call.
"""
@@ -138,7 +143,7 @@ class FunctionCallRunnerItem:
tool_call_id: str
arguments: Mapping[str, Any]
context: OpenAILLMContext | LLMContext
llm_specific_extra: Optional[Dict[str, Any]] = None
append_extra_context_messages: Optional[List[LLMContextMessage]] = None
run_llm: Optional[bool] = None
@@ -460,7 +465,7 @@ class LLMService(AIService):
tool_call_id=function_call.tool_call_id,
arguments=function_call.arguments,
context=function_call.context,
llm_specific_extra=function_call.llm_specific_extra,
append_extra_context_messages=function_call.append_extra_context_messages,
)
)
@@ -585,7 +590,7 @@ class LLMService(AIService):
function_name=runner_item.function_name,
tool_call_id=runner_item.tool_call_id,
arguments=runner_item.arguments,
llm_specific_extra=runner_item.llm_specific_extra,
append_extra_context_messages=runner_item.append_extra_context_messages,
cancel_on_interruption=item.cancel_on_interruption,
)