From 217f03b9cc599d5bb7443c0ce741f1ea62a7c735 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 2 Dec 2025 14:11:39 -0500 Subject: [PATCH 01/21] Add additional functionality related to "thinking", for Google and Anthropic LLMs. Thinking, sometimes called "extended thinking" or "reasoning", is an LLM process where the model takes some additional time before giving an answer. It's useful for complex tasks that may require some level of planning and structured, step-by-step reasoning. The model can output its thoughts (or thought summaries, depending on the model) in addition to the answer. The thoughts are usually pretty granular and not really suitable for being spoken out loud in a conversation, but can be useful for logging or prompt debugging. Here's what's added: 1. New typed input parameters for Google and Anthropic LLMs that control the models' thinking behavior (like how much thinking to do, and whether to output thoughts or thought summaries). 2. New frames for representing thoughts output by LLMs. 3. A generic mechanism for associating extra LLM-specific data with a function call in context, used specifically to support Google's function-call-related "thought signatures", which are necessary to ensure thinking continuity between function calls in a chain (where the model thinks, makes a function call, thinks some more, etc.) 4. A generic mechanism for recording LLM thoughts to context, used specifically to support Anthropic, whose thought signatures are expected to appear alongside the text of the thoughts within assistant context messages. 5. An expansion of `TranscriptProcessor` to process LLM thoughts in addition to user and assistant utterances. --- .../07n-interruptible-google-http.py | 6 +- .../foundational/07n-interruptible-google.py | 6 +- .../07s-interruptible-google-audio-in.py | 6 +- .../foundational/49-thinking-functions.py | 222 ++++++++++++++++++ examples/foundational/49-thinking.py | 198 ++++++++++++++++ .../adapters/services/anthropic_adapter.py | 36 ++- .../adapters/services/gemini_adapter.py | 44 ++++ src/pipecat/frames/frames.py | 85 ++++++- .../aggregators/llm_response_universal.py | 83 +++++++ .../processors/transcript_processor.py | 139 ++++++++++- src/pipecat/services/anthropic/llm.py | 57 ++++- src/pipecat/services/google/llm.py | 68 +++++- src/pipecat/services/llm_service.py | 6 + 13 files changed, 940 insertions(+), 16 deletions(-) create mode 100644 examples/foundational/49-thinking-functions.py create mode 100644 examples/foundational/49-thinking.py diff --git a/examples/foundational/07n-interruptible-google-http.py b/examples/foundational/07n-interruptible-google-http.py index 2ef65e474..4a0382990 100644 --- a/examples/foundational/07n-interruptible-google-http.py +++ b/examples/foundational/07n-interruptible-google-http.py @@ -75,8 +75,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleLLMService( api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.5-flash", - # turn on thinking if you want it - # params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}),) + # force a certain amount of thinking if you want it + # params=GoogleLLMService.InputParams( + # thinking=GoogleLLMService.ThinkingConfig(thinking_budget=4096) + # ), ) messages = [ diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index 73dd49e78..28b61c151 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -75,8 +75,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleLLMService( api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.5-flash", - # turn on thinking if you want it - # params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}),) + # force a certain amount of thinking if you want it + # params=GoogleLLMService.InputParams( + # thinking=GoogleLLMService.ThinkingConfig(thinking_budget=4096) + # ), ) messages = [ diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 67772e40d..90bff6062 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -224,8 +224,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = GoogleLLMService( api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.5-flash", - # turn on thinking if you want it - # params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}), + # force a certain amount of thinking if you want it + # params=GoogleLLMService.InputParams( + # thinking=GoogleLLMService.ThinkingConfig(thinking_budget=4096) + # ), ) tts = GoogleTTSService( diff --git a/examples/foundational/49-thinking-functions.py b/examples/foundational/49-thinking-functions.py new file mode 100644 index 000000000..2b96e304d --- /dev/null +++ b/examples/foundational/49-thinking-functions.py @@ -0,0 +1,222 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os +import random +import sys + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, ThoughtTranscriptionMessage, TranscriptionMessage +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.anthropic.llm import AnthropicLLMService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +async def check_flight_status(params: FunctionCallParams, flight_number: str): + """Check the status of a flight. Returns status (e.g., "on time", "delayed") and departure time. + + Args: + flight_number (str): The flight number, e.g. "AA100". + """ + await params.result_callback({"status": "delayed", "departure_time": "14:30"}) + + +async def book_taxi(params: FunctionCallParams, time: str): + """Book a taxi for a given time. Returns status (e.g., "done"). + + Args: + time (str): The time to book the taxi for, e.g. "15:00". + """ + await params.result_callback({"status": "done"}) + + +# LLM provider constants +LLM_ANTHROPIC = "anthropic" +LLM_GOOGLE = "google" +LLM_DEFAULT = LLM_GOOGLE + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot( + transport: BaseTransport, runner_args: RunnerArguments, llm_provider: str = LLM_DEFAULT +): + logger.info(f"Starting bot with {llm_provider.capitalize()} LLM") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + if llm_provider == LLM_ANTHROPIC: + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), + params=AnthropicLLMService.InputParams( + thinking=AnthropicLLMService.ThinkingConfig(type="enabled", budget_tokens=2048) + ), + ) + elif llm_provider == LLM_GOOGLE: + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + params=GoogleLLMService.InputParams( + thinking=GoogleLLMService.ThinkingConfig( + thinking_budget=-1, # Dynamic thinking + include_thoughts=True, + ) + ), + ) + else: + raise ValueError(f"Unsupported LLM provider: {llm_provider}") + + llm.register_direct_function(check_flight_status) + llm.register_direct_function(book_taxi) + + tools = ToolsSchema(standard_tools=[check_flight_status, book_taxi]) + + transcript = TranscriptProcessor() + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + transcript.thought(), # Thought transcripts + tts, # TTS + transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + # This example comes from Gemini docs. + messages.append( + { + "role": "user", + "content": "Check the status of flight AA100 and book me a taxi 2 hours beforehand if the flight is delayed.", + } + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + for msg in frame.messages: + if isinstance(msg, (ThoughtTranscriptionMessage, TranscriptionMessage)): + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + role = "THOUGHT" if isinstance(msg, ThoughtTranscriptionMessage) else msg.role + logger.info(f"Transcript: {timestamp}{role}: {msg.content}") + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + # Get llm_provider from module attribute set in __main__ + llm_provider = getattr(sys.modules[__name__], "llm_provider", LLM_DEFAULT) + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args, llm_provider) + + +if __name__ == "__main__": + # Parse custom arguments before calling runner main() + parser = argparse.ArgumentParser(description="Thinking LLM Bot") + parser.add_argument( + "--llm", + type=str, + choices=[LLM_ANTHROPIC, LLM_GOOGLE], + default=LLM_DEFAULT, + help=f"LLM provider to use (default: {LLM_DEFAULT})", + ) + # Parse only known args to allow runner's main() to handle its own args + args, remaining = parser.parse_known_args() + + # Store the llm_provider in sys.modules for bot() function to access + sys.modules[__name__].llm_provider = args.llm + + # Restore sys.argv with remaining args for runner's main() + sys.argv[1:] = remaining + + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/49-thinking.py b/examples/foundational/49-thinking.py new file mode 100644 index 000000000..512163be4 --- /dev/null +++ b/examples/foundational/49-thinking.py @@ -0,0 +1,198 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os +import random +import sys + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, ThoughtTranscriptionMessage, TranscriptionMessage +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.anthropic.llm import AnthropicLLMService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# LLM provider constants +LLM_ANTHROPIC = "anthropic" +LLM_GOOGLE = "google" +LLM_DEFAULT = LLM_GOOGLE + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot( + transport: BaseTransport, runner_args: RunnerArguments, llm_provider: str = LLM_DEFAULT +): + logger.info(f"Starting bot with {llm_provider.capitalize()} LLM") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + if llm_provider == LLM_ANTHROPIC: + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), + params=AnthropicLLMService.InputParams( + thinking=AnthropicLLMService.ThinkingConfig(type="enabled", budget_tokens=2048) + ), + ) + elif llm_provider == LLM_GOOGLE: + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + params=GoogleLLMService.InputParams( + thinking=GoogleLLMService.ThinkingConfig( + thinking_budget=-1, # Dynamic thinking + include_thoughts=True, + ) + ), + ) + else: + raise ValueError(f"Unsupported LLM provider: {llm_provider}") + + transcript = TranscriptProcessor() + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + transcript.thought(), # Thought transcripts + tts, # TTS + transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Choose a random prompt to demonstrate thinking capabilities. + # These prompts were chosen from Google and Anthropic docs. + thinking_prompt_1 = "Analogize photosynthesis and growing up." + thinking_prompt_2 = "Compare and contrast electric cars and hybrid cars." + thinking_prompt_3 = "Are there an infinite number of prime numbers such that n mod 4 == 3?" + selected_prompt = random.choice([thinking_prompt_1, thinking_prompt_2, thinking_prompt_3]) + + # Kick off the conversation. + messages.append({"role": "user", "content": selected_prompt}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + for msg in frame.messages: + if isinstance(msg, (ThoughtTranscriptionMessage, TranscriptionMessage)): + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + role = "THOUGHT" if isinstance(msg, ThoughtTranscriptionMessage) else msg.role + logger.info(f"Transcript: {timestamp}{role}: {msg.content}") + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + # Get llm_provider from module attribute set in __main__ + llm_provider = getattr(sys.modules[__name__], "llm_provider", LLM_DEFAULT) + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args, llm_provider) + + +if __name__ == "__main__": + # Parse custom arguments before calling runner main() + parser = argparse.ArgumentParser(description="Thinking LLM Bot") + parser.add_argument( + "--llm", + type=str, + choices=[LLM_ANTHROPIC, LLM_GOOGLE], + default=LLM_DEFAULT, + help=f"LLM provider to use (default: {LLM_DEFAULT})", + ) + # Parse only known args to allow runner's main() to handle its own args + args, remaining = parser.parse_known_args() + + # Store the llm_provider in sys.modules for bot() function to access + sys.modules[__name__].llm_provider = args.llm + + # Restore sys.argv with remaining args for runner's main() + sys.argv[1:] = remaining + + from pipecat.runner.run import main + + main() diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index 75fa5899d..e111b34df 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -165,9 +165,43 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): def _from_universal_context_message(self, message: LLMContextMessage) -> MessageParam: if isinstance(message, LLMSpecificMessage): - return copy.deepcopy(message.message) + return self._from_anthropic_specific_message(message) return self._from_standard_message(message) + def _from_anthropic_specific_message(self, message: LLMSpecificMessage) -> MessageParam: + """Convert LLMSpecificMessage to Anthropic format. + + Assumes that we already know the message is intended for Anthropic. + + Args: + message: Message in LLMSpecificMessage format. + """ + # Handle special case of thought messages. + # These can be converted to standalone "assistant" messages; later + # these thinking messages will be properly merged into the assistant + # response messages before the context is sent to Anthropic for the + # next turn. + if ( + isinstance(message.message, dict) + and message.message.get("type") == "thought" + and (text := message.message.get("text")) + and isinstance(metadata := message.message.get("metadata"), dict) + and (signature := metadata.get("signature")) + ): + return { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": text, + "signature": signature, + } + ], + } + + # Fallback to assumption that the message is already in Anthropic format + return copy.deepcopy(message.message) + def _from_standard_message(self, message: LLMStandardMessage) -> MessageParam: """Convert standard universal context message to Anthropic format. diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index a4f70b1fa..fd91b818f 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -167,6 +167,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): class MessageConversionResult: """Result of converting a single universal context message to Google format. + # TODO: content could be other things, like {"tool_call_extra": ...}, for example. All bets are off when it's LLMSpecificMessage. Either content (a Google Content object) or a system instruction string is guaranteed to be set. @@ -219,6 +220,20 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, ), ) + + # If we found a function-call-related thought_signature, modify the + # corresponding function call message to include it + if ( + isinstance(result.content, dict) + and result.content.get("type") == "tool_call_extra" + and isinstance(data := result.content.get("data"), dict) + and (thought_signature := data.get("thought_signature")) + ): + self._apply_function_call_thought_signature_to_messages( + thought_signature, result.content.get("tool_call_id"), messages + ) + continue + # Each result is either a Content or a system instruction if result.content: messages.append(result.content) @@ -410,3 +425,32 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): content=Content(role=role, parts=parts), tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, ) + + def _apply_function_call_thought_signature_to_messages( + self, thought_signature: bytes, tool_call_id: str, messages: List[Content] + ) -> None: + """Apply tool_call_extra metadata to the corresponding function call message. + + Args: + thought_signature: The thought signature bytes to apply. + tool_call_id: ID of the tool call message to find and modify. + messages: List of Content messages to search through. + """ + # Search backwards through messages to find the matching function call + for message in reversed(messages): + if not isinstance(message, Content) or not message.parts: + continue + # Find the specific part with the matching function call + for part in message.parts: + if ( + hasattr(part, "function_call") + and part.function_call + and part.function_call.id == tool_call_id + ): + part.thought_signature = thought_signature + break + else: + # Continue outer loop if inner loop didn't break + continue + # Break outer loop if inner loop broke (found match) + break diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 9cb969f28..2c3b802ec 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -512,6 +512,14 @@ class TranscriptionMessage: timestamp: Optional[str] = None +@dataclass +class ThoughtTranscriptionMessage: + """An LLM thought message in a conversation transcript.""" + + content: str + timestamp: Optional[str] = None + + @dataclass class TranscriptionUpdateFrame(DataFrame): """Frame containing new messages added to conversation transcript. @@ -556,7 +564,7 @@ class TranscriptionUpdateFrame(DataFrame): messages: List of new transcript messages that were added. """ - messages: List[TranscriptionMessage] + messages: List[TranscriptionMessage | ThoughtTranscriptionMessage] def __str__(self): pts = format_pts(self.pts) @@ -577,6 +585,73 @@ class LLMContextFrame(Frame): context: "LLMContext" +@dataclass +class LLMThoughtStartFrame(ControlFrame): + """Frame indicating the start of an LLM thought. + + Parameters: + append_to_context: Whether the thought should be appended to the LLM context. + If it is appended, the `llm` field is required, since it will be + appended as an `LLMSpecificMessage`. + llm: Optional identifier of the LLM provider for LLM-specific handling. + Only required if `append_to_context` is True. + """ + + append_to_context: bool = False + llm: Optional[str] = None + + def __post_init__(self): + super().__post_init__() + if self.append_to_context and self.llm is None: + raise ValueError("When append_to_context is True, llm must be set") + + def __str__(self): + pts = format_pts(self.pts) + return ( + f"{self.name}(pts: {pts}, append_to_context: {self.append_to_context}, llm: {self.llm})" + ) + + +@dataclass +class LLMThoughtTextFrame(DataFrame): + """Frame containing the text (or text chunk) of an LLM thought. + + Note that despite this containing text, it is a DataFrame and not a + TextFrame, to avoid most typical text processing, such as TTS. + + Parameters: + text: The text (or text chunk) of the thought. + """ + + text: str + includes_inter_frame_spaces: bool = field(init=False) + + def __post_init__(self): + super().__post_init__() + # Assume that thought text chunks include all necessary spaces + self.includes_inter_frame_spaces = True + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, thought text: {self.text})" + + +@dataclass +class LLMThoughtEndFrame(ControlFrame): + """Frame indicating the end of an LLM thought. + + Parameters: + thought_metadata: Optional metadata associated with the thought, + e.g. thought signature. + """ + + thought_metadata: Optional[Dict[str, Any]] = None + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, metadata: {self.thought_metadata})" + + @dataclass class LLMMessagesFrame(DataFrame): """Frame containing LLM messages for chat completion. @@ -1119,12 +1194,16 @@ class FunctionCallFromLLM: tool_call_id: A unique identifier for the function call. arguments: The arguments to pass to the function. context: The LLM context when the function call was made. + llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: + {"google": {"thought_signature": ...}} + Uses the LLM adapter's ID for LLM-specific messages as the key. """ function_name: str tool_call_id: str arguments: Mapping[str, Any] context: Any + llm_specific_extra: Optional[Dict[str, Any]] = None @dataclass @@ -1662,6 +1741,9 @@ class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame): function_name: Name of the function being executed. tool_call_id: Unique identifier for this function call. arguments: Arguments passed to the function. + llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: + {"google": {"thought_signature": ...}} + Uses the LLM adapter's ID for LLM-specific messages as the key. cancel_on_interruption: Whether to cancel this call if interrupted. """ @@ -1669,6 +1751,7 @@ class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame): function_name: str tool_call_id: str arguments: Any + llm_specific_extra: Optional[Dict[str, Any]] = None cancel_on_interruption: bool = False diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 69fc649ce..04f9c4ae9 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -47,6 +47,9 @@ from pipecat.frames.frames import ( LLMRunFrame, LLMSetToolChoiceFrame, LLMSetToolsFrame, + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, SpeechControlParamsFrame, StartFrame, TextFrame, @@ -592,6 +595,10 @@ class LLMAssistantAggregator(LLMContextAggregator): self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {} self._context_updated_tasks: Set[asyncio.Task] = set() + self._thought_aggregation_enabled = False + self._thought_llm: str = "" + self._thought_aggregation: List[TextPartForConcatenation] = [] + @property def has_function_calls_in_progress(self) -> bool: """Check if there are any function calls currently in progress. @@ -601,6 +608,17 @@ class LLMAssistantAggregator(LLMContextAggregator): """ return bool(self._function_calls_in_progress) + async def reset(self): + """Reset the aggregation state.""" + await super().reset() + await self._reset_thought_aggregation() # Just to be safe + + async def _reset_thought_aggregation(self): + """Reset the thought aggregation state.""" + self._thought_aggregation_enabled = False + self._thought_llm = "" + self._thought_aggregation = [] + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames for assistant response aggregation and function call management. @@ -619,6 +637,12 @@ class LLMAssistantAggregator(LLMContextAggregator): await self._handle_llm_end(frame) elif isinstance(frame, TextFrame): await self._handle_text(frame) + elif isinstance(frame, LLMThoughtStartFrame): + await self._handle_thought_start(frame) + elif isinstance(frame, LLMThoughtTextFrame): + await self._handle_thought_text(frame) + elif isinstance(frame, LLMThoughtEndFrame): + await self._handle_thought_end(frame) elif isinstance(frame, LLMRunFrame): await self._handle_llm_run(frame) elif isinstance(frame, LLMMessagesAppendFrame): @@ -716,6 +740,24 @@ class LLMAssistantAggregator(LLMContextAggregator): } ) + # If there's LLM-specific extra data associated with this function call + # add it to the context as an adjacent LLM-specific message. The + # LLM-specific adapter can then use this extra data as needed, for + # example by merging it into the tool call message. This is how Google's + # "thought_signature" makes it into the tool call message. + if frame.llm_specific_extra: + for key, value in frame.llm_specific_extra.items(): + self._context.add_message( + LLMSpecificMessage( + llm=key, + message={ + "type": "tool_call_extra", + "data": value, + "tool_call_id": frame.tool_call_id, + }, + ) + ) + self._function_calls_in_progress[frame.tool_call_id] = frame async def _handle_function_call_result(self, frame: FunctionCallResultFrame): @@ -824,6 +866,47 @@ class LLMAssistantAggregator(LLMContextAggregator): ) ) + async def _handle_thought_start(self, frame: LLMThoughtStartFrame): + if not self._started: + return + + await self._reset_thought_aggregation() + self._thought_aggregation_enabled = frame.append_to_context + self._thought_llm = frame.llm + + async def _handle_thought_text(self, frame: LLMThoughtTextFrame): + if not self._started or not self._thought_aggregation_enabled: + return + + # Make sure we really have text (spaces count, too!) + if len(frame.text) == 0: + return + + self._thought_aggregation.append( + TextPartForConcatenation( + frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces + ) + ) + + async def _handle_thought_end(self, frame: LLMThoughtEndFrame): + if not self._started or not self._thought_aggregation_enabled: + return + + thought = concatenate_aggregated_text(self._thought_aggregation) + llm = self._thought_llm + await self._reset_thought_aggregation() + + self._context.add_message( + LLMSpecificMessage( + llm=llm, + message={ + "type": "thought", + "text": thought, + "metadata": frame.thought_metadata, + }, + ) + ) + def _context_updated_task_finished(self, task: asyncio.Task): self._context_updated_tasks.discard(task) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 93e0c37b4..0dae1ad6a 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -20,6 +20,10 @@ from pipecat.frames.frames import ( EndFrame, Frame, InterruptionFrame, + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, + ThoughtTranscriptionMessage, TranscriptionFrame, TranscriptionMessage, TranscriptionUpdateFrame, @@ -202,10 +206,113 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): await self.push_frame(frame, direction) +class ThoughtTranscriptProcessor(BaseTranscriptProcessor): + """Processes LLM thought frames into timestamped thought messages. + + This processor aggregates LLM thought text frames into complete thoughts + and emits them as thought transcript messages. Thoughts are completed when: + + - A thought ends (LLMThoughtEndFrame) + - The bot is interrupted (InterruptionFrame) + - The pipeline ends (EndFrame) + """ + + def __init__(self, **kwargs): + """Initialize processor with thought aggregation state. + + Args: + **kwargs: Additional arguments passed to parent class. + """ + super().__init__(**kwargs) + self._current_thought_parts: List[TextPartForConcatenation] = [] + self._thought_start_time: Optional[str] = None + self._thought_active = False + + async def _emit_aggregated_thought(self): + """Aggregates and emits thought text fragments as a thought transcript message. + + This method aggregates thought fragments that may arrive in multiple + LLMThoughtTextFrame instances and emits them as a single ThoughtTranscriptionMessage. + """ + if self._current_thought_parts and self._thought_start_time: + content = concatenate_aggregated_text(self._current_thought_parts) + if content: + logger.trace(f"Emitting aggregated thought message: {content}") + message = ThoughtTranscriptionMessage( + content=content, + timestamp=self._thought_start_time, + ) + await self._emit_update([message]) + else: + logger.trace("No thought content to emit after stripping whitespace") + + # Reset aggregation state + self._current_thought_parts = [] + self._thought_start_time = None + self._thought_active = False + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames into thought transcript messages. + + Handles different frame types: + + - LLMThoughtStartFrame: Begins aggregating a new thought + - LLMThoughtTextFrame: Aggregates text for current thought + - LLMThoughtEndFrame: Completes current thought + - InterruptionFrame: Completes current thought due to interruption + - EndFrame: Completes current thought at pipeline end + - CancelFrame: Completes current thought due to cancellation + + Args: + frame: Input frame to process. + direction: Frame processing direction. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, (InterruptionFrame, CancelFrame)): + # Push frame first otherwise our emitted transcription update frame + # might get cleaned up. + await self.push_frame(frame, direction) + # Emit accumulated thought with interruptions + if self._thought_active: + await self._emit_aggregated_thought() + elif isinstance(frame, LLMThoughtStartFrame): + # Start a new thought + self._thought_active = True + self._thought_start_time = time_now_iso8601() + self._current_thought_parts = [] + # Push frame. + await self.push_frame(frame, direction) + elif isinstance(frame, LLMThoughtTextFrame): + # Aggregate thought text if we have an active thought + if self._thought_active: + self._current_thought_parts.append( + TextPartForConcatenation( + frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces + ) + ) + # Push frame. + await self.push_frame(frame, direction) + elif isinstance(frame, LLMThoughtEndFrame): + # Emit accumulated thought when thought ends + if self._thought_active: + await self._emit_aggregated_thought() + # Push frame. + await self.push_frame(frame, direction) + elif isinstance(frame, EndFrame): + # Emit accumulated thought at pipeline end if still active + if self._thought_active: + await self._emit_aggregated_thought() + # Push frame. + await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) + + class TranscriptProcessor: """Factory for creating and managing transcript processors. - Provides unified access to user and assistant transcript processors + Provides unified access to user, assistant, and thought transcript processors with shared event handling. Example:: @@ -219,9 +326,10 @@ class TranscriptProcessor: transcript.user(), # User transcripts context_aggregator.user(), llm, + transcript.thought(), # Thought transcripts tts, transport.output(), - transcript.assistant_tts(), # Assistant transcripts + transcript.assistant(), # Assistant transcripts context_aggregator.assistant(), ] ) @@ -235,6 +343,7 @@ class TranscriptProcessor: """Initialize factory.""" self._user_processor = None self._assistant_processor = None + self._thought_processor = None self._event_handlers = {} def user(self, **kwargs) -> UserTranscriptProcessor: @@ -277,6 +386,26 @@ class TranscriptProcessor: return self._assistant_processor + def thought(self, **kwargs) -> ThoughtTranscriptProcessor: + """Get the thought transcript processor. + + Args: + **kwargs: Arguments specific to ThoughtTranscriptProcessor. + + Returns: + The thought transcript processor instance. + """ + if self._thought_processor is None: + self._thought_processor = ThoughtTranscriptProcessor(**kwargs) + # Apply any registered event handlers + for event_name, handler in self._event_handlers.items(): + + @self._thought_processor.event_handler(event_name) + async def thought_handler(processor, frame): + return await handler(processor, frame) + + return self._thought_processor + def event_handler(self, event_name: str): """Register event handler for both processors. @@ -303,6 +432,12 @@ class TranscriptProcessor: async def assistant_handler(processor, frame): return await handler(processor, frame) + if self._thought_processor: + + @self._thought_processor.event_handler(event_name) + async def thought_handler(processor, frame): + return await handler(processor, frame) + return handler return decorator diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index a5c67e90e..662975056 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -17,7 +17,7 @@ import io import json import re from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union import httpx from loguru import logger @@ -40,6 +40,9 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, LLMUpdateSettingsFrame, UserImageRawFrame, ) @@ -110,6 +113,24 @@ class AnthropicLLMService(LLMService): # Overriding the default adapter to use the Anthropic one. adapter_class = AnthropicLLMAdapter + class ThinkingConfig(BaseModel): + """Configuration for extended thinking. + + Parameters: + type: Type of thinking mode (currently only "enabled" or "disabled"). + budget_tokens: Maximum number of tokens for thinking. + With today's models, the minimum is 1024. + Only allowed if type is "enabled". + """ + + # Why `| str` here? To not break compatibility in case Anthropic adds + # more types in the future. + type: Literal["enabled", "disabled"] | str + + # Why not enforce minimnum of 1024 here? To not break compatibility in + # case Anthropic changes this requirement in the future. + budget_tokens: int + class InputParams(BaseModel): """Input parameters for Anthropic model inference. @@ -124,6 +145,10 @@ class AnthropicLLMService(LLMService): temperature: Sampling temperature between 0.0 and 1.0. top_k: Top-k sampling parameter. top_p: Top-p sampling parameter between 0.0 and 1.0. + thinking: Extended thinking configuration. + Enabling extended thinking causes the model to spend more time "thinking" before responding. + It also causes this service to emit LLMThinking*Frames during response generation. + Extended thinking is disabled by default. extra: Additional parameters to pass to the API. """ @@ -133,6 +158,9 @@ class AnthropicLLMService(LLMService): temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0) top_k: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0) top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0) + thinking: Optional["AnthropicLLMService.ThinkingConfig"] = Field( + default_factory=lambda: NOT_GIVEN + ) extra: Optional[Dict[str, Any]] = Field(default_factory=dict) def model_post_init(self, __context): @@ -191,6 +219,7 @@ class AnthropicLLMService(LLMService): "temperature": params.temperature, "top_k": params.top_k, "top_p": params.top_p, + "thinking": params.thinking, "extra": params.extra if isinstance(params.extra, dict) else {}, } @@ -354,12 +383,21 @@ class AnthropicLLMService(LLMService): "top_p": self._settings["top_p"], } + # Add thinking parameter if set + if self._settings["thinking"]: + params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True) + # Messages, system, tools params.update(params_from_context) params.update(self._settings["extra"]) - response = await self._create_message_stream(self._client.messages.create, params) + # "Interleaved thinking" needed to allow thinking between sequences + # of function calls, when extended thinking is enabled. + # Note that this requires us to use `client.beta`, below. + params.update({"betas": ["interleaved-thinking-2025-05-14"]}) + + response = await self._create_message_stream(self._client.beta.messages.create, params) await self.stop_ttfb_metrics() @@ -380,10 +418,25 @@ class AnthropicLLMService(LLMService): completion_tokens_estimate += self._estimate_tokens( event.delta.partial_json ) + elif hasattr(event.delta, "thinking"): + await self.push_frame(LLMThoughtTextFrame(text=event.delta.thinking)) + elif hasattr(event.delta, "signature"): + await self.push_frame( + LLMThoughtEndFrame( + thought_metadata={"signature": event.delta.signature} + ) + ) elif event.type == "content_block_start": if event.content_block.type == "tool_use": tool_use_block = event.content_block json_accumulator = "" + elif event.content_block.type == "thinking": + await self.push_frame( + LLMThoughtStartFrame( + append_to_context=True, + llm=self.get_llm_adapter().id_for_llm_specific_messages, + ) + ) elif ( event.type == "message_delta" and hasattr(event.delta, "stop_reason") diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 840b473b2..114df5f28 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -16,7 +16,7 @@ import json import os import uuid from dataclasses import dataclass -from typing import Any, AsyncIterator, Dict, List, Optional +from typing import Any, AsyncIterator, Dict, List, Literal, Optional from loguru import logger from PIL import Image @@ -34,6 +34,9 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, LLMUpdateSettingsFrame, OutputImageRawFrame, UserImageRawFrame, @@ -665,6 +668,34 @@ class GoogleLLMService(LLMService): # Overriding the default adapter to use the Gemini one. adapter_class = GeminiLLMAdapter + class ThinkingConfig(BaseModel): + """Configuration for controlling the model's internal "thinking" process used before generating a response. + + Gemini 2.5 and 3 series models have this thinking process. + + Parameters: + thinking_level: Thinking level for Gemini 3 Pro. Can be "low" or "high". + If not provided, Gemini 3 Pro defaults to "high". + Note: Gemini 2.5 series should use thinking_budget instead. + thinking_budget: Token budget for thinking, for Gemini 2.5 series. + -1 for dynamic thinking (model decides), 0 to disable thinking, + or a specific token count (e.g., 128-32768 for 2.5 Pro). + If not provided, most models today default to dynamic thinking. + See https://ai.google.dev/gemini-api/docs/thinking#set-budget + for default values and allowed ranges. + Note: Gemini 3 Pro should use thinking_level instead. + include_thoughts: Whether to include thought summaries in the response. + Today's models default to not including thoughts (False). + """ + + thinking_budget: Optional[int] = Field(default=None) + + # Why `| str` here? To not break compatibility in case Google adds more + # levels in the future. + thinking_level: Optional[Literal["low", "high"] | str] = Field(default=None) + + include_thoughts: Optional[bool] = Field(default=None) + class InputParams(BaseModel): """Input parameters for Google AI models. @@ -673,6 +704,12 @@ class GoogleLLMService(LLMService): temperature: Sampling temperature between 0.0 and 2.0. top_k: Top-k sampling parameter. top_p: Top-p sampling parameter between 0.0 and 1.0. + thinking: Thinking configuration with thinking_budget, thinking_level, and include_thoughts. + Used to control the model's internal "thinking" process used before generating a response. + Gemini 2.5 series models use thinking_budget; Gemini 3 models use thinking_level. + If this is not provided, Pipecat disables thinking for all + models where that's possible (the 2.5 series, except 2.5 Pro), + to reduce latency. extra: Additional parameters as a dictionary. """ @@ -680,6 +717,7 @@ class GoogleLLMService(LLMService): temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) top_k: Optional[int] = Field(default=None, ge=0) top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0) + thinking: Optional["GoogleLLMService.ThinkingConfig"] = Field(default=None) extra: Optional[Dict[str, Any]] = Field(default_factory=dict) def __init__( @@ -720,6 +758,7 @@ class GoogleLLMService(LLMService): "temperature": params.temperature, "top_k": params.top_k, "top_p": params.top_p, + "thinking": params.thinking, "extra": params.extra if isinstance(params.extra, dict) else {}, } self._tools = tools @@ -830,6 +869,12 @@ class GoogleLLMService(LLMService): if v is not None } + # Add thinking parameters if configured + if self._settings["thinking"]: + generation_params["thinking_config"] = self._settings["thinking"].model_dump( + exclude_unset=True + ) + if self._settings["extra"]: generation_params.update(self._settings["extra"]) @@ -918,9 +963,17 @@ class GoogleLLMService(LLMService): for candidate in chunk.candidates: if candidate.content and candidate.content.parts: for part in candidate.content.parts: - if not part.thought and part.text: - search_result += part.text - await self.push_frame(LLMTextFrame(part.text)) + if part.text: + if part.thought: + # Gemini emits fully-formed thoughts rather + # than chunks so bracket each thought in + # start/end + await self.push_frame(LLMThoughtStartFrame()) + await self.push_frame(LLMThoughtTextFrame(part.text)) + await self.push_frame(LLMThoughtEndFrame()) + else: + search_result += part.text + await self.push_frame(LLMTextFrame(part.text)) elif part.function_call: function_call = part.function_call id = function_call.id or str(uuid.uuid4()) @@ -931,6 +984,13 @@ class GoogleLLMService(LLMService): tool_call_id=id, function_name=function_call.name, arguments=function_call.args or {}, + llm_specific_extra={ + self.get_llm_adapter().id_for_llm_specific_messages: { + "thought_signature": part.thought_signature + } + } + if part.thought_signature + else None, ) ) elif part.inline_data and part.inline_data.data: diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 6e5355263..71e31d9b6 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -127,6 +127,9 @@ class FunctionCallRunnerItem: tool_call_id: A unique identifier for the function call. arguments: The arguments for the function. context: The LLM context. + llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: + {"google": {"thought_signature": ...}} + Uses the LLM adapter's ID for LLM-specific messages as the key. run_llm: Optional flag to control LLM execution after function call. """ @@ -135,6 +138,7 @@ class FunctionCallRunnerItem: tool_call_id: str arguments: Mapping[str, Any] context: OpenAILLMContext | LLMContext + llm_specific_extra: Optional[Dict[str, Any]] = None run_llm: Optional[bool] = None @@ -456,6 +460,7 @@ class LLMService(AIService): tool_call_id=function_call.tool_call_id, arguments=function_call.arguments, context=function_call.context, + llm_specific_extra=function_call.llm_specific_extra, ) ) @@ -580,6 +585,7 @@ class LLMService(AIService): function_name=runner_item.function_name, tool_call_id=runner_item.tool_call_id, arguments=runner_item.arguments, + llm_specific_extra=runner_item.llm_specific_extra, cancel_on_interruption=item.cancel_on_interruption, ) From 0cdf0c4504545e2cf7d97f8bb79e053fbfa778eb Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 4 Dec 2025 09:50:45 -0500 Subject: [PATCH 02/21] =?UTF-8?q?Bump=20Google=20GenAI=20library=20version?= =?UTF-8?q?=20to=20at=20least=201.51.0,=20as=20that's=20the=20version=20wh?= =?UTF-8?q?ere=20`thinking=5Flevel`=E2=80=94required=20for=20controlling?= =?UTF-8?q?=20Gemini=203=20Pro=20thinking=E2=80=94is=20introduced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- uv.lock | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bb72dba61..5cff2fa14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ fal = [ "fal-client~=0.5.9" ] fireworks = [] fish = [ "ormsgpack~=1.7.0", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] -google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.41.0,<2", "pipecat-ai[websockets-base]" ] +google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.51.0,<2", "pipecat-ai[websockets-base]" ] gradium = [ "pipecat-ai[websockets-base]" ] grok = [] groq = [ "groq~=0.23.0" ] diff --git a/uv.lock b/uv.lock index cdb084284..cebf3ab96 100644 --- a/uv.lock +++ b/uv.lock @@ -1853,6 +1853,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/a4/7319a2a8add4cc352be9e3efeff5e2aacee917c85ca2fa1647e29089983c/google_auth-2.41.1-py2.py3-none-any.whl", hash = "sha256:754843be95575b9a19c604a848a41be03f7f2afd8c019f716dc1f51ee41c639d", size = 221302, upload-time = "2025-09-30T22:51:24.212Z" }, ] +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + [[package]] name = "google-cloud-speech" version = "2.33.0" @@ -1920,11 +1925,11 @@ wheels = [ [[package]] name = "google-genai" -version = "1.41.0" +version = "1.53.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "google-auth" }, + { name = "google-auth", extra = ["requests"] }, { name = "httpx" }, { name = "pydantic" }, { name = "requests" }, @@ -1932,9 +1937,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/8b/ee20bcf707769b3b0e1106c3b5c811507736af7e8a60f29a70af1750ba19/google_genai-1.41.0.tar.gz", hash = "sha256:134f861bb0ace4e34af0501ecb75ceee15f7662fd8120698cd185e8cb39f2800", size = 245812, upload-time = "2025-10-02T22:30:29.699Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/b3/36fbfde2e21e6d3bc67780b61da33632f495ab1be08076cf0a16af74098f/google_genai-1.53.0.tar.gz", hash = "sha256:938a26d22f3fd32c6eeeb4276ef204ef82884e63af9842ce3eac05ceb39cbd8d", size = 260102, upload-time = "2025-12-03T17:21:23.233Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/14/e5e8fbca8863fee718208566c4e927b8e9f45fd46ec5cf89e24759da545b/google_genai-1.41.0-py3-none-any.whl", hash = "sha256:111a3ee64c1a0927d3879faddb368234594432479a40c311e5fe4db338ca8778", size = 245931, upload-time = "2025-10-02T22:30:27.885Z" }, + { url = "https://files.pythonhosted.org/packages/40/f2/97fefdd1ad1f3428321bac819ae7a83ccc59f6439616054736b7819fa56c/google_genai-1.53.0-py3-none-any.whl", hash = "sha256:65a3f99e5c03c372d872cda7419f5940e723374bb12a2f3ffd5e3e56e8eb2094", size = 262015, upload-time = "2025-12-03T17:21:21.934Z" }, ] [[package]] @@ -4695,7 +4700,7 @@ requires-dist = [ { name = "faster-whisper", marker = "extra == 'whisper'", specifier = "~=1.1.1" }, { name = "google-cloud-speech", marker = "extra == 'google'", specifier = ">=2.33.0,<3" }, { name = "google-cloud-texttospeech", marker = "extra == 'google'", specifier = ">=2.31.0,<3" }, - { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.41.0,<2" }, + { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.51.0,<2" }, { name = "groq", marker = "extra == 'groq'", specifier = "~=0.23.0" }, { name = "hume", marker = "extra == 'hume'", specifier = ">=0.11.2" }, { name = "langchain", marker = "extra == 'langchain'", specifier = "~=0.3.20" }, From c8c6f424cd6a02f57e72b9f984db62c958447da4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 4 Dec 2025 14:29:08 -0500 Subject: [PATCH 03/21] Add support for Gemini 3 Pro non-function-call-related thought signatures --- .../foundational/49-thinking-functions.py | 2 + examples/foundational/49-thinking.py | 2 + .../adapters/services/gemini_adapter.py | 61 ++++++++++++++++++- src/pipecat/frames/frames.py | 27 +++++++- .../aggregators/llm_response_universal.py | 14 +++++ src/pipecat/services/google/llm.py | 12 ++++ 6 files changed, 115 insertions(+), 3 deletions(-) diff --git a/examples/foundational/49-thinking-functions.py b/examples/foundational/49-thinking-functions.py index 2b96e304d..ffc93dd04 100644 --- a/examples/foundational/49-thinking-functions.py +++ b/examples/foundational/49-thinking-functions.py @@ -108,8 +108,10 @@ async def run_bot( elif llm_provider == LLM_GOOGLE: llm = GoogleLLMService( api_key=os.getenv("GOOGLE_API_KEY"), + # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower params=GoogleLLMService.InputParams( thinking=GoogleLLMService.ThinkingConfig( + # thinking_level="low", # Use this field instead of thinking_budget for Gemini 3 Pro. Defaults to "high". thinking_budget=-1, # Dynamic thinking include_thoughts=True, ) diff --git a/examples/foundational/49-thinking.py b/examples/foundational/49-thinking.py index 512163be4..8b9421e1c 100644 --- a/examples/foundational/49-thinking.py +++ b/examples/foundational/49-thinking.py @@ -87,8 +87,10 @@ async def run_bot( elif llm_provider == LLM_GOOGLE: llm = GoogleLLMService( api_key=os.getenv("GOOGLE_API_KEY"), + # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower params=GoogleLLMService.InputParams( thinking=GoogleLLMService.ThinkingConfig( + # thinking_level="low", # Use this field instead of thinking_budget for Gemini 3 Pro. Defaults to "high". thinking_budget=-1, # Dynamic thinking include_thoughts=True, ) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index fd91b818f..a81131fce 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -210,6 +210,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): system_instruction = None messages = [] tool_call_id_to_name_mapping = {} + non_fn_thought_signatures = [] # Process each message, preserving Google-formatted messages and converting others for message in universal_context_messages: @@ -234,6 +235,17 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): ) continue + # If we found a standalone non-function-call-related thought + # signature (Gemini 3 Pro), store it to apply later to the + # corresponding assistant message + if ( + isinstance(result.content, dict) + and result.content.get("type") == "thought_signature" + and (thought_signature := result.content.get("signature")) + ): + non_fn_thought_signatures.append(thought_signature) + continue + # Each result is either a Content or a system instruction if result.content: messages.append(result.content) @@ -244,6 +256,10 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): if result.tool_call_id_to_name_mapping: tool_call_id_to_name_mapping.update(result.tool_call_id_to_name_mapping) + # Apply non-function-call-related thought signatures to the appropriate + # messages + self._apply_non_function_thought_signatures_to_messages(non_fn_thought_signatures, messages) + # Check if we only have function-related messages (no regular text) has_regular_messages = any( len(msg.parts) == 1 @@ -434,7 +450,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Args: thought_signature: The thought signature bytes to apply. tool_call_id: ID of the tool call message to find and modify. - messages: List of Content messages to search through. + messages: List of messages to search through. """ # Search backwards through messages to find the matching function call for message in reversed(messages): @@ -454,3 +470,46 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): continue # Break outer loop if inner loop broke (found match) break + + def _apply_non_function_thought_signatures_to_messages( + self, thought_signatures: List[bytes], messages: List[Content] + ) -> None: + """Apply non-function-call-related thought signatures to the last part of each non-function-call assistant message. + + Gemini 3 Pro outputs a thought signature at the end of each assistant + response. + + Args: + thought_signatures: The list of thought signature bytes to apply. + messages: List of messages to search through. + """ + if not thought_signatures: + return + + # Find all assistant (model) messages that aren't function calls + non_fn_assistant_messages = [] + for message in messages: + if not isinstance(message, Content) or not message.parts: + continue + # Check if this is a model message without function calls + if message.role == "model": + has_function_call = any( + hasattr(part, "function_call") and part.function_call for part in message.parts + ) + if not has_function_call: + non_fn_assistant_messages.append(message) + + # Warn if counts don't match + if len(thought_signatures) != len(non_fn_assistant_messages): + logger.warning( + f"Thought signature count ({len(thought_signatures)}) doesn't match " + f"non-function-call assistant message count ({len(non_fn_assistant_messages)})" + ) + + # Apply thought signatures to the corresponding assistant messages + # Match them in order (oldest to newest) + for i, thought_signature in enumerate(thought_signatures): + if i < len(non_fn_assistant_messages): + message = non_fn_assistant_messages[i] + if message.parts: + message.parts[-1].thought_signature = thought_signature diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 2c3b802ec..d538ce6b4 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -594,7 +594,8 @@ class LLMThoughtStartFrame(ControlFrame): If it is appended, the `llm` field is required, since it will be appended as an `LLMSpecificMessage`. llm: Optional identifier of the LLM provider for LLM-specific handling. - Only required if `append_to_context` is True. + Only required if `append_to_context` is True, as the thought is + appended to context as an `LLMSpecificMessage`. """ append_to_context: bool = False @@ -642,7 +643,7 @@ class LLMThoughtEndFrame(ControlFrame): Parameters: thought_metadata: Optional metadata associated with the thought, - e.g. thought signature. + e.g. an Anthropic thought signature. """ thought_metadata: Optional[Dict[str, Any]] = None @@ -652,6 +653,28 @@ class LLMThoughtEndFrame(ControlFrame): return f"{self.name}(pts: {pts}, metadata: {self.thought_metadata})" +@dataclass +class LLMThoughtSignatureFrame(DataFrame): + """Frame containing a standalone LLM thought signature (as opposed to a thought signature associated with a thought). + + This is useful for Gemini 3 Pro, which can output a signature at the end of + a response. + + Parameters: + llm: Identifier of the LLM provider for LLM-specific handling. + Needed because the thought signature is appended to context as an + `LLMSpecificMessage`. + signature: The thought signature data. + """ + + llm: str + signature: Any + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, signature: {self.signature})" + + @dataclass class LLMMessagesFrame(DataFrame): """Frame containing LLM messages for chat completion. diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 04f9c4ae9..4a074da93 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -48,6 +48,7 @@ from pipecat.frames.frames import ( LLMSetToolChoiceFrame, LLMSetToolsFrame, LLMThoughtEndFrame, + LLMThoughtSignatureFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, SpeechControlParamsFrame, @@ -643,6 +644,8 @@ class LLMAssistantAggregator(LLMContextAggregator): await self._handle_thought_text(frame) elif isinstance(frame, LLMThoughtEndFrame): await self._handle_thought_end(frame) + elif isinstance(frame, LLMThoughtSignatureFrame): + await self._handle_standalone_thought_signature(frame) elif isinstance(frame, LLMRunFrame): await self._handle_llm_run(frame) elif isinstance(frame, LLMMessagesAppendFrame): @@ -907,6 +910,17 @@ class LLMAssistantAggregator(LLMContextAggregator): ) ) + async def _handle_standalone_thought_signature(self, frame: LLMThoughtSignatureFrame): + self._context.add_message( + LLMSpecificMessage( + llm=frame.llm, + message={ + "type": "thought_signature", + "signature": frame.signature, + }, + ) + ) + def _context_updated_task_finished(self, task: asyncio.Task): self._context_updated_tasks.discard(task) diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 114df5f28..408918287 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -35,6 +35,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMTextFrame, LLMThoughtEndFrame, + LLMThoughtSignatureFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, LLMUpdateSettingsFrame, @@ -1000,6 +1001,17 @@ class GoogleLLMService(LLMService): ) await self.push_frame(frame) + # With Gemini 3 Pro, thought signatures can be + # included in any kind of part, not just function + # calls. It will come in the last part of a response. + if part.thought_signature and not part.function_call: + await self.push_frame( + LLMThoughtSignatureFrame( + llm=self.get_llm_adapter().id_for_llm_specific_messages, + signature=part.thought_signature, + ) + ) + if ( candidate.grounding_metadata and candidate.grounding_metadata.grounding_chunks From 15f5583fd220b5107d3d838e09c8311052386b94 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 4 Dec 2025 14:47:31 -0500 Subject: [PATCH 04/21] Simplify, at the expense of a bit of not-yet-needed flexibility: rather than associating a loose `thought_metadata` with each thought, use a `signature`. Thought signatures are the only "thought metadata" we use today. --- src/pipecat/adapters/services/anthropic_adapter.py | 3 +-- src/pipecat/frames/frames.py | 9 +++++---- .../processors/aggregators/llm_response_universal.py | 2 +- src/pipecat/services/anthropic/llm.py | 6 +----- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index e111b34df..da03ad013 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -185,8 +185,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): isinstance(message.message, dict) and message.message.get("type") == "thought" and (text := message.message.get("text")) - and isinstance(metadata := message.message.get("metadata"), dict) - and (signature := metadata.get("signature")) + and (signature := message.message.get("signature")) ): return { "role": "assistant", diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index d538ce6b4..9209d603a 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -642,15 +642,16 @@ class LLMThoughtEndFrame(ControlFrame): """Frame indicating the end of an LLM thought. Parameters: - thought_metadata: Optional metadata associated with the thought, - e.g. an Anthropic thought signature. + signature: Optional signature associated with the thought. + This is used by Anthropic, which includes a signature at the end of + each thought. """ - thought_metadata: Optional[Dict[str, Any]] = None + signature: Any = None def __str__(self): pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, metadata: {self.thought_metadata})" + return f"{self.name}(pts: {pts}, signature: {self.signature})" @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 4a074da93..76641f0e0 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -905,7 +905,7 @@ class LLMAssistantAggregator(LLMContextAggregator): message={ "type": "thought", "text": thought, - "metadata": frame.thought_metadata, + "signature": frame.signature, }, ) ) diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 662975056..348745e84 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -421,11 +421,7 @@ class AnthropicLLMService(LLMService): elif hasattr(event.delta, "thinking"): await self.push_frame(LLMThoughtTextFrame(text=event.delta.thinking)) elif hasattr(event.delta, "signature"): - await self.push_frame( - LLMThoughtEndFrame( - thought_metadata={"signature": event.delta.signature} - ) - ) + await self.push_frame(LLMThoughtEndFrame(signature=event.delta.signature)) elif event.type == "content_block_start": if event.content_block.type == "tool_use": tool_use_block = event.content_block From 747bd4f737877a7eca07d3fde8cbbdba505f13bc Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 4 Dec 2025 15:55:37 -0500 Subject: [PATCH 05/21] Tweak the prompt of the thinking + functions example to not confuse Gemini as much (Gemini found the original prompt a bit ambiguous, it seems) --- examples/foundational/49-thinking-functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/49-thinking-functions.py b/examples/foundational/49-thinking-functions.py index ffc93dd04..e363b87fc 100644 --- a/examples/foundational/49-thinking-functions.py +++ b/examples/foundational/49-thinking-functions.py @@ -169,7 +169,7 @@ async def run_bot( messages.append( { "role": "user", - "content": "Check the status of flight AA100 and book me a taxi 2 hours beforehand if the flight is delayed.", + "content": "Check the status of flight AA100 and, if it's delayed, book me a taxi 2 hours before its departure time.", } ) await task.queue_frames([LLMRunFrame()]) From 4ea51ff67cb9c18979335403270686b4ca856300 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 4 Dec 2025 17:02:40 -0500 Subject: [PATCH 06/21] Slight refactor of handling thought-signature-containing special context messages in the Gemini adapter --- .../adapters/services/anthropic_adapter.py | 2 +- .../adapters/services/gemini_adapter.py | 71 ++++++++++--------- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index da03ad013..8b5da3da5 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -198,7 +198,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): ], } - # Fallback to assumption that the message is already in Anthropic format + # Fall back to assuming that the message is already in Anthropic format return copy.deepcopy(message.message) def _from_standard_message(self, message: LLMStandardMessage) -> MessageParam: diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index a81131fce..ec0afa885 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -167,7 +167,6 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): class MessageConversionResult: """Result of converting a single universal context message to Google format. - # TODO: content could be other things, like {"tool_call_extra": ...}, for example. All bets are off when it's LLMSpecificMessage. Either content (a Google Content object) or a system instruction string is guaranteed to be set. @@ -212,9 +211,44 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): tool_call_id_to_name_mapping = {} non_fn_thought_signatures = [] - # Process each message, preserving Google-formatted messages and converting others + # Process each message, converting to Google format as needed for message in universal_context_messages: - result = self._from_universal_context_message( + # We have a Google-specific message; this may either be a + # thought-signature-containing message that we need to handle in a + # special way, or a message already in Google format that we can + # use directly + if isinstance(message, LLMSpecificMessage): + # Special handling for function-call-related thought signature + # messages + if ( + isinstance(message.message, dict) + and message.message.get("type") == "tool_call_extra" + and isinstance(data := message.message.get("data"), dict) + and (thought_signature := data.get("thought_signature")) + ): + self._apply_function_call_thought_signature_to_messages( + thought_signature, message.message.get("tool_call_id"), messages + ) + continue + + # Special handling for non-function-call-related thought + # signature messages (Gemini 3 Pro) + if ( + isinstance(message.message, dict) + and message.message.get("type") == "thought_signature" + and (thought_signature := message.message.get("signature")) + ): + non_fn_thought_signatures.append(thought_signature) + continue + + # Fall back to assuming that the message is already in Google + # format + messages.append(message.message) + continue + + # We have a standard universal context message; convert it to + # Google format + result = self._from_standard_message( message, params=self.MessageConversionParams( already_have_system_instruction=bool(system_instruction), @@ -222,30 +256,6 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): ), ) - # If we found a function-call-related thought_signature, modify the - # corresponding function call message to include it - if ( - isinstance(result.content, dict) - and result.content.get("type") == "tool_call_extra" - and isinstance(data := result.content.get("data"), dict) - and (thought_signature := data.get("thought_signature")) - ): - self._apply_function_call_thought_signature_to_messages( - thought_signature, result.content.get("tool_call_id"), messages - ) - continue - - # If we found a standalone non-function-call-related thought - # signature (Gemini 3 Pro), store it to apply later to the - # corresponding assistant message - if ( - isinstance(result.content, dict) - and result.content.get("type") == "thought_signature" - and (thought_signature := result.content.get("signature")) - ): - non_fn_thought_signatures.append(thought_signature) - continue - # Each result is either a Content or a system instruction if result.content: messages.append(result.content) @@ -278,13 +288,6 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): return self.ConvertedMessages(messages=messages, system_instruction=system_instruction) - def _from_universal_context_message( - self, message: LLMContextMessage, *, params: MessageConversionParams - ) -> MessageConversionResult: - if isinstance(message, LLMSpecificMessage): - return self.MessageConversionResult(content=message.message) - return self._from_standard_message(message, params=params) - def _from_standard_message( self, message: LLMStandardMessage, *, params: MessageConversionParams ) -> MessageConversionResult: From 49f1f7d6a280aab5c30755d947a80ae33e030be2 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 5 Dec 2025 10:02:21 -0500 Subject: [PATCH 07/21] Added CHANGELOG entry describing new thinking-related functionality --- changelog/3175.added.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 changelog/3175.added.md diff --git a/changelog/3175.added.md b/changelog/3175.added.md new file mode 100644 index 000000000..49e109094 --- /dev/null +++ b/changelog/3175.added.md @@ -0,0 +1,39 @@ +- Added additional functionality related to "thinking", for Google and Anthropic + LLMs. + + 1. New typed parameters for Google and Anthropic LLMs that control the + models' thinking behavior (like how much thinking to do, and whether to + output thoughts or thought summaries): + - `AnthropicLLMService.ThinkingConfig` + - `GoogleLLMService.ThinkingConfig` + 2. New frames for representing thoughts output by LLMs: + - `LLMThoughtStartFrame` + - `LLMThoughtTextFrame` + - `LLMThoughtEndFrame` + 3. A generic mechanism for associating extra LLM-specific data with a function + call in context, used specifically to support Google's function-call-related + "thought signatures", which are necessary to ensure thinking continuity + between function calls in a chain (where the model thinks, makes a function + call, thinks some more, etc.). See: + - `FunctionCallInProgressFrame.llm_specific_extra` + - `LLMAssistantAggregator` handling of the above field + - `GeminiLLMAdapter` handling of`"tool_call_extra"` context messages + 4. A generic mechanism for recording LLM thoughts to context, used + specifically to support Anthropic, whose thought signatures are expected to + appear alongside the text of the thoughts within assistant context + messages. See: + - `LLMThoughtEndFrame.signature` + - `LLMAssistantAggregator` handling of the above field + - `AnthropicLLMAdapter` handling of `"thought"` context messages + 5. A generic mechanism for recording standalone thought signatures to context, + used specifically to support Gemini 3 Pro, which may return + non-function-call-related thought signatures at the end of + non-function-call assistant responses. See: + - `LLMThoughtSignatureFrame` + - `LLMAssistantAggregator` handling of the above frame + - `GeminiLLMAdapter` handling of `"thought_signature"` context messages + 6. An expansion of `TranscriptProcessor` to process LLM thoughts in addition + to user and assistant utterances. See: + - `TranscriptProcessor.thought()` + - `ThoughtTranscriptionMessage`, which may now also be emitted with the + `"on_transcript_update"` event From 44aa11737b3d6df2ce3c1ecdf8d341a0c62a1af7 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 5 Dec 2025 10:16:51 -0500 Subject: [PATCH 08/21] Minor docstring update for accuracy --- src/pipecat/adapters/services/anthropic_adapter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index 8b5da3da5..4cecaf810 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -171,10 +171,12 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): def _from_anthropic_specific_message(self, message: LLMSpecificMessage) -> MessageParam: """Convert LLMSpecificMessage to Anthropic format. - Assumes that we already know the message is intended for Anthropic. + Anthropic-specific messages may either be special thought messages that + need to be handled in a special way, or messages already in Anthropic + format. Args: - message: Message in LLMSpecificMessage format. + message: Anthropic-specific message. """ # Handle special case of thought messages. # These can be converted to standalone "assistant" messages; later From ef703e9d160746af2f81e79141d888f75eb09153 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 09:59:32 -0500 Subject: [PATCH 09/21] Get rid of `ThoughtTranscriptProcessor`, moving its logic into `AssistantTranscriptProcessor` instead --- .../foundational/49-thinking-functions.py | 3 +- examples/foundational/49-thinking.py | 3 +- .../processors/transcript_processor.py | 206 +++++------------- 3 files changed, 56 insertions(+), 156 deletions(-) diff --git a/examples/foundational/49-thinking-functions.py b/examples/foundational/49-thinking-functions.py index e363b87fc..d24adbf3e 100644 --- a/examples/foundational/49-thinking-functions.py +++ b/examples/foundational/49-thinking-functions.py @@ -144,10 +144,9 @@ async def run_bot( transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM - transcript.thought(), # Thought transcripts tts, # TTS transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts + transcript.assistant(), # Assistant transcripts (including thoughts) context_aggregator.assistant(), # Assistant spoken responses ] ) diff --git a/examples/foundational/49-thinking.py b/examples/foundational/49-thinking.py index 8b9421e1c..756134076 100644 --- a/examples/foundational/49-thinking.py +++ b/examples/foundational/49-thinking.py @@ -118,10 +118,9 @@ async def run_bot( transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM - transcript.thought(), # Thought transcripts tts, # TTS transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts + transcript.assistant(), # Assistant transcripts (including thoughts) context_aggregator.assistant(), # Assistant spoken responses ] ) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 0dae1ad6a..150d6c5c8 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -85,14 +85,20 @@ class UserTranscriptProcessor(BaseTranscriptProcessor): class AssistantTranscriptProcessor(BaseTranscriptProcessor): - """Processes assistant TTS text frames into timestamped conversation messages. + """Processes assistant TTS text frames and LLM thought frames into timestamped messages. - This processor aggregates TTS text frames into complete utterances and emits them as - transcript messages. Utterances are completed when: + This processor aggregates both TTS text frames and LLM thought frames into + complete utterances and thoughts, emitting them as transcript messages. + An assistant utterance is completed when: - The bot stops speaking (BotStoppedSpeakingFrame) - The bot is interrupted (InterruptionFrame) - - The pipeline ends (EndFrame) + - The pipeline ends (EndFrame, CancelFrame) + + A thought is completed when: + - The thought ends (LLMThoughtEndFrame) + - The bot is interrupted (InterruptionFrame) + - The pipeline ends (EndFrame, CancelFrame) """ def __init__(self, **kwargs): @@ -102,131 +108,36 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): **kwargs: Additional arguments passed to parent class. """ super().__init__(**kwargs) - self._current_text_parts: List[TextPartForConcatenation] = [] - self._aggregation_start_time: Optional[str] = None - async def _emit_aggregated_text(self): + self._current_assistant_text_parts: List[TextPartForConcatenation] = [] + self._assistant_text_start_time: Optional[str] = None + + self._current_thought_parts: List[TextPartForConcatenation] = [] + self._thought_start_time: Optional[str] = None + self._thought_active = False + + async def _emit_aggregated_assistant_text(self): """Aggregates and emits text fragments as a transcript message. - This method uses a heuristic to automatically detect whether text fragments - contain embedded spacing (spaces at the beginning or end of fragments) or not, - and applies the appropriate joining strategy. It handles fragments from different - TTS services with different formatting patterns. - - Examples: - Fragments with embedded spacing (concatenated):: - - TTSTextFrame: ["Hello"] - TTSTextFrame: [" there"] # Leading space - TTSTextFrame: ["!"] - TTSTextFrame: [" How"] # Leading space - TTSTextFrame: ["'s"] - TTSTextFrame: [" it"] # Leading space - - Result: "Hello there! How's it" - - Fragments with trailing spaces (concatenated):: - - TTSTextFrame: ["Hel"] - TTSTextFrame: ["lo "] # Trailing space - TTSTextFrame: ["to "] # Trailing space - TTSTextFrame: ["you"] - - Result: "Hello to you" - - Word-by-word fragments without spacing (joined with spaces):: - - TTSTextFrame: ["Hello"] - TTSTextFrame: ["there"] - TTSTextFrame: ["how"] - TTSTextFrame: ["are"] - TTSTextFrame: ["you"] - - Result: "Hello there how are you" + This method aggregates text fragments that may arrive in multiple + TTSTextFrame instances and emits them as a single TranscriptionMessage. """ - if self._current_text_parts and self._aggregation_start_time: - content = concatenate_aggregated_text(self._current_text_parts) + if self._current_assistant_text_parts and self._assistant_text_start_time: + content = concatenate_aggregated_text(self._current_assistant_text_parts) if content: logger.trace(f"Emitting aggregated assistant message: {content}") message = TranscriptionMessage( role="assistant", content=content, - timestamp=self._aggregation_start_time, + timestamp=self._assistant_text_start_time, ) await self._emit_update([message]) else: logger.trace("No content to emit after stripping whitespace") # Reset aggregation state - self._current_text_parts = [] - self._aggregation_start_time = None - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames into assistant conversation messages. - - Handles different frame types: - - - TTSTextFrame: Aggregates text for current utterance - - BotStoppedSpeakingFrame: Completes current utterance - - InterruptionFrame: Completes current utterance due to interruption - - EndFrame: Completes current utterance at pipeline end - - CancelFrame: Completes current utterance due to cancellation - - Args: - frame: Input frame to process. - direction: Frame processing direction. - """ - await super().process_frame(frame, direction) - - if isinstance(frame, (InterruptionFrame, CancelFrame)): - # Push frame first otherwise our emitted transcription update frame - # might get cleaned up. - await self.push_frame(frame, direction) - # Emit accumulated text with interruptions - await self._emit_aggregated_text() - elif isinstance(frame, TTSTextFrame): - # Start timestamp on first text part - if not self._aggregation_start_time: - self._aggregation_start_time = time_now_iso8601() - - self._current_text_parts.append( - TextPartForConcatenation( - frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces - ) - ) - - # Push frame. - await self.push_frame(frame, direction) - elif isinstance(frame, (BotStoppedSpeakingFrame, EndFrame)): - # Emit accumulated text when bot finishes speaking or pipeline ends. - await self._emit_aggregated_text() - # Push frame. - await self.push_frame(frame, direction) - else: - await self.push_frame(frame, direction) - - -class ThoughtTranscriptProcessor(BaseTranscriptProcessor): - """Processes LLM thought frames into timestamped thought messages. - - This processor aggregates LLM thought text frames into complete thoughts - and emits them as thought transcript messages. Thoughts are completed when: - - - A thought ends (LLMThoughtEndFrame) - - The bot is interrupted (InterruptionFrame) - - The pipeline ends (EndFrame) - """ - - def __init__(self, **kwargs): - """Initialize processor with thought aggregation state. - - Args: - **kwargs: Additional arguments passed to parent class. - """ - super().__init__(**kwargs) - self._current_thought_parts: List[TextPartForConcatenation] = [] - self._thought_start_time: Optional[str] = None - self._thought_active = False + self._current_assistant_text_parts = [] + self._assistant_text_start_time = None async def _emit_aggregated_thought(self): """Aggregates and emits thought text fragments as a thought transcript message. @@ -252,16 +163,18 @@ class ThoughtTranscriptProcessor(BaseTranscriptProcessor): self._thought_active = False async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames into thought transcript messages. + """Process frames into assistant conversation messages and thought messages. Handles different frame types: + - TTSTextFrame: Aggregates text for current utterance - LLMThoughtStartFrame: Begins aggregating a new thought - LLMThoughtTextFrame: Aggregates text for current thought - LLMThoughtEndFrame: Completes current thought - - InterruptionFrame: Completes current thought due to interruption - - EndFrame: Completes current thought at pipeline end - - CancelFrame: Completes current thought due to cancellation + - BotStoppedSpeakingFrame: Completes current utterance + - InterruptionFrame: Completes current utterance and thought due to interruption + - EndFrame: Completes current utterance and thought at pipeline end + - CancelFrame: Completes current utterance and thought due to cancellation Args: frame: Input frame to process. @@ -273,7 +186,8 @@ class ThoughtTranscriptProcessor(BaseTranscriptProcessor): # Push frame first otherwise our emitted transcription update frame # might get cleaned up. await self.push_frame(frame, direction) - # Emit accumulated thought with interruptions + # Emit accumulated text and thought with interruptions + await self._emit_aggregated_assistant_text() if self._thought_active: await self._emit_aggregated_thought() elif isinstance(frame, LLMThoughtStartFrame): @@ -299,9 +213,24 @@ class ThoughtTranscriptProcessor(BaseTranscriptProcessor): await self._emit_aggregated_thought() # Push frame. await self.push_frame(frame, direction) - elif isinstance(frame, EndFrame): + elif isinstance(frame, TTSTextFrame): + # Start timestamp on first text part + if not self._assistant_text_start_time: + self._assistant_text_start_time = time_now_iso8601() + + self._current_assistant_text_parts.append( + TextPartForConcatenation( + frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces + ) + ) + + # Push frame. + await self.push_frame(frame, direction) + elif isinstance(frame, (BotStoppedSpeakingFrame, EndFrame)): + # Emit accumulated text when bot finishes speaking or pipeline ends. + await self._emit_aggregated_assistant_text() # Emit accumulated thought at pipeline end if still active - if self._thought_active: + if isinstance(frame, EndFrame) and self._thought_active: await self._emit_aggregated_thought() # Push frame. await self.push_frame(frame, direction) @@ -312,8 +241,9 @@ class ThoughtTranscriptProcessor(BaseTranscriptProcessor): class TranscriptProcessor: """Factory for creating and managing transcript processors. - Provides unified access to user, assistant, and thought transcript processors - with shared event handling. + Provides unified access to user and assistant transcript processors + with shared event handling. The assistant processor handles both TTS text + and LLM thought frames. Example:: @@ -326,10 +256,9 @@ class TranscriptProcessor: transcript.user(), # User transcripts context_aggregator.user(), llm, - transcript.thought(), # Thought transcripts tts, transport.output(), - transcript.assistant(), # Assistant transcripts + transcript.assistant(), # Assistant transcripts (including thoughts) context_aggregator.assistant(), ] ) @@ -343,7 +272,6 @@ class TranscriptProcessor: """Initialize factory.""" self._user_processor = None self._assistant_processor = None - self._thought_processor = None self._event_handlers = {} def user(self, **kwargs) -> UserTranscriptProcessor: @@ -386,26 +314,6 @@ class TranscriptProcessor: return self._assistant_processor - def thought(self, **kwargs) -> ThoughtTranscriptProcessor: - """Get the thought transcript processor. - - Args: - **kwargs: Arguments specific to ThoughtTranscriptProcessor. - - Returns: - The thought transcript processor instance. - """ - if self._thought_processor is None: - self._thought_processor = ThoughtTranscriptProcessor(**kwargs) - # Apply any registered event handlers - for event_name, handler in self._event_handlers.items(): - - @self._thought_processor.event_handler(event_name) - async def thought_handler(processor, frame): - return await handler(processor, frame) - - return self._thought_processor - def event_handler(self, event_name: str): """Register event handler for both processors. @@ -432,12 +340,6 @@ class TranscriptProcessor: async def assistant_handler(processor, frame): return await handler(processor, frame) - if self._thought_processor: - - @self._thought_processor.event_handler(event_name) - async def thought_handler(processor, frame): - return await handler(processor, frame) - return handler return decorator From 8ccc2cbf3168e7aea7d351409cb12591b008dfbc Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 10:14:31 -0500 Subject: [PATCH 10/21] Add unit tests for `ThoughtTranscriptProcessor` --- tests/test_transcript_processor.py | 310 +++++++++++++++++++++++++++++ 1 file changed, 310 insertions(+) diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index d86e42101..a5e0702a0 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -16,6 +16,10 @@ from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, InterruptionFrame, + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, + ThoughtTranscriptionMessage, TranscriptionFrame, TranscriptionMessage, TranscriptionUpdateFrame, @@ -485,3 +489,309 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): self.assertEqual(message.role, "assistant") # Should be properly joined without extra spaces self.assertEqual(message.content, "Hello there! How's it going?") + + +class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): + """Tests for thought transcription in AssistantTranscriptProcessor""" + + async def test_basic_thought_transcription(self): + """Test basic thought frame processing""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Create frames for a simple thought + frames_to_send = [ + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text="Let me think about this..."), + LLMThoughtEndFrame(), + ] + + expected_down_frames = [ + LLMThoughtStartFrame, + LLMThoughtTextFrame, + TranscriptionUpdateFrame, + LLMThoughtEndFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify update was received + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertIsInstance(message, ThoughtTranscriptionMessage) + self.assertEqual(message.content, "Let me think about this...") + self.assertIsNotNone(message.timestamp) + + async def test_thought_aggregation(self): + """Test that thought text frames are properly aggregated""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Create frames simulating chunked thought text + frames_to_send = [ + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text="The user "), + LLMThoughtTextFrame(text="is asking "), + LLMThoughtTextFrame(text="about electric "), + LLMThoughtTextFrame(text="cars."), + LLMThoughtEndFrame(), + ] + + expected_down_frames = [ + LLMThoughtStartFrame, + LLMThoughtTextFrame, + LLMThoughtTextFrame, + LLMThoughtTextFrame, + LLMThoughtTextFrame, + TranscriptionUpdateFrame, + LLMThoughtEndFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify aggregation + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertIsInstance(message, ThoughtTranscriptionMessage) + self.assertEqual(message.content, "The user is asking about electric cars.") + + async def test_thought_with_interruption(self): + """Test that thoughts are properly captured when interrupted""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text="I need to consider "), + LLMThoughtTextFrame(text="multiple factors"), + SleepFrame(), + InterruptionFrame(), # User interrupts + ] + + expected_down_frames = [ + LLMThoughtStartFrame, + LLMThoughtTextFrame, + LLMThoughtTextFrame, + InterruptionFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify thought was captured on interruption + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertIsInstance(message, ThoughtTranscriptionMessage) + self.assertEqual(message.content, "I need to consider multiple factors") + + async def test_thought_with_cancel(self): + """Test that thoughts are properly captured when cancelled""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text="Starting analysis"), + SleepFrame(), + CancelFrame(), + ] + + expected_down_frames = [ + LLMThoughtStartFrame, + LLMThoughtTextFrame, + CancelFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + send_end_frame=False, + ) + + # Verify thought was captured on cancellation + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertIsInstance(message, ThoughtTranscriptionMessage) + self.assertEqual(message.content, "Starting analysis") + + async def test_thought_with_end_frame(self): + """Test that thoughts are captured when pipeline ends normally""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text="Final thought"), + # Pipeline ends here; run_test will automatically send EndFrame + ] + + expected_down_frames = [ + LLMThoughtStartFrame, + LLMThoughtTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify thought was captured on EndFrame + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertIsInstance(message, ThoughtTranscriptionMessage) + self.assertEqual(message.content, "Final thought") + + async def test_multiple_thoughts(self): + """Test multiple separate thoughts in sequence""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + # First thought + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text="First consideration"), + LLMThoughtEndFrame(), + # Second thought + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text="Second consideration"), + LLMThoughtEndFrame(), + ] + + expected_down_frames = [ + LLMThoughtStartFrame, + LLMThoughtTextFrame, + TranscriptionUpdateFrame, + LLMThoughtEndFrame, + LLMThoughtStartFrame, + LLMThoughtTextFrame, + TranscriptionUpdateFrame, + LLMThoughtEndFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify both thoughts were captured + self.assertEqual(len(received_updates), 2) + + first_message = received_updates[0].messages[0] + self.assertIsInstance(first_message, ThoughtTranscriptionMessage) + self.assertEqual(first_message.content, "First consideration") + + second_message = received_updates[1].messages[0] + self.assertIsInstance(second_message, ThoughtTranscriptionMessage) + self.assertEqual(second_message.content, "Second consideration") + + # Verify timestamps are different + self.assertNotEqual(first_message.timestamp, second_message.timestamp) + + async def test_empty_thought_handling(self): + """Test that empty thoughts are not emitted""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + LLMThoughtStartFrame(), + LLMThoughtTextFrame(text=""), # Empty + LLMThoughtTextFrame(text=" "), # Just whitespace + LLMThoughtEndFrame(), + ] + + expected_down_frames = [ + LLMThoughtStartFrame, + LLMThoughtTextFrame, + LLMThoughtTextFrame, + LLMThoughtEndFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify no updates emitted for empty content + self.assertEqual(len(received_updates), 0) + + async def test_thought_without_start_frame(self): + """Test that thought text without start frame is ignored""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Send thought text without start frame + frames_to_send = [ + LLMThoughtTextFrame(text="This should be ignored"), + LLMThoughtEndFrame(), + ] + + expected_down_frames = [ + LLMThoughtTextFrame, + LLMThoughtEndFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify no updates since thought wasn't properly started + self.assertEqual(len(received_updates), 0) From 61674d7758e6b4940623acd1f766f025db37b07e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 10:27:36 -0500 Subject: [PATCH 11/21] Add `process_thought` constructor argument to `TranscriptProcessor` to control whether to handle thoughts in addition to assistant utterances. Defaults to `False`. --- .../foundational/49-thinking-functions.py | 2 +- examples/foundational/49-thinking.py | 2 +- .../processors/transcript_processor.py | 33 ++++++++++++------- tests/test_transcript_processor.py | 16 ++++----- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/examples/foundational/49-thinking-functions.py b/examples/foundational/49-thinking-functions.py index d24adbf3e..411316efe 100644 --- a/examples/foundational/49-thinking-functions.py +++ b/examples/foundational/49-thinking-functions.py @@ -125,7 +125,7 @@ async def run_bot( tools = ToolsSchema(standard_tools=[check_flight_status, book_taxi]) - transcript = TranscriptProcessor() + transcript = TranscriptProcessor(process_thoughts=True) messages = [ { diff --git a/examples/foundational/49-thinking.py b/examples/foundational/49-thinking.py index 756134076..74da17c66 100644 --- a/examples/foundational/49-thinking.py +++ b/examples/foundational/49-thinking.py @@ -99,7 +99,7 @@ async def run_bot( else: raise ValueError(f"Unsupported LLM provider: {llm_provider}") - transcript = TranscriptProcessor() + transcript = TranscriptProcessor(process_thoughts=True) messages = [ { diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 150d6c5c8..0dd59f1b3 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -101,14 +101,16 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): - The pipeline ends (EndFrame, CancelFrame) """ - def __init__(self, **kwargs): + def __init__(self, *, process_thoughts: bool = False, **kwargs): """Initialize processor with aggregation state. Args: + process_thoughts: Whether to process LLM thought frames. Defaults to False. **kwargs: Additional arguments passed to parent class. """ super().__init__(**kwargs) + self._process_thoughts = process_thoughts self._current_assistant_text_parts: List[TextPartForConcatenation] = [] self._assistant_text_start_time: Optional[str] = None @@ -188,18 +190,19 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): await self.push_frame(frame, direction) # Emit accumulated text and thought with interruptions await self._emit_aggregated_assistant_text() - if self._thought_active: + if self._process_thoughts and self._thought_active: await self._emit_aggregated_thought() elif isinstance(frame, LLMThoughtStartFrame): # Start a new thought - self._thought_active = True - self._thought_start_time = time_now_iso8601() - self._current_thought_parts = [] + if self._process_thoughts: + self._thought_active = True + self._thought_start_time = time_now_iso8601() + self._current_thought_parts = [] # Push frame. await self.push_frame(frame, direction) elif isinstance(frame, LLMThoughtTextFrame): # Aggregate thought text if we have an active thought - if self._thought_active: + if self._process_thoughts and self._thought_active: self._current_thought_parts.append( TextPartForConcatenation( frame.text, includes_inter_part_spaces=frame.includes_inter_frame_spaces @@ -209,7 +212,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): await self.push_frame(frame, direction) elif isinstance(frame, LLMThoughtEndFrame): # Emit accumulated thought when thought ends - if self._thought_active: + if self._process_thoughts and self._thought_active: await self._emit_aggregated_thought() # Push frame. await self.push_frame(frame, direction) @@ -230,7 +233,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): # Emit accumulated text when bot finishes speaking or pipeline ends. await self._emit_aggregated_assistant_text() # Emit accumulated thought at pipeline end if still active - if isinstance(frame, EndFrame) and self._thought_active: + if isinstance(frame, EndFrame) and self._process_thoughts and self._thought_active: await self._emit_aggregated_thought() # Push frame. await self.push_frame(frame, direction) @@ -268,8 +271,14 @@ class TranscriptProcessor: print(f"New messages: {frame.messages}") """ - def __init__(self): - """Initialize factory.""" + def __init__(self, *, process_thoughts: bool = False): + """Initialize factory. + + Args: + process_thoughts: Whether the assistant processor should handle LLM thought + frames. Defaults to False. + """ + self._process_thoughts = process_thoughts self._user_processor = None self._assistant_processor = None self._event_handlers = {} @@ -304,7 +313,9 @@ class TranscriptProcessor: The assistant transcript processor instance. """ if self._assistant_processor is None: - self._assistant_processor = AssistantTranscriptProcessor(**kwargs) + self._assistant_processor = AssistantTranscriptProcessor( + process_thoughts=self._process_thoughts, **kwargs + ) # Apply any registered event handlers for event_name, handler in self._event_handlers.items(): diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index a5e0702a0..c8e15eb24 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -496,7 +496,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_basic_thought_transcription(self): """Test basic thought frame processing""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] @@ -533,7 +533,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_thought_aggregation(self): """Test that thought text frames are properly aggregated""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] @@ -575,7 +575,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_thought_with_interruption(self): """Test that thoughts are properly captured when interrupted""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] @@ -613,7 +613,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_thought_with_cancel(self): """Test that thoughts are properly captured when cancelled""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] @@ -649,7 +649,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_thought_with_end_frame(self): """Test that thoughts are captured when pipeline ends normally""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] @@ -683,7 +683,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_multiple_thoughts(self): """Test multiple separate thoughts in sequence""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] @@ -735,7 +735,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_empty_thought_handling(self): """Test that empty thoughts are not emitted""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] @@ -768,7 +768,7 @@ class TestThoughtTranscription(unittest.IsolatedAsyncioTestCase): async def test_thought_without_start_frame(self): """Test that thought text without start frame is ignored""" - processor = AssistantTranscriptProcessor() + processor = AssistantTranscriptProcessor(process_thoughts=True) received_updates: List[TranscriptionUpdateFrame] = [] From 17203ba3e699bd723b29a71548e1b9cb8ae7e68c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 10:50:19 -0500 Subject: [PATCH 12/21] Change `FunctionInProgressFrame.llm_specific_extra` to a more generic `FunctionInProgressFrame.append_extra_context_messages`. --- .../adapters/services/gemini_adapter.py | 5 ++--- src/pipecat/frames/frames.py | 19 +++++++++--------- .../aggregators/llm_response_universal.py | 20 +++---------------- src/pipecat/services/google/llm.py | 17 ++++++++++------ src/pipecat/services/llm_service.py | 19 +++++++++++------- 5 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index ec0afa885..52d8cd114 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -222,9 +222,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # messages if ( isinstance(message.message, dict) - and message.message.get("type") == "tool_call_extra" - and isinstance(data := message.message.get("data"), dict) - and (thought_signature := data.get("thought_signature")) + and message.message.get("type") == "fn_call_thought_signature" + and (thought_signature := message.message.get("signature")) ): self._apply_function_call_thought_signature_to_messages( thought_signature, message.message.get("tool_call_id"), messages diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 9209d603a..43cc357a2 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -38,7 +38,7 @@ from pipecat.utils.time import nanoseconds_to_str from pipecat.utils.utils import obj_count, obj_id if TYPE_CHECKING: - from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven + from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage, NotGiven from pipecat.processors.frame_processor import FrameProcessor @@ -1218,16 +1218,16 @@ class FunctionCallFromLLM: tool_call_id: A unique identifier for the function call. arguments: The arguments to pass to the function. context: The LLM context when the function call was made. - llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: - {"google": {"thought_signature": ...}} - Uses the LLM adapter's ID for LLM-specific messages as the key. + append_extra_context_messages: Optional extra messages to append to the + context after the function call message. Used to add Google + function-call-related thought signatures to the context. """ function_name: str tool_call_id: str arguments: Mapping[str, Any] context: Any - llm_specific_extra: Optional[Dict[str, Any]] = None + append_extra_context_messages: Optional[List["LLMContextMessage"]] = None @dataclass @@ -1765,18 +1765,17 @@ class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame): function_name: Name of the function being executed. tool_call_id: Unique identifier for this function call. arguments: Arguments passed to the function. - llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: - {"google": {"thought_signature": ...}} - Uses the LLM adapter's ID for LLM-specific messages as the key. cancel_on_interruption: Whether to cancel this call if interrupted. - + append_extra_context_messages: Optional extra messages to append to the + context after the function call message. Used to add Google + function-call-related thought signatures to the context. """ function_name: str tool_call_id: str arguments: Any - llm_specific_extra: Optional[Dict[str, Any]] = None cancel_on_interruption: bool = False + append_extra_context_messages: Optional[List["LLMContextMessage"]] = None @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 76641f0e0..2f7b5b97e 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -743,23 +743,9 @@ class LLMAssistantAggregator(LLMContextAggregator): } ) - # If there's LLM-specific extra data associated with this function call - # add it to the context as an adjacent LLM-specific message. The - # LLM-specific adapter can then use this extra data as needed, for - # example by merging it into the tool call message. This is how Google's - # "thought_signature" makes it into the tool call message. - if frame.llm_specific_extra: - for key, value in frame.llm_specific_extra.items(): - self._context.add_message( - LLMSpecificMessage( - llm=key, - message={ - "type": "tool_call_extra", - "data": value, - "tool_call_id": frame.tool_call_id, - }, - ) - ) + # Append to context any specified extra context messages + if frame.append_extra_context_messages: + self._context.add_messages(frame.append_extra_context_messages) self._function_calls_in_progress[frame.tool_call_id] = frame diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 408918287..393f2c814 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -43,7 +43,7 @@ from pipecat.frames.frames import ( UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, @@ -985,11 +985,16 @@ class GoogleLLMService(LLMService): tool_call_id=id, function_name=function_call.name, arguments=function_call.args or {}, - llm_specific_extra={ - self.get_llm_adapter().id_for_llm_specific_messages: { - "thought_signature": part.thought_signature - } - } + append_extra_context_messages=[ + LLMSpecificMessage( + llm=self.get_llm_adapter().id_for_llm_specific_messages, + message={ + "type": "fn_call_thought_signature", + "signature": part.thought_signature, + "tool_call_id": id, + }, + ) + ] if part.thought_signature else None, ) diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 71e31d9b6..91358dcf3 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -14,6 +14,7 @@ from typing import ( Awaitable, Callable, Dict, + List, Mapping, Optional, Protocol, @@ -44,7 +45,11 @@ from pipecat.frames.frames import ( StartFrame, UserImageRequestFrame, ) -from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage +from pipecat.processors.aggregators.llm_context import ( + LLMContext, + LLMContextMessage, + LLMSpecificMessage, +) from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, LLMUserAggregatorParams, @@ -127,9 +132,9 @@ class FunctionCallRunnerItem: tool_call_id: A unique identifier for the function call. arguments: The arguments for the function. context: The LLM context. - llm_specific_extra: Optional extra data specific to particular LLMs, e.g.: - {"google": {"thought_signature": ...}} - Uses the LLM adapter's ID for LLM-specific messages as the key. + append_extra_context_messages: Optional extra messages to append to the + context after the function call message. Used to add Google + function-call-related thought signatures to the context. run_llm: Optional flag to control LLM execution after function call. """ @@ -138,7 +143,7 @@ class FunctionCallRunnerItem: tool_call_id: str arguments: Mapping[str, Any] context: OpenAILLMContext | LLMContext - llm_specific_extra: Optional[Dict[str, Any]] = None + append_extra_context_messages: Optional[List[LLMContextMessage]] = None run_llm: Optional[bool] = None @@ -460,7 +465,7 @@ class LLMService(AIService): tool_call_id=function_call.tool_call_id, arguments=function_call.arguments, context=function_call.context, - llm_specific_extra=function_call.llm_specific_extra, + append_extra_context_messages=function_call.append_extra_context_messages, ) ) @@ -585,7 +590,7 @@ class LLMService(AIService): function_name=runner_item.function_name, tool_call_id=runner_item.tool_call_id, arguments=runner_item.arguments, - llm_specific_extra=runner_item.llm_specific_extra, + append_extra_context_messages=runner_item.append_extra_context_messages, cancel_on_interruption=item.cancel_on_interruption, ) From 7e92597c0efca6c6b4d3a18e1d34bd0af50320a4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 11:10:05 -0500 Subject: [PATCH 13/21] Remove `LLMThoughtSignatureFrame` in favor of using the more generic `LLMMessagesAppendFrame` --- .../adapters/services/gemini_adapter.py | 8 +++---- src/pipecat/frames/frames.py | 22 ------------------ .../aggregators/llm_response_universal.py | 14 ----------- src/pipecat/services/google/llm.py | 23 +++++++++++-------- 4 files changed, 18 insertions(+), 49 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 52d8cd114..f02f2acfc 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -222,10 +222,10 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # messages if ( isinstance(message.message, dict) - and message.message.get("type") == "fn_call_thought_signature" + and message.message.get("type") == "fn_thought_signature" and (thought_signature := message.message.get("signature")) ): - self._apply_function_call_thought_signature_to_messages( + self._apply_function_thought_signature_to_messages( thought_signature, message.message.get("tool_call_id"), messages ) continue @@ -234,7 +234,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # signature messages (Gemini 3 Pro) if ( isinstance(message.message, dict) - and message.message.get("type") == "thought_signature" + and message.message.get("type") == "non_fn_thought_signature" and (thought_signature := message.message.get("signature")) ): non_fn_thought_signatures.append(thought_signature) @@ -444,7 +444,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, ) - def _apply_function_call_thought_signature_to_messages( + def _apply_function_thought_signature_to_messages( self, thought_signature: bytes, tool_call_id: str, messages: List[Content] ) -> None: """Apply tool_call_extra metadata to the corresponding function call message. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 43cc357a2..335b92753 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -654,28 +654,6 @@ class LLMThoughtEndFrame(ControlFrame): return f"{self.name}(pts: {pts}, signature: {self.signature})" -@dataclass -class LLMThoughtSignatureFrame(DataFrame): - """Frame containing a standalone LLM thought signature (as opposed to a thought signature associated with a thought). - - This is useful for Gemini 3 Pro, which can output a signature at the end of - a response. - - Parameters: - llm: Identifier of the LLM provider for LLM-specific handling. - Needed because the thought signature is appended to context as an - `LLMSpecificMessage`. - signature: The thought signature data. - """ - - llm: str - signature: Any - - def __str__(self): - pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, signature: {self.signature})" - - @dataclass class LLMMessagesFrame(DataFrame): """Frame containing LLM messages for chat completion. diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 2f7b5b97e..debfefb07 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -48,7 +48,6 @@ from pipecat.frames.frames import ( LLMSetToolChoiceFrame, LLMSetToolsFrame, LLMThoughtEndFrame, - LLMThoughtSignatureFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, SpeechControlParamsFrame, @@ -644,8 +643,6 @@ class LLMAssistantAggregator(LLMContextAggregator): await self._handle_thought_text(frame) elif isinstance(frame, LLMThoughtEndFrame): await self._handle_thought_end(frame) - elif isinstance(frame, LLMThoughtSignatureFrame): - await self._handle_standalone_thought_signature(frame) elif isinstance(frame, LLMRunFrame): await self._handle_llm_run(frame) elif isinstance(frame, LLMMessagesAppendFrame): @@ -896,17 +893,6 @@ class LLMAssistantAggregator(LLMContextAggregator): ) ) - async def _handle_standalone_thought_signature(self, frame: LLMThoughtSignatureFrame): - self._context.add_message( - LLMSpecificMessage( - llm=frame.llm, - message={ - "type": "thought_signature", - "signature": frame.signature, - }, - ) - ) - def _context_updated_task_finished(self, task: asyncio.Task): self._context_updated_tasks.discard(task) diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 393f2c814..0d1d18735 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -32,10 +32,10 @@ from pipecat.frames.frames import ( LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, + LLMMessagesAppendFrame, LLMMessagesFrame, LLMTextFrame, LLMThoughtEndFrame, - LLMThoughtSignatureFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, LLMUpdateSettingsFrame, @@ -986,13 +986,12 @@ class GoogleLLMService(LLMService): function_name=function_call.name, arguments=function_call.args or {}, append_extra_context_messages=[ - LLMSpecificMessage( - llm=self.get_llm_adapter().id_for_llm_specific_messages, - message={ - "type": "fn_call_thought_signature", + self.get_llm_adapter().create_llm_specific_message( + { + "type": "fn_thought_signature", "signature": part.thought_signature, "tool_call_id": id, - }, + } ) ] if part.thought_signature @@ -1011,9 +1010,15 @@ class GoogleLLMService(LLMService): # calls. It will come in the last part of a response. if part.thought_signature and not part.function_call: await self.push_frame( - LLMThoughtSignatureFrame( - llm=self.get_llm_adapter().id_for_llm_specific_messages, - signature=part.thought_signature, + LLMMessagesAppendFrame( + [ + self.get_llm_adapter().create_llm_specific_message( + { + "type": "non_fn_thought_signature", + "signature": part.thought_signature, + } + ) + ] ) ) From aa0529ff821cbfa4db0e91372fac5e2a9d9aa4d1 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 11:27:33 -0500 Subject: [PATCH 14/21] Update comments for accuracy --- src/pipecat/adapters/services/gemini_adapter.py | 13 ++++++++----- src/pipecat/services/google/llm.py | 9 ++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index f02f2acfc..1c258651e 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -231,7 +231,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): continue # Special handling for non-function-call-related thought - # signature messages (Gemini 3 Pro) + # signature messages (Gemini 3 Pro mainly, but possibly others, + # too, especially when functions are involved in the + # conversation) if ( isinstance(message.message, dict) and message.message.get("type") == "non_fn_thought_signature" @@ -447,7 +449,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): def _apply_function_thought_signature_to_messages( self, thought_signature: bytes, tool_call_id: str, messages: List[Content] ) -> None: - """Apply tool_call_extra metadata to the corresponding function call message. + """Apply a function-related thought signature to the corresponding function call message. Args: thought_signature: The thought signature bytes to apply. @@ -476,10 +478,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): def _apply_non_function_thought_signatures_to_messages( self, thought_signatures: List[bytes], messages: List[Content] ) -> None: - """Apply non-function-call-related thought signatures to the last part of each non-function-call assistant message. + """Apply non-function-call-related thought signatures to the last part of corresponding non-function-call assistant messages. - Gemini 3 Pro outputs a thought signature at the end of each assistant - response. + Gemini 3 Pro (and, somewhat surprisingly, other models, too, when + functions are involved in the conversation) outputs a thought signature + at the end of assistant responses. Args: thought_signatures: The list of thought signature bytes to apply. diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 0d1d18735..504aef503 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -1005,9 +1005,12 @@ class GoogleLLMService(LLMService): ) await self.push_frame(frame) - # With Gemini 3 Pro, thought signatures can be - # included in any kind of part, not just function - # calls. It will come in the last part of a response. + # With Gemini 3 Pro (and, somewhat surprisingly, + # other models models, too, especially when + # functions are involved in the conversation), + # thought signatures can be included in any kind of + # part, not just function calls. It will come in + # the last part of a response. if part.thought_signature and not part.function_call: await self.push_frame( LLMMessagesAppendFrame( From 1249ee3de3f4991c80badf419aa15630b307dfb1 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 13:07:25 -0500 Subject: [PATCH 15/21] Better handle Gemini non-function thought signatures --- .../adapters/services/gemini_adapter.py | 74 ++++++++++++++----- src/pipecat/services/google/llm.py | 2 +- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 1c258651e..f5a237ef7 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -209,7 +209,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): system_instruction = None messages = [] tool_call_id_to_name_mapping = {} - non_fn_thought_signatures = [] + non_fn_signed_parts = [] # Process each message, converting to Google format as needed for message in universal_context_messages: @@ -237,9 +237,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): if ( isinstance(message.message, dict) and message.message.get("type") == "non_fn_thought_signature" - and (thought_signature := message.message.get("signature")) + and (signed_part := message.message.get("signed_part")) ): - non_fn_thought_signatures.append(thought_signature) + non_fn_signed_parts.append(signed_part) continue # Fall back to assuming that the message is already in Google @@ -269,7 +269,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # Apply non-function-call-related thought signatures to the appropriate # messages - self._apply_non_function_thought_signatures_to_messages(non_fn_thought_signatures, messages) + self._apply_non_function_thought_signatures_to_messages(non_fn_signed_parts, messages) # Check if we only have function-related messages (no regular text) has_regular_messages = any( @@ -476,19 +476,19 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): break def _apply_non_function_thought_signatures_to_messages( - self, thought_signatures: List[bytes], messages: List[Content] + self, signed_parts: List[Part], messages: List[Content] ) -> None: - """Apply non-function-call-related thought signatures to the last part of corresponding non-function-call assistant messages. + """Apply (optional, but recommended) non-function-call-related thought signatures to the last part of corresponding non-function-call assistant messages. Gemini 3 Pro (and, somewhat surprisingly, other models, too, when functions are involved in the conversation) outputs a thought signature at the end of assistant responses. Args: - thought_signatures: The list of thought signature bytes to apply. + signed_parts: A list of signed received Parts containing thought signatures to apply. messages: List of messages to search through. """ - if not thought_signatures: + if not signed_parts: return # Find all assistant (model) messages that aren't function calls @@ -504,17 +504,51 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): if not has_function_call: non_fn_assistant_messages.append(message) - # Warn if counts don't match - if len(thought_signatures) != len(non_fn_assistant_messages): - logger.warning( - f"Thought signature count ({len(thought_signatures)}) doesn't match " - f"non-function-call assistant message count ({len(non_fn_assistant_messages)})" - ) - # Apply thought signatures to the corresponding assistant messages - # Match them in order (oldest to newest) - for i, thought_signature in enumerate(thought_signatures): - if i < len(non_fn_assistant_messages): + # Match them using content heuristics, maintaining order (messages without signatures are skipped) + message_start_index = 0 # Track where to start searching for the next match + for signed_part in signed_parts: + thought_signature = getattr(signed_part, "thought_signature", None) + if not thought_signature: + continue + + # Search through remaining non-function assistant messages for a match + for i in range(message_start_index, len(non_fn_assistant_messages)): message = non_fn_assistant_messages[i] - if message.parts: - message.parts[-1].thought_signature = thought_signature + if not message.parts: + continue + + last_part = message.parts[-1] + matched = False + + # Check if signed part has text and last message part text has the same text or + # - is a prefix of that text (in case spoken text was truncated due to interruption) + # - is prefixed by that text (in case signed part was not the end of the assistant response... + # which is NOT supposed to happen, according to Google's docs, but seems to, for long responses...) + if hasattr(signed_part, "text") and signed_part.text: + if hasattr(last_part, "text") and last_part.text: + # Normalize whitespace for comparison + signed_text = " ".join(signed_part.text.split()) + last_text = " ".join(last_part.text.split()) + if ( + last_text == signed_text + or signed_text.startswith(last_text) + or last_text.startswith(signed_text) + ): + last_part.thought_signature = thought_signature + matched = True + + # Check if signed part has inline_data and last message part has matching inline_data + elif hasattr(signed_part, "inline_data") and signed_part.inline_data: + if ( + hasattr(last_part, "inline_data") + and last_part.inline_data + and last_part.inline_data.data == signed_part.inline_data.data + ): + last_part.thought_signature = thought_signature + matched = True + + # If we found a match, update start index and stop searching for this signed part + if matched: + message_start_index = i + 1 + break diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 504aef503..73f75447f 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -1018,7 +1018,7 @@ class GoogleLLMService(LLMService): self.get_llm_adapter().create_llm_specific_message( { "type": "non_fn_thought_signature", - "signature": part.thought_signature, + "signed_part": part, } ) ] From 229ff794d669d0a332ba50a43c8331f767975da5 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 15:56:40 -0500 Subject: [PATCH 16/21] Better handle Gemini non-function thought signatures --- .../adapters/services/gemini_adapter.py | 76 +++++++++++++------ src/pipecat/services/google/llm.py | 37 +++++++-- 2 files changed, 83 insertions(+), 30 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index f5a237ef7..5a7387aca 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -209,7 +209,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): system_instruction = None messages = [] tool_call_id_to_name_mapping = {} - non_fn_signed_parts = [] + non_fn_thought_signatures = [] # Process each message, converting to Google format as needed for message in universal_context_messages: @@ -230,16 +230,17 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): ) continue - # Special handling for non-function-call-related thought - # signature messages (Gemini 3 Pro mainly, but possibly others, - # too, especially when functions are involved in the - # conversation) + # Special handling for non-function-call-related thought- + # signature-containing messages if ( isinstance(message.message, dict) and message.message.get("type") == "non_fn_thought_signature" - and (signed_part := message.message.get("signed_part")) + and (thought_signature := message.message.get("signature")) + and (bookmark := message.message.get("bookmark")) ): - non_fn_signed_parts.append(signed_part) + non_fn_thought_signatures.append( + {"signature": thought_signature, "bookmark": bookmark} + ) continue # Fall back to assuming that the message is already in Google @@ -269,7 +270,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # Apply non-function-call-related thought signatures to the appropriate # messages - self._apply_non_function_thought_signatures_to_messages(non_fn_signed_parts, messages) + self._apply_non_function_thought_signatures_to_messages(non_fn_thought_signatures, messages) # Check if we only have function-related messages (no regular text) has_regular_messages = any( @@ -476,21 +477,37 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): break def _apply_non_function_thought_signatures_to_messages( - self, signed_parts: List[Part], messages: List[Content] + self, thought_signatures: List[dict], messages: List[Content] ) -> None: """Apply (optional, but recommended) non-function-call-related thought signatures to the last part of corresponding non-function-call assistant messages. Gemini 3 Pro (and, somewhat surprisingly, other models, too, when - functions are involved in the conversation) outputs a thought signature + functions are involved in the conversation) outputs thought signatures at the end of assistant responses. Args: - signed_parts: A list of signed received Parts containing thought signatures to apply. + thought_signatures: A list of dicts containing: + - "signature": a thought signature + - "bookmark": a bookmark to identify the message part to apply the signature to. + The bookmark may contain either: + - "text" + - "inline_data" messages: List of messages to search through. """ - if not signed_parts: + if not thought_signatures: return + # For debugging, print out thought signatures and their bookmarks + logger.trace(f"Thought signatures to apply: {len(thought_signatures)}") + for ts in thought_signatures: + bookmark = ts.get("bookmark") + if bookmark.get("text"): + text = bookmark["text"] + log_display_text = f"{text[:50]}..." if len(text) > 50 else text + logger.trace(f" - At text: {log_display_text}") + elif bookmark.get("inline_data"): + logger.trace(f" - At inline data") + # Find all assistant (model) messages that aren't function calls non_fn_assistant_messages = [] for message in messages: @@ -507,9 +524,10 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # Apply thought signatures to the corresponding assistant messages # Match them using content heuristics, maintaining order (messages without signatures are skipped) message_start_index = 0 # Track where to start searching for the next match - for signed_part in signed_parts: - thought_signature = getattr(signed_part, "thought_signature", None) - if not thought_signature: + for thought_signature_dict in thought_signatures: + signature = thought_signature_dict.get("signature") + bookmark = thought_signature_dict.get("bookmark") + if not signature: continue # Search through remaining non-function assistant messages for a match @@ -521,31 +539,41 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): last_part = message.parts[-1] matched = False - # Check if signed part has text and last message part text has the same text or + # If it's a text bookmark, check that the last message part text has the same text or # - is a prefix of that text (in case spoken text was truncated due to interruption) - # - is prefixed by that text (in case signed part was not the end of the assistant response... - # which is NOT supposed to happen, according to Google's docs, but seems to, for long responses...) - if hasattr(signed_part, "text") and signed_part.text: + # - is prefixed by that text (in case bookmark represents just first chunk of multi-chunk text) + if bookmark_text := bookmark.get("text"): if hasattr(last_part, "text") and last_part.text: # Normalize whitespace for comparison - signed_text = " ".join(signed_part.text.split()) + signed_text = " ".join(bookmark_text.split()) last_text = " ".join(last_part.text.split()) if ( last_text == signed_text or signed_text.startswith(last_text) or last_text.startswith(signed_text) ): - last_part.thought_signature = thought_signature + log_display_text = ( + f"{last_part.text[:50]}..." + if len(last_part.text) > 50 + else last_part.text + ) + logger.trace( + f"Applying thought signature to part with matching text: {log_display_text}" + ) + last_part.thought_signature = signature matched = True # Check if signed part has inline_data and last message part has matching inline_data - elif hasattr(signed_part, "inline_data") and signed_part.inline_data: + elif inline_data := bookmark.get("inline_data"): if ( hasattr(last_part, "inline_data") and last_part.inline_data - and last_part.inline_data.data == signed_part.inline_data.data + and last_part.inline_data.data == inline_data.data ): - last_part.thought_signature = thought_signature + logger.trace( + f"Applying thought signature to part with matching inline_data" + ) + last_part.thought_signature = signature matched = True # If we found a match, update start index and stop searching for this signed part diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 73f75447f..6ccea5eff 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -942,6 +942,7 @@ class GoogleLLMService(LLMService): ) function_calls = [] + previous_part = None async for chunk in response: # Stop TTFB metrics after the first chunk await self.stop_ttfb_metrics() @@ -1005,26 +1006,50 @@ class GoogleLLMService(LLMService): ) await self.push_frame(frame) - # With Gemini 3 Pro (and, somewhat surprisingly, - # other models models, too, especially when + # With Gemini 3 Pro (and, contrary to Google's + # docs, other models models, too, especially when # functions are involved in the conversation), - # thought signatures can be included in any kind of - # part, not just function calls. It will come in - # the last part of a response. + # thought signatures can be associated with any + # kind of Part, not just function calls. + # + # They should always be included in the last + # response Part. (*) + # + # (*) Since we're using the streaming API, though, + # where text Parts may be split across multiple + # chunks (each represented by a Part, confusingly), + # signatures may actually appear with the first + # chunk (Gemini 2.5) or in a trailing empty-text + # chunk (Gemini 3 Pro). if part.thought_signature and not part.function_call: + # Save a "bookmark" for the signature, so we + # can later stick it in the right place in + # context when sending it back to the LLM to + # continue the conversation. + bookmark = {} + if part.inline_data and part.inline_data.data: + bookmark["inline_data"] = {"inline_data": part.inline_data} + elif part.text is not None: + # Account for Gemini 3 Pro trailing + # empty-text chunk by using search_result, + # which accumulates all text so far. + bookmark["text"] = search_result await self.push_frame( LLMMessagesAppendFrame( [ self.get_llm_adapter().create_llm_specific_message( { "type": "non_fn_thought_signature", - "signed_part": part, + "signature": part.thought_signature, + "bookmark": bookmark, } ) ] ) ) + previous_part = part + if ( candidate.grounding_metadata and candidate.grounding_metadata.grounding_chunks From c5ff5cc2194017e318c2eddd16f5a2adf2e9be0a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 8 Dec 2025 16:09:59 -0500 Subject: [PATCH 17/21] Update CHANGELOG --- changelog/3175.added.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/changelog/3175.added.md b/changelog/3175.added.md index 49e109094..16c946068 100644 --- a/changelog/3175.added.md +++ b/changelog/3175.added.md @@ -10,14 +10,17 @@ - `LLMThoughtStartFrame` - `LLMThoughtTextFrame` - `LLMThoughtEndFrame` - 3. A generic mechanism for associating extra LLM-specific data with a function - call in context, used specifically to support Google's function-call-related + 3. A mechanism for appending arbitrary context messages after a function call + message, used specifically to support Google's function-call-related "thought signatures", which are necessary to ensure thinking continuity between function calls in a chain (where the model thinks, makes a function call, thinks some more, etc.). See: - - `FunctionCallInProgressFrame.llm_specific_extra` - - `LLMAssistantAggregator` handling of the above field - - `GeminiLLMAdapter` handling of`"tool_call_extra"` context messages + - `append_extra_context_messages` field in `FunctionInProgressFrame` and + helper types + - `GoogleLLMService` leveraging the new mechanism to add a Google-specific + `"fn_thought_signature"` message + - `LLMAssistantAggregator` handling of `append_extra_context_messages` + - `GeminiLLMAdapter` handling of `"fn_thought_signature"` messages 4. A generic mechanism for recording LLM thoughts to context, used specifically to support Anthropic, whose thought signatures are expected to appear alongside the text of the thoughts within assistant context @@ -25,15 +28,14 @@ - `LLMThoughtEndFrame.signature` - `LLMAssistantAggregator` handling of the above field - `AnthropicLLMAdapter` handling of `"thought"` context messages - 5. A generic mechanism for recording standalone thought signatures to context, - used specifically to support Gemini 3 Pro, which may return - non-function-call-related thought signatures at the end of - non-function-call assistant responses. See: - - `LLMThoughtSignatureFrame` - - `LLMAssistantAggregator` handling of the above frame - - `GeminiLLMAdapter` handling of `"thought_signature"` context messages + 5. Google-specific logic for inserting non-function-call-related thought + signatures into the context, to help maintain thinking continuity in a + chain of LLM calls. See: + - `GoogleLLMService` sending `LLMMessagesAppendFrame`s to add LLM-specific + `"non_fn_thought_signature"` messages to context + - `GeminiLLMAdapter` handling of `"non_fn_thought_signature"` messages 6. An expansion of `TranscriptProcessor` to process LLM thoughts in addition to user and assistant utterances. See: - - `TranscriptProcessor.thought()` - - `ThoughtTranscriptionMessage`, which may now also be emitted with the + - `TranscriptProcessor(process_thoughts=True)` (defaults to `False`) + - `ThoughtTranscriptionMessage`, which is now also emitted with the `"on_transcript_update"` event From 0e88ad672e8197c5047a072a8933456cbf8ad44c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 11 Dec 2025 14:41:16 -0500 Subject: [PATCH 18/21] Add `ThoughtTranscriptionMessage.role`, which is always `"assistant"` --- src/pipecat/frames/frames.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 335b92753..542f21fc0 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -516,6 +516,7 @@ class TranscriptionMessage: class ThoughtTranscriptionMessage: """An LLM thought message in a conversation transcript.""" + role: Literal["assistant"] = field(default="assistant", init=False) content: str timestamp: Optional[str] = None From 28248e9b00b8687c5052658358c03131166a5dde Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 11 Dec 2025 15:07:35 -0500 Subject: [PATCH 19/21] Split up thinking examples so that there isn't an `llm` command-line arg for controlling which LLM to use. This change is preparation for adding these examples to our suite of evals. --- ...-thinking.py => 49a-thinking-anthropic.py} | 83 ++------ examples/foundational/49b-thinking-google.py | 159 +++++++++++++++ ...py => 49c-thinking-functions-anthropic.py} | 64 +----- .../49d-thinking-functions-google.py | 182 ++++++++++++++++++ 4 files changed, 369 insertions(+), 119 deletions(-) rename examples/foundational/{49-thinking.py => 49a-thinking-anthropic.py} (64%) create mode 100644 examples/foundational/49b-thinking-google.py rename examples/foundational/{49-thinking-functions.py => 49c-thinking-functions-anthropic.py} (74%) create mode 100644 examples/foundational/49d-thinking-functions-google.py diff --git a/examples/foundational/49-thinking.py b/examples/foundational/49a-thinking-anthropic.py similarity index 64% rename from examples/foundational/49-thinking.py rename to examples/foundational/49a-thinking-anthropic.py index 74da17c66..6017f335e 100644 --- a/examples/foundational/49-thinking.py +++ b/examples/foundational/49a-thinking-anthropic.py @@ -4,10 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import argparse import os -import random -import sys from dotenv import load_dotenv from loguru import logger @@ -28,18 +25,12 @@ from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.google.llm import GoogleLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams load_dotenv(override=True) -# LLM provider constants -LLM_ANTHROPIC = "anthropic" -LLM_GOOGLE = "google" -LLM_DEFAULT = LLM_GOOGLE - # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -65,10 +56,8 @@ transport_params = { } -async def run_bot( - transport: BaseTransport, runner_args: RunnerArguments, llm_provider: str = LLM_DEFAULT -): - logger.info(f"Starting bot with {llm_provider.capitalize()} LLM") +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -77,27 +66,12 @@ async def run_bot( voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) - if llm_provider == LLM_ANTHROPIC: - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - params=AnthropicLLMService.InputParams( - thinking=AnthropicLLMService.ThinkingConfig(type="enabled", budget_tokens=2048) - ), - ) - elif llm_provider == LLM_GOOGLE: - llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower - params=GoogleLLMService.InputParams( - thinking=GoogleLLMService.ThinkingConfig( - # thinking_level="low", # Use this field instead of thinking_budget for Gemini 3 Pro. Defaults to "high". - thinking_budget=-1, # Dynamic thinking - include_thoughts=True, - ) - ), - ) - else: - raise ValueError(f"Unsupported LLM provider: {llm_provider}") + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), + params=AnthropicLLMService.InputParams( + thinking=AnthropicLLMService.ThinkingConfig(type="enabled", budget_tokens=2048) + ), + ) transcript = TranscriptProcessor(process_thoughts=True) @@ -137,15 +111,16 @@ async def run_bot( @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # Choose a random prompt to demonstrate thinking capabilities. - # These prompts were chosen from Google and Anthropic docs. - thinking_prompt_1 = "Analogize photosynthesis and growing up." - thinking_prompt_2 = "Compare and contrast electric cars and hybrid cars." - thinking_prompt_3 = "Are there an infinite number of prime numbers such that n mod 4 == 3?" - selected_prompt = random.choice([thinking_prompt_1, thinking_prompt_2, thinking_prompt_3]) - - # Kick off the conversation. - messages.append({"role": "user", "content": selected_prompt}) + # Kick off the conversation, using a prompt conducive to demonstrating + # thinking (chosen from Google and Anthropic docs). + messages.append( + { + "role": "user", + "content": "Analogize photosynthesis and growing up.", + # "content": "Compare and contrast electric cars and hybrid cars." + # "content": "Are there an infinite number of prime numbers such that n mod 4 == 3?" + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") @@ -169,31 +144,11 @@ async def run_bot( async def bot(runner_args: RunnerArguments): """Main bot entry point compatible with Pipecat Cloud.""" - # Get llm_provider from module attribute set in __main__ - llm_provider = getattr(sys.modules[__name__], "llm_provider", LLM_DEFAULT) transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args, llm_provider) + await run_bot(transport, runner_args) if __name__ == "__main__": - # Parse custom arguments before calling runner main() - parser = argparse.ArgumentParser(description="Thinking LLM Bot") - parser.add_argument( - "--llm", - type=str, - choices=[LLM_ANTHROPIC, LLM_GOOGLE], - default=LLM_DEFAULT, - help=f"LLM provider to use (default: {LLM_DEFAULT})", - ) - # Parse only known args to allow runner's main() to handle its own args - args, remaining = parser.parse_known_args() - - # Store the llm_provider in sys.modules for bot() function to access - sys.modules[__name__].llm_provider = args.llm - - # Restore sys.argv with remaining args for runner's main() - sys.argv[1:] = remaining - from pipecat.runner.run import main main() diff --git a/examples/foundational/49b-thinking-google.py b/examples/foundational/49b-thinking-google.py new file mode 100644 index 000000000..85df6da39 --- /dev/null +++ b/examples/foundational/49b-thinking-google.py @@ -0,0 +1,159 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, ThoughtTranscriptionMessage, TranscriptionMessage +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower + params=GoogleLLMService.InputParams( + thinking=GoogleLLMService.ThinkingConfig( + # thinking_level="low", # Use this field instead of thinking_budget for Gemini 3 Pro. Defaults to "high". + thinking_budget=-1, # Dynamic thinking + include_thoughts=True, + ) + ), + ) + + transcript = TranscriptProcessor(process_thoughts=True) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts (including thoughts) + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation, using a prompt conducive to demonstrating + # thinking (chosen from Google and Anthropic docs). + messages.append( + { + "role": "user", + "content": "Analogize photosynthesis and growing up.", + # "content": "Compare and contrast electric cars and hybrid cars." + # "content": "Are there an infinite number of prime numbers such that n mod 4 == 3?" + } + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + for msg in frame.messages: + if isinstance(msg, (ThoughtTranscriptionMessage, TranscriptionMessage)): + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + role = "THOUGHT" if isinstance(msg, ThoughtTranscriptionMessage) else msg.role + logger.info(f"Transcript: {timestamp}{role}: {msg.content}") + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/49-thinking-functions.py b/examples/foundational/49c-thinking-functions-anthropic.py similarity index 74% rename from examples/foundational/49-thinking-functions.py rename to examples/foundational/49c-thinking-functions-anthropic.py index 411316efe..3d71f2c47 100644 --- a/examples/foundational/49-thinking-functions.py +++ b/examples/foundational/49c-thinking-functions-anthropic.py @@ -4,10 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import argparse import os -import random -import sys from dotenv import load_dotenv from loguru import logger @@ -29,7 +26,6 @@ from pipecat.runner.utils import create_transport from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.google.llm import GoogleLLMService from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -56,11 +52,6 @@ async def book_taxi(params: FunctionCallParams, time: str): await params.result_callback({"status": "done"}) -# LLM provider constants -LLM_ANTHROPIC = "anthropic" -LLM_GOOGLE = "google" -LLM_DEFAULT = LLM_GOOGLE - # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -86,10 +77,8 @@ transport_params = { } -async def run_bot( - transport: BaseTransport, runner_args: RunnerArguments, llm_provider: str = LLM_DEFAULT -): - logger.info(f"Starting bot with {llm_provider.capitalize()} LLM") +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -98,27 +87,12 @@ async def run_bot( voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) - if llm_provider == LLM_ANTHROPIC: - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - params=AnthropicLLMService.InputParams( - thinking=AnthropicLLMService.ThinkingConfig(type="enabled", budget_tokens=2048) - ), - ) - elif llm_provider == LLM_GOOGLE: - llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower - params=GoogleLLMService.InputParams( - thinking=GoogleLLMService.ThinkingConfig( - # thinking_level="low", # Use this field instead of thinking_budget for Gemini 3 Pro. Defaults to "high". - thinking_budget=-1, # Dynamic thinking - include_thoughts=True, - ) - ), - ) - else: - raise ValueError(f"Unsupported LLM provider: {llm_provider}") + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), + params=AnthropicLLMService.InputParams( + thinking=AnthropicLLMService.ThinkingConfig(type="enabled", budget_tokens=2048) + ), + ) llm.register_direct_function(check_flight_status) llm.register_direct_function(book_taxi) @@ -193,31 +167,11 @@ async def run_bot( async def bot(runner_args: RunnerArguments): """Main bot entry point compatible with Pipecat Cloud.""" - # Get llm_provider from module attribute set in __main__ - llm_provider = getattr(sys.modules[__name__], "llm_provider", LLM_DEFAULT) transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args, llm_provider) + await run_bot(transport, runner_args) if __name__ == "__main__": - # Parse custom arguments before calling runner main() - parser = argparse.ArgumentParser(description="Thinking LLM Bot") - parser.add_argument( - "--llm", - type=str, - choices=[LLM_ANTHROPIC, LLM_GOOGLE], - default=LLM_DEFAULT, - help=f"LLM provider to use (default: {LLM_DEFAULT})", - ) - # Parse only known args to allow runner's main() to handle its own args - args, remaining = parser.parse_known_args() - - # Store the llm_provider in sys.modules for bot() function to access - sys.modules[__name__].llm_provider = args.llm - - # Restore sys.argv with remaining args for runner's main() - sys.argv[1:] = remaining - from pipecat.runner.run import main main() diff --git a/examples/foundational/49d-thinking-functions-google.py b/examples/foundational/49d-thinking-functions-google.py new file mode 100644 index 000000000..3ec2b62d8 --- /dev/null +++ b/examples/foundational/49d-thinking-functions-google.py @@ -0,0 +1,182 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame, ThoughtTranscriptionMessage, TranscriptionMessage +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams + +load_dotenv(override=True) + + +async def check_flight_status(params: FunctionCallParams, flight_number: str): + """Check the status of a flight. Returns status (e.g., "on time", "delayed") and departure time. + + Args: + flight_number (str): The flight number, e.g. "AA100". + """ + await params.result_callback({"status": "delayed", "departure_time": "14:30"}) + + +async def book_taxi(params: FunctionCallParams, time: str): + """Book a taxi for a given time. Returns status (e.g., "done"). + + Args: + time (str): The time to book the taxi for, e.g. "15:00". + """ + await params.result_callback({"status": "done"}) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + # model="gemini-3-pro-preview", # A more powerful reasoning model, but slower + params=GoogleLLMService.InputParams( + thinking=GoogleLLMService.ThinkingConfig( + # thinking_level="low", # Use this field instead of thinking_budget for Gemini 3 Pro. Defaults to "high". + thinking_budget=-1, # Dynamic thinking + include_thoughts=True, + ) + ), + ) + + llm.register_direct_function(check_flight_status) + llm.register_direct_function(book_taxi) + + tools = ToolsSchema(standard_tools=[check_flight_status, book_taxi]) + + transcript = TranscriptProcessor(process_thoughts=True) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts (including thoughts) + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + # This example comes from Gemini docs. + messages.append( + { + "role": "user", + "content": "Check the status of flight AA100 and, if it's delayed, book me a taxi 2 hours before its departure time.", + } + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + for msg in frame.messages: + if isinstance(msg, (ThoughtTranscriptionMessage, TranscriptionMessage)): + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + role = "THOUGHT" if isinstance(msg, ThoughtTranscriptionMessage) else msg.role + logger.info(f"Transcript: {timestamp}{role}: {msg.content}") + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() From 12979293ad3fa74e650067bc30d3b2ee2d3a1fea Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 11 Dec 2025 15:58:48 -0500 Subject: [PATCH 20/21] Add thinking examples to eval suite --- examples/foundational/49a-thinking-anthropic.py | 17 ++++++++++++----- examples/foundational/49b-thinking-google.py | 17 ++++++++++++----- .../49c-thinking-functions-anthropic.py | 12 ++++++++++-- .../49d-thinking-functions-google.py | 12 ++++++++++-- scripts/evals/run-release-evals.py | 13 +++++++++++++ 5 files changed, 57 insertions(+), 14 deletions(-) diff --git a/examples/foundational/49a-thinking-anthropic.py b/examples/foundational/49a-thinking-anthropic.py index 6017f335e..a4a315ac5 100644 --- a/examples/foundational/49a-thinking-anthropic.py +++ b/examples/foundational/49a-thinking-anthropic.py @@ -111,16 +111,23 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # Kick off the conversation, using a prompt conducive to demonstrating - # thinking (chosen from Google and Anthropic docs). + # Kick off the conversation. messages.append( { "role": "user", - "content": "Analogize photosynthesis and growing up.", - # "content": "Compare and contrast electric cars and hybrid cars." - # "content": "Are there an infinite number of prime numbers such that n mod 4 == 3?" + "content": "Say hello briefly.", } ) + # Here are some example example prompts conducive to demonstrating + # thinking (picked from Google and Anthropic docs). + # messages.append( + # { + # "role": "user", + # "content": "Analogize photosynthesis and growing up. Keep your answer concise.", + # # "content": "Compare and contrast electric cars and hybrid cars." + # # "content": "Are there an infinite number of prime numbers such that n mod 4 == 3?" + # } + # ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/foundational/49b-thinking-google.py b/examples/foundational/49b-thinking-google.py index 85df6da39..40d82dec0 100644 --- a/examples/foundational/49b-thinking-google.py +++ b/examples/foundational/49b-thinking-google.py @@ -116,16 +116,23 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # Kick off the conversation, using a prompt conducive to demonstrating - # thinking (chosen from Google and Anthropic docs). + # Kick off the conversation. messages.append( { "role": "user", - "content": "Analogize photosynthesis and growing up.", - # "content": "Compare and contrast electric cars and hybrid cars." - # "content": "Are there an infinite number of prime numbers such that n mod 4 == 3?" + "content": "Say hello briefly.", } ) + # Here are some example example prompts conducive to demonstrating + # thinking (picked from Google and Anthropic docs). + # messages.append( + # { + # "role": "user", + # "content": "Analogize photosynthesis and growing up. Keep your answer concise.", + # # "content": "Compare and contrast electric cars and hybrid cars." + # # "content": "Are there an infinite number of prime numbers such that n mod 4 == 3?" + # } + # ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/foundational/49c-thinking-functions-anthropic.py b/examples/foundational/49c-thinking-functions-anthropic.py index 3d71f2c47..e821b9d09 100644 --- a/examples/foundational/49c-thinking-functions-anthropic.py +++ b/examples/foundational/49c-thinking-functions-anthropic.py @@ -138,13 +138,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") # Kick off the conversation. - # This example comes from Gemini docs. messages.append( { "role": "user", - "content": "Check the status of flight AA100 and, if it's delayed, book me a taxi 2 hours before its departure time.", + "content": "Say hello briefly.", } ) + # Here is an example prompt conducive to demonstrating thinking and + # function calling. + # This example comes from Gemini docs. + # messages.append( + # { + # "role": "user", + # "content": "Check the status of flight AA100 and, if it's delayed, book me a taxi 2 hours before its departure time.", + # } + # ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/foundational/49d-thinking-functions-google.py b/examples/foundational/49d-thinking-functions-google.py index 3ec2b62d8..cdf4621b1 100644 --- a/examples/foundational/49d-thinking-functions-google.py +++ b/examples/foundational/49d-thinking-functions-google.py @@ -143,13 +143,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_client_connected(transport, client): logger.info(f"Client connected") # Kick off the conversation. - # This example comes from Gemini docs. messages.append( { "role": "user", - "content": "Check the status of flight AA100 and, if it's delayed, book me a taxi 2 hours before its departure time.", + "content": "Say hello briefly.", } ) + # Here is an example prompt conducive to demonstrating thinking and + # function calling. + # This example comes from Gemini docs. + # messages.append( + # { + # "role": "user", + # "content": "Check the status of flight AA100 and, if it's delayed, book me a taxi 2 hours before its departure time.", + # } + # ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index f45128133..863514c64 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -74,6 +74,11 @@ EVAL_CONVERSATION = EvalConfig( eval_speaks_first=True, ) +EVAL_FLIGHT_STATUS = EvalConfig( + prompt="Check the status of flight AA100.", + eval="The user says something about the status of flight AA100, such as whether it's on time or delayed.", +) + TESTS_07 = [ # 07 series @@ -204,6 +209,13 @@ TESTS_44 = [ ("44-voicemail-detection.py", EVAL_CONVERSATION), ] +TESTS_49 = [ + ("49a-thinking-anthropic.py", EVAL_SIMPLE_MATH), + ("49b-thinking-google.py", EVAL_SIMPLE_MATH), + ("49c-thinking-functions-anthropic.py", EVAL_FLIGHT_STATUS), + ("49d-thinking-functions-google.py", EVAL_FLIGHT_STATUS), +] + TESTS = [ *TESTS_07, *TESTS_12, @@ -216,6 +228,7 @@ TESTS = [ *TESTS_40, *TESTS_43, *TESTS_44, + *TESTS_49, ] From ccdd6cde52da82ea383c1984083a157f44dec42d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 11 Dec 2025 17:05:09 -0500 Subject: [PATCH 21/21] Fix a couple of typos in comments --- examples/foundational/49a-thinking-anthropic.py | 2 +- examples/foundational/49b-thinking-google.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/foundational/49a-thinking-anthropic.py b/examples/foundational/49a-thinking-anthropic.py index a4a315ac5..4066a15c0 100644 --- a/examples/foundational/49a-thinking-anthropic.py +++ b/examples/foundational/49a-thinking-anthropic.py @@ -118,7 +118,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "content": "Say hello briefly.", } ) - # Here are some example example prompts conducive to demonstrating + # Here are some example prompts conducive to demonstrating # thinking (picked from Google and Anthropic docs). # messages.append( # { diff --git a/examples/foundational/49b-thinking-google.py b/examples/foundational/49b-thinking-google.py index 40d82dec0..947ab39c9 100644 --- a/examples/foundational/49b-thinking-google.py +++ b/examples/foundational/49b-thinking-google.py @@ -123,7 +123,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): "content": "Say hello briefly.", } ) - # Here are some example example prompts conducive to demonstrating + # Here are some example prompts conducive to demonstrating # thinking (picked from Google and Anthropic docs). # messages.append( # {