Merge pull request #3420 from pipecat-ai/pk/fix-gemini-3-parallel-function-calls

Fix parallel function calling with Gemini 3.

scripts/evals/run-release-evals.py

@@ -30,10 +30,15 @@ EVAL_SIMPLE_MATH = EvalConfig(
 )
 
 EVAL_WEATHER = EvalConfig(
-    prompt="What's the weather in San Francisco? Temperature should be in fahrenheits.",
+    prompt="What's the weather in San Francisco? Temperature should be in Fahrenheit.",
     eval="The user talks about the weather in San Francisco, including the degrees.",
 )
 
+EVAL_WEATHER_AND_RESTAURANT = EvalConfig(
+    prompt="What's the weather in San Francisco, and what's a good restaurant there? Temperature should be in Fahrenheit.",
+    eval="The user talks about the weather in San Francisco, including the degrees, and provides a restaurant recommendation.",
+)
+
 EVAL_ONLINE_SEARCH = EvalConfig(
     prompt="What's the current date in UTC?",
     eval=f"Current date in UTC is {datetime.now(timezone.utc).strftime('%A, %B %d, %Y')}.",
@@ -146,10 +151,16 @@ TESTS_12 = [
     ("12d-describe-image-moondream.py", EVAL_VISION_IMAGE()),
 ]
 
+# For a few major services, we also test parallel function calling.
+# (We don't bother doing this with every single service, as it's expensive and
+# most rely on the same OpenAI-compatible implementation.)
 TESTS_14 = [
     ("14-function-calling.py", EVAL_WEATHER),
+    ("14-function-calling.py", EVAL_WEATHER_AND_RESTAURANT),
     ("14a-function-calling-anthropic.py", EVAL_WEATHER),
+    ("14a-function-calling-anthropic.py", EVAL_WEATHER_AND_RESTAURANT),
     ("14e-function-calling-google.py", EVAL_WEATHER),
+    ("14e-function-calling-google.py", EVAL_WEATHER_AND_RESTAURANT),
     ("14f-function-calling-groq.py", EVAL_WEATHER),
     ("14g-function-calling-grok.py", EVAL_WEATHER),
     ("14h-function-calling-azure.py", EVAL_WEATHER),
@@ -161,6 +172,7 @@ TESTS_14 = [
     ("14p-function-calling-gemini-vertex-ai.py", EVAL_WEATHER),
     ("14q-function-calling-qwen.py", EVAL_WEATHER),
     ("14r-function-calling-aws.py", EVAL_WEATHER),
+    ("14r-function-calling-aws.py", EVAL_WEATHER_AND_RESTAURANT),
     ("14v-function-calling-openai.py", EVAL_WEATHER),
     ("14w-function-calling-mistral.py", EVAL_WEATHER),
     ("14x-function-calling-openpipe.py", EVAL_WEATHER),

src/pipecat/adapters/services/gemini_adapter.py

@@ -255,6 +255,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
         # Apply thought signatures to the corresponding messages
         self._apply_thought_signatures_to_messages(thought_signature_dicts, messages)
 
+        # When thinking is enabled, merge parallel tool calls into single messages
+        messages = self._merge_parallel_tool_calls_for_thinking(thought_signature_dicts, messages)
+
         # Check if we only have function-related messages (no regular text)
         has_regular_messages = any(
             len(msg.parts) == 1
@@ -433,6 +436,103 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
             tool_call_id_to_name_mapping=tool_call_id_to_name_mapping,
         )
 
+    def _merge_parallel_tool_calls_for_thinking(
+        self, thought_signature_dicts: List[dict], messages: List[Content]
+    ) -> List[Content]:
+        """Merge parallel tool calls into single Content objects when thinking is enabled.
+
+        Gemini expects parallel tool calls (multiple function calls made
+        simultaneously) to be in a single Content with multiple function_call
+        Parts. This method takes a list of Content messages, where parallel
+        tool calls may be split across multiple messages, and merges them into
+        single messages.
+
+        This only has an effect when thought_signatures are present (i.e., when
+        thinking is enabled). When thinking is disabled, merging doesn't matter.
+        When thinking is enabled, there is a guarantee that the first tool call
+        (and only the first) in any batch of parallel tool calls will have a
+        thought_signature. This allows us to distinguish:
+
+        - Parallel tool calls: share a single thought_signature (on the first call)
+        - Sequential tool calls: each have their own thought_signature
+
+        Algorithm: A tool call message with a thought_signature starts a new
+        parallel group. Any tool call messages after it without a
+        thought_signature get merged into that group, regardless of what
+        messages appear in between.
+
+        Args:
+            thought_signature_dicts: A list of thought signature dicts, used
+                to determine if the work of merging is necessary.
+            messages: List of Content messages to process.
+
+        Returns:
+            List of Content messages with parallel tool calls merged when
+            thought_signatures are present, otherwise unchanged.
+        """
+        if not messages:
+            return messages
+
+        # Fast-exit if no function-call-related thought signatures
+        # This is a shortcut for determining both:
+        # - whether thinking is enabled, and
+        # - whether there are function calls in the messages
+        has_function_call_signatures = any(
+            ts.get("bookmark", {}).get("function_call") for ts in thought_signature_dicts
+        )
+        if not has_function_call_signatures:
+            return messages
+
+        def is_tool_call_message(msg: Content) -> bool:
+            """Check if message contains only function_call parts."""
+            return (
+                msg.role == "model"
+                and msg.parts
+                and all(getattr(part, "function_call", None) for part in msg.parts)
+            )
+
+        def message_has_thought_signature(msg: Content) -> bool:
+            """Check if any part in the message has a thought_signature."""
+            return any(getattr(part, "thought_signature", None) for part in msg.parts)
+
+        merged_messages = []
+        i = 0
+
+        while i < len(messages):
+            current = messages[i]
+
+            # If this is a tool call message with a thought signature, start merging
+            if is_tool_call_message(current) and message_has_thought_signature(current):
+                merged_parts = list(current.parts)
+                other_messages = []
+                j = i + 1
+
+                # Scan forward, merging tool calls without signatures, collecting others
+                while j < len(messages):
+                    next_msg = messages[j]
+                    if is_tool_call_message(next_msg):
+                        if message_has_thought_signature(next_msg):
+                            # New parallel group starts, stop here
+                            break
+                        else:
+                            # Merge this call into the current group
+                            merged_parts.extend(next_msg.parts)
+                            j += 1
+                    else:
+                        # Collect non-tool-call message, keep scanning
+                        other_messages.append(next_msg)
+                        j += 1
+
+                # Output merged calls, then collected other messages
+                merged_messages.append(Content(role="model", parts=merged_parts))
+                merged_messages.extend(other_messages)
+                i = j
+            else:
+                merged_messages.append(current)
+                i += 1
+
+        return merged_messages
+
     def _apply_thought_signatures_to_messages(
         self, thought_signature_dicts: List[dict], messages: List[Content]
     ) -> None: