From 99d94fc62506b67a256826d8ce756ace30ad3114 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 24 Oct 2025 10:04:02 -0400 Subject: [PATCH 1/4] Update Gemini service to use "user" role for function responses, as shown in the Gemini docs --- src/pipecat/adapters/services/gemini_adapter.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index abe33a8ec..53e4e4ee0 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -242,7 +242,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Converts to Google Content with:: Content( - role="model", + role="user", parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))] ) """ @@ -273,7 +273,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): ) ) elif role == "tool": - role = "model" + role = "user" try: response = json.loads(message["content"]) if isinstance(response, dict): @@ -285,11 +285,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string. response_dict = {"value": message["content"]} parts.append( - Part( - function_response=FunctionResponse( - name="tool_call_result", # seems to work to hard-code the same name every time - response=response_dict, - ) + Part.from_function_response( + name="tool_call_result", # seems to work to hard-code the same name every time + response=response_dict, ) ) elif isinstance(content, str): From 4ad15f9a01e43cece28f11be08e683b9e482d48d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 24 Oct 2025 10:38:17 -0400 Subject: [PATCH 2/4] Update Gemini service to include function name when sending function responses in context --- .../adapters/services/anthropic_adapter.py | 2 +- .../adapters/services/bedrock_adapter.py | 2 +- .../adapters/services/gemini_adapter.py | 106 +++++++++++++----- 3 files changed, 81 insertions(+), 29 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index adfe81005..a106b4de4 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -110,7 +110,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): system = NOT_GIVEN messages = [] - # first, map messages using self._from_universal_context_message(m) + # First, map messages using self._from_universal_context_message(m) try: messages = [self._from_universal_context_message(m) for m in universal_context_messages] except Exception as e: diff --git a/src/pipecat/adapters/services/bedrock_adapter.py b/src/pipecat/adapters/services/bedrock_adapter.py index 681dfb3dc..852ea17a4 100644 --- a/src/pipecat/adapters/services/bedrock_adapter.py +++ b/src/pipecat/adapters/services/bedrock_adapter.py @@ -107,7 +107,7 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): system = None messages = [] - # first, map messages using self._from_universal_context_message(m) + # First, map messages using self._from_universal_context_message(m) try: messages = [self._from_universal_context_message(m) for m in universal_context_messages] except Exception as e: diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 53e4e4ee0..75577f160 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -8,8 +8,8 @@ import base64 import json -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, TypedDict +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, TypedDict from loguru import logger from openai import NotGiven @@ -133,6 +133,28 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): messages: List[Content] system_instruction: Optional[str] = None + @dataclass + class MessageConversionResult: + """Result of converting a single universal context message to Google format. + + Either content (a Google Content object) or a system instruction string + is guaranteed to be set. + + Also returns a tool call ID to name mapping for any tool calls + discovered in the message. + """ + + content: Optional[Content] = None + system_instruction: Optional[str] = None + tool_call_id_to_name_mapping: Dict[str, str] = field(default_factory=dict) + + @dataclass + class MessageConversionParams: + """Parameters for converting a single universal context message to Google format.""" + + already_have_system_instruction: bool + tool_call_id_to_name_mapping: Dict[str, str] + def _from_universal_context_messages( self, universal_context_messages: List[LLMContextMessage] ) -> ConvertedMessages: @@ -156,24 +178,26 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): """ system_instruction = None messages = [] + tool_call_id_to_name_mapping = {} # Process each message, preserving Google-formatted messages and converting others for message in universal_context_messages: - if isinstance(message, LLMSpecificMessage): - # Assume that LLMSpecificMessage wraps a message in Google format - messages.append(message.message) - continue - - # Convert standard format to Google format - converted = self._from_standard_message( - message, already_have_system_instruction=bool(system_instruction) + result = self._from_universal_context_message( + message, + params=self.MessageConversionParams( + already_have_system_instruction=bool(system_instruction), + tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, + ), ) - if isinstance(converted, Content): - # Regular (non-system) message - messages.append(converted) - else: - # System instruction - system_instruction = converted + # Each result is either a Content or a system instruction + if result.content: + messages.append(result.content) + elif result.system_instruction: + system_instruction = result.system_instruction + + # Merge tool call ID to name mapping + if result.tool_call_id_to_name_mapping: + tool_call_id_to_name_mapping.update(result.tool_call_id_to_name_mapping) # Check if we only have function-related messages (no regular text) has_regular_messages = any( @@ -193,9 +217,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): return self.ConvertedMessages(messages=messages, system_instruction=system_instruction) + def _from_universal_context_message( + self, message: LLMContextMessage, *, params: MessageConversionParams + ) -> MessageConversionResult: + if isinstance(message, LLMSpecificMessage): + return message.message + return self._from_standard_message(message, params=params) + def _from_standard_message( - self, message: LLMStandardMessage, already_have_system_instruction: bool - ) -> Content | str: + self, message: LLMStandardMessage, *, params: MessageConversionParams + ) -> MessageConversionResult: """Convert standard universal context message to Google Content object. Handles conversion of text, images, and function calls to Google's @@ -205,10 +236,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): Args: message: Message in standard universal context format. already_have_system_instruction: Whether we already have a system instruction + params: Parameters for conversion. Returns: - Content object with role and parts, or a plain string for system - messages. + MessageConversionResult containing either a Content object or a + system instruction string. Examples: Standard text message:: @@ -248,26 +280,36 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): """ role = message["role"] content = message.get("content", []) + if role == "system": - if already_have_system_instruction: + if params.already_have_system_instruction: role = "user" # Convert system message to user role if we already have a system instruction else: - # System instructions are returned as plain text + system_instruction: str = None if isinstance(content, str): - return content + system_instruction = content elif isinstance(content, list): # If content is a list, we assume it's a list of text parts, per the standard - return " ".join(part["text"] for part in content if part.get("type") == "text") + system_instruction = " ".join( + part["text"] for part in content if part.get("type") == "text" + ) + if system_instruction: + return self.MessageConversionResult(system_instruction=system_instruction) elif role == "assistant": role = "model" parts = [] + tool_call_id_to_name_mapping = {} + if message.get("tool_calls"): for tc in message["tool_calls"]: + id = tc["id"] + name = tc["function"]["name"] + tool_call_id_to_name_mapping[id] = name parts.append( Part( function_call=FunctionCall( - name=tc["function"]["name"], + name=name, args=json.loads(tc["function"]["arguments"]), ) ) @@ -284,9 +326,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # Response might not be JSON-deserializable. # This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string. response_dict = {"value": message["content"]} + + # Get function name from mapping using tool_call_id, or fallback + tool_call_id = message.get("tool_call_id") + function_name = "tool_call_result" # Default fallback + if tool_call_id and tool_call_id in params.tool_call_id_to_name_mapping: + function_name = params.tool_call_id_to_name_mapping[tool_call_id] + parts.append( Part.from_function_response( - name="tool_call_result", # seems to work to hard-code the same name every time + name=function_name, response=response_dict, ) ) @@ -310,4 +359,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): audio_bytes = base64.b64decode(input_audio["data"]) parts.append(Part(inline_data=Blob(mime_type="audio/wav", data=audio_bytes))) - return Content(role=role, parts=parts) + return self.MessageConversionResult( + content=Content(role=role, parts=parts), + tool_call_id_to_name_mapping=tool_call_id_to_name_mapping, + ) From 949b8070230e496db1509c7b3a31939977de94e2 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 24 Oct 2025 11:36:25 -0400 Subject: [PATCH 3/4] Close genai client more gracefully to avoid printed warnings. We're now following the genai library guidance: https://github.com/googleapis/python-genai?tab=readme-ov-file#close-a-client --- src/pipecat/services/google/llm.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index b45c276d0..c59cb41ae 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -1034,6 +1034,23 @@ class GoogleLLMService(LLMService): if context: await self._process_context(context) + async def stop(self, frame): + """Override stop to gracefully close the client.""" + await super().stop(frame) + await self._close_client() + + async def cancel(self, frame): + """Override cancel to gracefully close the client.""" + await super().cancel(frame) + await self._close_client() + + async def _close_client(self): + try: + await self._client.aio.aclose() + except Exception: + # Do nothing - we're shutting down anyway + pass + def create_context_aggregator( self, context: OpenAILLMContext, From 6feaf9178920f9fddef5961ab2b431101e7d41de Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Oct 2025 09:42:24 -0400 Subject: [PATCH 4/4] Fix a bug in `GeminiLLMAdapter`'s handling of Gemini-specific context messages --- src/pipecat/adapters/services/gemini_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 75577f160..26f037127 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -221,7 +221,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): self, message: LLMContextMessage, *, params: MessageConversionParams ) -> MessageConversionResult: if isinstance(message, LLMSpecificMessage): - return message.message + return self.MessageConversionResult(content=message.message) return self._from_standard_message(message, params=params) def _from_standard_message(