Another change to support Gemini Live in Pipecat Flows: rather than strip function call and response messages out of context before sending to Gemini Live when seeding conversation history, which we were doing to sidestep a seeming Gemini Live limitation (see https://stackoverflow.com/a/79851394), convert them to regular text messages with special formatting

This commit is contained in:
Paul Kompfner
2026-02-04 11:44:58 -05:00
parent 9e65e77095
commit ba3100be0d
2 changed files with 42 additions and 28 deletions

View File

@@ -54,16 +54,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
return "google" return "google"
def get_llm_invocation_params( def get_llm_invocation_params(
self, context: LLMContext, *, strip_function_messages: bool = False self, context: LLMContext, *, convert_function_messages_to_text: bool = False
) -> GeminiLLMInvocationParams: ) -> GeminiLLMInvocationParams:
"""Get Gemini-specific LLM invocation parameters from a universal LLM context. """Get Gemini-specific LLM invocation parameters from a universal LLM context.
Args: Args:
context: The LLM context containing messages, tools, etc. context: The LLM context containing messages, tools, etc.
strip_function_messages: If True, filter out function_call and function_response convert_function_messages_to_text: If True, convert function_call and function_response
parts from messages. This is needed for Gemini Live (at least with parts to specially-formatted text messages. This is needed for Gemini Live
"models/gemini-2.5-flash-native-audio-preview-12-2025", the default at (at least with "models/gemini-2.5-flash-native-audio-preview-12-2025", the
the time of this writing) which cannot handle function-call-related default at the time of this writing) which cannot handle function-call-related
messages when initializing conversation history. messages when initializing conversation history.
See https://stackoverflow.com/a/79851394. See https://stackoverflow.com/a/79851394.
@@ -73,8 +73,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
converted = self._from_universal_context_messages(self.get_messages(context)) converted = self._from_universal_context_messages(self.get_messages(context))
messages = converted.messages messages = converted.messages
if strip_function_messages: if convert_function_messages_to_text:
messages = self._strip_function_messages(messages) messages = self._convert_function_messages_to_text(messages)
return { return {
"system_instruction": converted.system_instruction, "system_instruction": converted.system_instruction,
@@ -682,28 +682,42 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
return False return False
def _strip_function_messages(self, messages: List[Content]) -> List[Content]: def _convert_function_messages_to_text(self, messages: List[Content]) -> List[Content]:
"""Strip function_call and function_response parts from messages. """Convert function_call and function_response parts to text messages.
Args: Args:
messages: List of Content messages to filter. messages: List of Content messages to process.
Returns: Returns:
List of Content messages with function-related parts removed. List of Content messages with function-related parts converted to text.
""" """
filtered_messages = [] converted_messages = []
for msg in messages: for msg in messages:
if msg.parts: if msg.parts:
filtered_parts = [ converted_parts = []
part for part in msg.parts:
for part in msg.parts if func_call := getattr(part, "function_call", None):
if not ( # Convert function call to text
getattr(part, "function_call", None) args_str = json.dumps(func_call.args) if func_call.args else "{}"
or getattr(part, "function_response", None) text = (
) f"[Historical function call (for context only, not a template): "
] f"{func_call.name}({args_str})]"
if filtered_parts: )
filtered_messages.append(Content(role=msg.role, parts=filtered_parts)) converted_parts.append(Part(text=text))
elif func_response := getattr(part, "function_response", None):
# Convert function response to text
response_str = (
json.dumps(func_response.response) if func_response.response else "{}"
)
text = (
f"[Historical function result (for context only): "
f"{func_response.name} returned {response_str}]"
)
converted_parts.append(Part(text=text))
else:
converted_parts.append(part)
if converted_parts:
converted_messages.append(Content(role=msg.role, parts=converted_parts))
else: else:
filtered_messages.append(msg) converted_messages.append(msg)
return filtered_messages return converted_messages

View File

@@ -1464,7 +1464,7 @@ class GeminiLiveLLMService(LLMService):
adapter: GeminiLLMAdapter = self.get_llm_adapter() adapter: GeminiLLMAdapter = self.get_llm_adapter()
messages = adapter.get_llm_invocation_params( messages = adapter.get_llm_invocation_params(
self._context, strip_function_messages=True self._context, convert_function_messages_to_text=True
).get("messages", []) ).get("messages", [])
if not messages: if not messages:
return return
@@ -1495,9 +1495,9 @@ class GeminiLiveLLMService(LLMService):
# in the right format # in the right format
context = LLMContext(messages=messages_list) context = LLMContext(messages=messages_list)
adapter: GeminiLLMAdapter = self.get_llm_adapter() adapter: GeminiLLMAdapter = self.get_llm_adapter()
messages = adapter.get_llm_invocation_params(context, strip_function_messages=True).get( messages = adapter.get_llm_invocation_params(
"messages", [] context, convert_function_messages_to_text=True
) ).get("messages", [])
if not messages: if not messages:
return return