Another change to support Gemini Live in Pipecat Flows: rather than strip function call and response messages out of context before sending to Gemini Live when seeding conversation history, which we were doing to sidestep a seeming Gemini Live limitation (see https://stackoverflow.com/a/79851394), convert them to regular text messages with special formatting
This commit is contained in:
@@ -54,16 +54,16 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
return "google"
|
return "google"
|
||||||
|
|
||||||
def get_llm_invocation_params(
|
def get_llm_invocation_params(
|
||||||
self, context: LLMContext, *, strip_function_messages: bool = False
|
self, context: LLMContext, *, convert_function_messages_to_text: bool = False
|
||||||
) -> GeminiLLMInvocationParams:
|
) -> GeminiLLMInvocationParams:
|
||||||
"""Get Gemini-specific LLM invocation parameters from a universal LLM context.
|
"""Get Gemini-specific LLM invocation parameters from a universal LLM context.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
context: The LLM context containing messages, tools, etc.
|
context: The LLM context containing messages, tools, etc.
|
||||||
strip_function_messages: If True, filter out function_call and function_response
|
convert_function_messages_to_text: If True, convert function_call and function_response
|
||||||
parts from messages. This is needed for Gemini Live (at least with
|
parts to specially-formatted text messages. This is needed for Gemini Live
|
||||||
"models/gemini-2.5-flash-native-audio-preview-12-2025", the default at
|
(at least with "models/gemini-2.5-flash-native-audio-preview-12-2025", the
|
||||||
the time of this writing) which cannot handle function-call-related
|
default at the time of this writing) which cannot handle function-call-related
|
||||||
messages when initializing conversation history.
|
messages when initializing conversation history.
|
||||||
See https://stackoverflow.com/a/79851394.
|
See https://stackoverflow.com/a/79851394.
|
||||||
|
|
||||||
@@ -73,8 +73,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
converted = self._from_universal_context_messages(self.get_messages(context))
|
converted = self._from_universal_context_messages(self.get_messages(context))
|
||||||
messages = converted.messages
|
messages = converted.messages
|
||||||
|
|
||||||
if strip_function_messages:
|
if convert_function_messages_to_text:
|
||||||
messages = self._strip_function_messages(messages)
|
messages = self._convert_function_messages_to_text(messages)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"system_instruction": converted.system_instruction,
|
"system_instruction": converted.system_instruction,
|
||||||
@@ -682,28 +682,42 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _strip_function_messages(self, messages: List[Content]) -> List[Content]:
|
def _convert_function_messages_to_text(self, messages: List[Content]) -> List[Content]:
|
||||||
"""Strip function_call and function_response parts from messages.
|
"""Convert function_call and function_response parts to text messages.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
messages: List of Content messages to filter.
|
messages: List of Content messages to process.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of Content messages with function-related parts removed.
|
List of Content messages with function-related parts converted to text.
|
||||||
"""
|
"""
|
||||||
filtered_messages = []
|
converted_messages = []
|
||||||
for msg in messages:
|
for msg in messages:
|
||||||
if msg.parts:
|
if msg.parts:
|
||||||
filtered_parts = [
|
converted_parts = []
|
||||||
part
|
for part in msg.parts:
|
||||||
for part in msg.parts
|
if func_call := getattr(part, "function_call", None):
|
||||||
if not (
|
# Convert function call to text
|
||||||
getattr(part, "function_call", None)
|
args_str = json.dumps(func_call.args) if func_call.args else "{}"
|
||||||
or getattr(part, "function_response", None)
|
text = (
|
||||||
)
|
f"[Historical function call (for context only, not a template): "
|
||||||
]
|
f"{func_call.name}({args_str})]"
|
||||||
if filtered_parts:
|
)
|
||||||
filtered_messages.append(Content(role=msg.role, parts=filtered_parts))
|
converted_parts.append(Part(text=text))
|
||||||
|
elif func_response := getattr(part, "function_response", None):
|
||||||
|
# Convert function response to text
|
||||||
|
response_str = (
|
||||||
|
json.dumps(func_response.response) if func_response.response else "{}"
|
||||||
|
)
|
||||||
|
text = (
|
||||||
|
f"[Historical function result (for context only): "
|
||||||
|
f"{func_response.name} returned {response_str}]"
|
||||||
|
)
|
||||||
|
converted_parts.append(Part(text=text))
|
||||||
|
else:
|
||||||
|
converted_parts.append(part)
|
||||||
|
if converted_parts:
|
||||||
|
converted_messages.append(Content(role=msg.role, parts=converted_parts))
|
||||||
else:
|
else:
|
||||||
filtered_messages.append(msg)
|
converted_messages.append(msg)
|
||||||
return filtered_messages
|
return converted_messages
|
||||||
|
|||||||
@@ -1464,7 +1464,7 @@ class GeminiLiveLLMService(LLMService):
|
|||||||
|
|
||||||
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
||||||
messages = adapter.get_llm_invocation_params(
|
messages = adapter.get_llm_invocation_params(
|
||||||
self._context, strip_function_messages=True
|
self._context, convert_function_messages_to_text=True
|
||||||
).get("messages", [])
|
).get("messages", [])
|
||||||
if not messages:
|
if not messages:
|
||||||
return
|
return
|
||||||
@@ -1495,9 +1495,9 @@ class GeminiLiveLLMService(LLMService):
|
|||||||
# in the right format
|
# in the right format
|
||||||
context = LLMContext(messages=messages_list)
|
context = LLMContext(messages=messages_list)
|
||||||
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
||||||
messages = adapter.get_llm_invocation_params(context, strip_function_messages=True).get(
|
messages = adapter.get_llm_invocation_params(
|
||||||
"messages", []
|
context, convert_function_messages_to_text=True
|
||||||
)
|
).get("messages", [])
|
||||||
|
|
||||||
if not messages:
|
if not messages:
|
||||||
return
|
return
|
||||||
|
|||||||
Reference in New Issue
Block a user