Update GoogleLLMService docstrings

This commit is contained in:
Mark Backman
2025-06-26 10:37:22 -04:00
parent 03e3e9fae9
commit 9b64d2c325

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Google Gemini integration for Pipecat.
This module provides Google Gemini integration for the Pipecat framework,
including LLM services, context management, and message aggregation.
"""
import base64 import base64
import io import io
import json import json
@@ -71,7 +77,14 @@ except ModuleNotFoundError as e:
class GoogleUserContextAggregator(OpenAIUserContextAggregator): class GoogleUserContextAggregator(OpenAIUserContextAggregator):
"""Google-specific user context aggregator.
Extends OpenAI user context aggregator to handle Google AI's specific
Content and Part message format for user messages.
"""
async def push_aggregation(self): async def push_aggregation(self):
"""Push aggregated user text as a Google Content message."""
if len(self._aggregation) > 0: if len(self._aggregation) > 0:
self._context.add_message(Content(role="user", parts=[Part(text=self._aggregation)])) self._context.add_message(Content(role="user", parts=[Part(text=self._aggregation)]))
@@ -88,10 +101,26 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator):
class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
"""Google-specific assistant context aggregator.
Extends OpenAI assistant context aggregator to handle Google AI's specific
Content and Part message format for assistant responses and function calls.
"""
async def handle_aggregation(self, aggregation: str): async def handle_aggregation(self, aggregation: str):
"""Handle aggregated assistant text response.
Args:
aggregation: The aggregated text response from the assistant.
"""
self._context.add_message(Content(role="model", parts=[Part(text=aggregation)])) self._context.add_message(Content(role="model", parts=[Part(text=aggregation)]))
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
"""Handle function call in progress frame.
Args:
frame: Frame containing function call details.
"""
self._context.add_message( self._context.add_message(
Content( Content(
role="model", role="model",
@@ -120,6 +149,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
) )
async def handle_function_call_result(self, frame: FunctionCallResultFrame): async def handle_function_call_result(self, frame: FunctionCallResultFrame):
"""Handle function call result frame.
Args:
frame: Frame containing function call result.
"""
if frame.result: if frame.result:
await self._update_function_call_result( await self._update_function_call_result(
frame.function_name, frame.tool_call_id, frame.result frame.function_name, frame.tool_call_id, frame.result
@@ -130,6 +164,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
) )
async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
"""Handle function call cancellation frame.
Args:
frame: Frame containing function call cancellation details.
"""
await self._update_function_call_result( await self._update_function_call_result(
frame.function_name, frame.tool_call_id, "CANCELLED" frame.function_name, frame.tool_call_id, "CANCELLED"
) )
@@ -144,6 +183,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
part.function_response.response = {"value": json.dumps(result)} part.function_response.response = {"value": json.dumps(result)}
async def handle_user_image_frame(self, frame: UserImageRawFrame): async def handle_user_image_frame(self, frame: UserImageRawFrame):
"""Handle user image frame.
Args:
frame: Frame containing user image data and request context.
"""
await self._update_function_call_result( await self._update_function_call_result(
frame.request.function_name, frame.request.tool_call_id, "COMPLETED" frame.request.function_name, frame.request.tool_call_id, "COMPLETED"
) )
@@ -157,17 +201,45 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
@dataclass @dataclass
class GoogleContextAggregatorPair: class GoogleContextAggregatorPair:
"""Pair of Google context aggregators for user and assistant messages.
Parameters:
_user: User context aggregator for handling user messages.
_assistant: Assistant context aggregator for handling assistant responses.
"""
_user: GoogleUserContextAggregator _user: GoogleUserContextAggregator
_assistant: GoogleAssistantContextAggregator _assistant: GoogleAssistantContextAggregator
def user(self) -> GoogleUserContextAggregator: def user(self) -> GoogleUserContextAggregator:
"""Get the user context aggregator.
Returns:
The user context aggregator instance.
"""
return self._user return self._user
def assistant(self) -> GoogleAssistantContextAggregator: def assistant(self) -> GoogleAssistantContextAggregator:
"""Get the assistant context aggregator.
Returns:
The assistant context aggregator instance.
"""
return self._assistant return self._assistant
class GoogleLLMContext(OpenAILLMContext): class GoogleLLMContext(OpenAILLMContext):
"""Google AI LLM context that extends OpenAI context for Google-specific formatting.
This class handles conversion between OpenAI-style messages and Google AI's
Content/Part format, including system messages, function calls, and media.
Args:
messages: Initial messages in OpenAI format.
tools: Available tools/functions for the model.
tool_choice: Tool choice configuration.
"""
def __init__( def __init__(
self, self,
messages: Optional[List[dict]] = None, messages: Optional[List[dict]] = None,
@@ -179,6 +251,14 @@ class GoogleLLMContext(OpenAILLMContext):
@staticmethod @staticmethod
def upgrade_to_google(obj: OpenAILLMContext) -> "GoogleLLMContext": def upgrade_to_google(obj: OpenAILLMContext) -> "GoogleLLMContext":
"""Upgrade an OpenAI context to a Google context.
Args:
obj: OpenAI LLM context to upgrade.
Returns:
GoogleLLMContext instance with converted messages.
"""
if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GoogleLLMContext): if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GoogleLLMContext):
logger.debug(f"Upgrading to Google: {obj}") logger.debug(f"Upgrading to Google: {obj}")
obj.__class__ = GoogleLLMContext obj.__class__ = GoogleLLMContext
@@ -186,10 +266,20 @@ class GoogleLLMContext(OpenAILLMContext):
return obj return obj
def set_messages(self, messages: List): def set_messages(self, messages: List):
"""Set messages and restructure them for Google format.
Args:
messages: List of messages to set.
"""
self._messages[:] = messages self._messages[:] = messages
self._restructure_from_openai_messages() self._restructure_from_openai_messages()
def add_messages(self, messages: List): def add_messages(self, messages: List):
"""Add messages to the context, converting to Google format as needed.
Args:
messages: List of messages to add (can be mixed formats).
"""
# Convert each message individually # Convert each message individually
converted_messages = [] converted_messages = []
for msg in messages: for msg in messages:
@@ -206,6 +296,11 @@ class GoogleLLMContext(OpenAILLMContext):
self._messages.extend(converted_messages) self._messages.extend(converted_messages)
def get_messages_for_logging(self): def get_messages_for_logging(self):
"""Get messages formatted for logging with sensitive data redacted.
Returns:
List of message dictionaries with inline data redacted.
"""
msgs = [] msgs = []
for message in self.messages: for message in self.messages:
obj = message.to_json_dict() obj = message.to_json_dict()
@@ -222,6 +317,14 @@ class GoogleLLMContext(OpenAILLMContext):
def add_image_frame_message( def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
): ):
"""Add an image message to the context.
Args:
format: Image format (e.g., 'RGB', 'RGBA').
size: Image dimensions as (width, height).
image: Raw image bytes.
text: Optional text to accompany the image.
"""
buffer = io.BytesIO() buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG") Image.frombytes(format, size, image).save(buffer, format="JPEG")
@@ -235,6 +338,12 @@ class GoogleLLMContext(OpenAILLMContext):
def add_audio_frames_message( def add_audio_frames_message(
self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows"
): ):
"""Add audio frames as a message to the context.
Args:
audio_frames: List of audio frames to add.
text: Text description of the audio content.
"""
if not audio_frames: if not audio_frames:
return return
@@ -448,17 +557,37 @@ class GoogleLLMContext(OpenAILLMContext):
class GoogleLLMService(LLMService): class GoogleLLMService(LLMService):
"""This class implements inference with Google's AI models. """Google AI (Gemini) LLM service implementation.
This service translates internally from OpenAILLMContext to the messages format This class implements inference with Google's AI models, translating internally
expected by the Google AI model. We are using the OpenAILLMContext as a lingua from OpenAILLMContext to the messages format expected by the Google AI model.
franca for all LLM services, so that it is easy to switch between different LLMs. We use OpenAILLMContext as a lingua franca for all LLM services to enable
easy switching between different LLMs.
Args:
api_key: Google AI API key for authentication.
model: Model name to use. Defaults to "gemini-2.0-flash".
params: Input parameters for the model.
system_instruction: System instruction/prompt for the model.
tools: List of available tools/functions.
tool_config: Configuration for tool usage.
**kwargs: Additional arguments passed to parent class.
""" """
# Overriding the default adapter to use the Gemini one. # Overriding the default adapter to use the Gemini one.
adapter_class = GeminiLLMAdapter adapter_class = GeminiLLMAdapter
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Google AI models.
Parameters:
max_tokens: Maximum number of tokens to generate.
temperature: Sampling temperature between 0.0 and 2.0.
top_k: Top-k sampling parameter.
top_p: Top-p sampling parameter between 0.0 and 1.0.
extra: Additional parameters as a dictionary.
"""
max_tokens: Optional[int] = Field(default=4096, ge=1) max_tokens: Optional[int] = Field(default=4096, ge=1)
temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
top_k: Optional[int] = Field(default=None, ge=0) top_k: Optional[int] = Field(default=None, ge=0)
@@ -495,6 +624,11 @@ class GoogleLLMService(LLMService):
self._tool_config = tool_config self._tool_config = tool_config
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate usage metrics.
Returns:
True, as Google AI provides token usage metrics.
"""
return True return True
def _create_client(self, api_key: str): def _create_client(self, api_key: str):
@@ -653,6 +787,12 @@ class GoogleLLMService(LLMService):
await self.push_frame(LLMFullResponseEndFrame()) await self.push_frame(LLMFullResponseEndFrame())
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle different frame types.
Args:
frame: The frame to process.
direction: Direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
context = None context = None
@@ -681,16 +821,15 @@ class GoogleLLMService(LLMService):
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> GoogleContextAggregatorPair: ) -> GoogleContextAggregatorPair:
"""Create an instance of GoogleContextAggregatorPair from an """Create Google-specific context aggregators.
OpenAILLMContext. Constructor keyword arguments for both the user and
assistant aggregators can be provided. Creates a pair of context aggregators optimized for Google's message format,
including support for function calls, tool usage, and image handling.
Args: Args:
context (OpenAILLMContext): The LLM context. context: The LLM context to create aggregators for.
user_params (LLMUserAggregatorParams, optional): User aggregator user_params: Parameters for user message aggregation.
parameters. assistant_params: Parameters for assistant message aggregation.
assistant_params (LLMAssistantAggregatorParams, optional): User
aggregator parameters.
Returns: Returns:
GoogleContextAggregatorPair: A pair of context aggregators, one for GoogleContextAggregatorPair: A pair of context aggregators, one for