wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New lint rules and remove unused example file

Browse Source
This commit is contained in:
vipyne
2025-07-01 16:25:59 -05:00
parent a63d0da528
commit 79e51051c7
5 changed files with 125 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
examples/foundational/26g-gemini-multimodal-live-groundingMetadata.py
View File

@@ -1,164 +0,0 @@
import argparse
import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
from pipecat.frames.frames import Frame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
from pipecat.services.google.frames import LLMSearchResponseFrame
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True)
SYSTEM_INSTRUCTION = """
You are a helpful AI assistant that actively uses Google Search to provide up-to-date, accurate information.
IMPORTANT: For ANY question about current events, news, recent developments, real-time information, or anything that might have changed recently, you MUST use the google_search tool to get the latest information.
You should use Google Search for:
- Current news and events
- Recent developments in any field
- Today's weather, stock prices, or other real-time data
- Any question that starts with "what's happening", "latest", "recent", "current", "today", etc.
- When you're not certain about recent information
Always be proactive about using search when the user asks about anything that could benefit from real-time information.
Your output will be converted to audio so don't include special characters in your answers.
Respond to what the user said in a creative and helpful way, always using search for current information.
"""
class GroundingMetadataProcessor(FrameProcessor):
"""Processor to capture and display grounding metadata from Gemini Live API."""
def __init__(self):
super().__init__()
self._grounding_count = 0
async def process_frame(self, frame: Frame, direction: FrameDirection):
# Always call super().process_frame first
await super().process_frame(frame, direction)
# Only log important frame types, not every audio frame
if hasattr(frame, '__class__'):
frame_type = frame.__class__.__name__
if frame_type in ['LLMTextFrame', 'TTSTextFrame', 'LLMFullResponseStartFrame', 'LLMFullResponseEndFrame']:
logger.debug(f"GroundingProcessor received: {frame_type}")
if isinstance(frame, LLMSearchResponseFrame):
self._grounding_count += 1
logger.info(f"\n🔍 GROUNDING METADATA RECEIVED #{self._grounding_count}")
logger.info(f"📝 Search Result Text: {frame.search_result[:200]}...")
if frame.rendered_content:
logger.info(f"🔗 Rendered Content: {frame.rendered_content}")
if frame.origins:
logger.info(f"📍 Number of Origins: {len(frame.origins)}")
for i, origin in enumerate(frame.origins):
logger.info(f" Origin {i+1}: {origin.site_title} - {origin.site_uri}")
if origin.results:
logger.info(f" Results: {len(origin.results)} items")
# Always push the frame downstream
await self.push_frame(frame, direction)
async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace):
logger.info(f"Starting Gemini Live Grounding Test Bot")
# Initialize the SmallWebRTCTransport with the connection
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
video_in_enabled=False,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
),
)
# Create tools using ToolsSchema with custom tools for Gemini
tools = ToolsSchema(
standard_tools=[], # No standard function declarations needed
custom_tools={
AdapterType.GEMINI: [
{"google_search": {}},
{"code_execution": {}}
]
}
)
llm = GeminiMultimodalLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=SYSTEM_INSTRUCTION,
voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck
transcribe_user_audio=True,
tools=tools,
)
# Create a processor to capture grounding metadata
grounding_processor = GroundingMetadataProcessor()
messages = [
{
"role": "user",
"content": 'Please introduce yourself and let me know that you can help with current information by searching the web. Ask me what current information I\'d like to know about.',
},
]
# Set up conversation context and management
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(),
context_aggregator.user(),
llm,
grounding_processor, # Add our grounding processor here
transport.output(),
context_aggregator.assistant(),
]
)
task = PipelineTask(pipeline)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
if __name__ == "__main__":
from run import main
main()

2
src/pipecat/services/gemini_multimodal_live/__init__.py
View File

@@ -1,2 +1,2 @@
from .gemini import GeminiMultimodalLiveLLMService
from .file_api import GeminiFileAPI from .file_api import GeminiFileAPI
from .gemini import GeminiMultimodalLiveLLMService

15
src/pipecat/services/gemini_multimodal_live/events.py
View File

@@ -45,11 +45,12 @@ class ContentPart(BaseModel):
text: Optional[str] = Field(default=None, validate_default=False) text: Optional[str] = Field(default=None, validate_default=False)
inlineData: Optional[MediaChunk] = Field(default=None, validate_default=False) inlineData: Optional[MediaChunk] = Field(default=None, validate_default=False)
fileData: Optional['FileData'] = Field(default=None, validate_default=False) fileData: Optional["FileData"] = Field(default=None, validate_default=False)
class FileData(BaseModel): class FileData(BaseModel):
"""Represents a file reference in the Gemini File API.""" """Represents a file reference in the Gemini File API."""
mimeType: str mimeType: str
fileUri: str fileUri: str
@@ -255,22 +256,26 @@ class Config(BaseModel):
class SearchEntryPoint(BaseModel): class SearchEntryPoint(BaseModel):
"""Represents the search entry point with rendered content for search suggestions.""" """Represents the search entry point with rendered content for search suggestions."""
renderedContent: Optional[str] = None renderedContent: Optional[str] = None
class WebSource(BaseModel): class WebSource(BaseModel):
"""Represents a web source from grounding chunks.""" """Represents a web source from grounding chunks."""
uri: Optional[str] = None uri: Optional[str] = None
title: Optional[str] = None title: Optional[str] = None
class GroundingChunk(BaseModel): class GroundingChunk(BaseModel):
"""Represents a grounding chunk containing web source information.""" """Represents a grounding chunk containing web source information."""
web: Optional[WebSource] = None web: Optional[WebSource] = None
class GroundingSegment(BaseModel): class GroundingSegment(BaseModel):
"""Represents a segment of text that is grounded.""" """Represents a segment of text that is grounded."""
startIndex: Optional[int] = None startIndex: Optional[int] = None
endIndex: Optional[int] = None endIndex: Optional[int] = None
text: Optional[str] = None text: Optional[str] = None
@@ -278,6 +283,7 @@ class GroundingSegment(BaseModel):
class GroundingSupport(BaseModel): class GroundingSupport(BaseModel):
"""Represents support information for grounded text segments.""" """Represents support information for grounded text segments."""
segment: Optional[GroundingSegment] = None segment: Optional[GroundingSegment] = None
groundingChunkIndices: Optional[List[int]] = None groundingChunkIndices: Optional[List[int]] = None
confidenceScores: Optional[List[float]] = None confidenceScores: Optional[List[float]] = None
@@ -285,6 +291,7 @@ class GroundingSupport(BaseModel):
class GroundingMetadata(BaseModel): class GroundingMetadata(BaseModel):
"""Represents grounding metadata from Google Search.""" """Represents grounding metadata from Google Search."""
searchEntryPoint: Optional[SearchEntryPoint] = None searchEntryPoint: Optional[SearchEntryPoint] = None
groundingChunks: Optional[List[GroundingChunk]] = None groundingChunks: Optional[List[GroundingChunk]] = None
groundingSupports: Optional[List[GroundingSupport]] = None groundingSupports: Optional[List[GroundingSupport]] = None
@@ -491,9 +498,9 @@ def parse_server_event(message_str):
# Only log grounding metadata detection if truly needed for debugging # Only log grounding metadata detection if truly needed for debugging
# In production, this could be removed entirely or moved to TRACE level # In production, this could be removed entirely or moved to TRACE level
if 'serverContent' in evt_dict: if "serverContent" in evt_dict:
server_content = evt_dict['serverContent'] server_content = evt_dict["serverContent"]
if 'groundingMetadata' in server_content: if "groundingMetadata" in server_content:
# Consider removing this log entirely for production # Consider removing this log entirely for production
pass pass

57
src/pipecat/services/gemini_multimodal_live/file_api.py
View File

@@ -4,12 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import aiohttp
import mimetypes import mimetypes
from typing import Dict, Any, Optional from typing import Any, Dict, Optional
import aiohttp
from loguru import logger from loguru import logger
class GeminiFileAPI: class GeminiFileAPI:
"""Client for the Gemini File API. """Client for the Gemini File API.
@@ -21,7 +22,9 @@ class GeminiFileAPI:
project storage limited to 20GB. project storage limited to 20GB.
""" """
def __init__(self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files"): def __init__(
self, api_key: str, base_url: str = "https://generativelanguage.googleapis.com/v1beta/files"
):
"""Initialize the Gemini File API client. """Initialize the Gemini File API client.
Args: Args:
@@ -33,7 +36,9 @@ class GeminiFileAPI:
# Upload URL uses the /upload/ path # Upload URL uses the /upload/ path
self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files" self.upload_base_url = "https://generativelanguage.googleapis.com/upload/v1beta/files"
async def upload_file(self, file_path: str, display_name: Optional[str] = None) -> Dict[str, Any]: async def upload_file(
self, file_path: str, display_name: Optional[str] = None
) -> Dict[str, Any]:
"""Upload a file to the Gemini File API using the correct resumable upload protocol. """Upload a file to the Gemini File API using the correct resumable upload protocol.
Args: Args:
@@ -66,14 +71,12 @@ class GeminiFileAPI:
"X-Goog-Upload-Command": "start", "X-Goog-Upload-Command": "start",
"X-Goog-Upload-Header-Content-Length": str(len(file_data)), "X-Goog-Upload-Header-Content-Length": str(len(file_data)),
"X-Goog-Upload-Header-Content-Type": mime_type, "X-Goog-Upload-Header-Content-Type": mime_type,
"Content-Type": "application/json" "Content-Type": "application/json",
} }
logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}") logger.debug(f"Step 1: Getting upload URL from {self.upload_base_url}")
async with session.post( async with session.post(
f"{self.upload_base_url}?key={self.api_key}", f"{self.upload_base_url}?key={self.api_key}", headers=headers, json=metadata
headers=headers,
json=metadata
) as response: ) as response:
if response.status != 200: if response.status != 200:
error_text = await response.text() error_text = await response.text()
@@ -92,15 +95,11 @@ class GeminiFileAPI:
upload_headers = { upload_headers = {
"Content-Length": str(len(file_data)), "Content-Length": str(len(file_data)),
"X-Goog-Upload-Offset": "0", "X-Goog-Upload-Offset": "0",
"X-Goog-Upload-Command": "upload, finalize" "X-Goog-Upload-Command": "upload, finalize",
} }
logger.debug(f"Step 2: Uploading file data to {upload_url}") logger.debug(f"Step 2: Uploading file data to {upload_url}")
async with session.post( async with session.post(upload_url, headers=upload_headers, data=file_data) as response:
upload_url,
headers=upload_headers,
data=file_data
) as response:
if response.status != 200: if response.status != 200:
error_text = await response.text() error_text = await response.text()
logger.error(f"Error uploading file data: {error_text}") logger.error(f"Error uploading file data: {error_text}")
@@ -120,13 +119,11 @@ class GeminiFileAPI:
File metadata File metadata
""" """
# Extract just the name part if a full path is provided # Extract just the name part if a full path is provided
if '/' in name: if "/" in name:
name = name.split('/')[-1] name = name.split("/")[-1]
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get( async with session.get(f"{self.base_url}/{name}?key={self.api_key}") as response:
f"{self.base_url}/{name}?key={self.api_key}"
) as response:
if response.status != 200: if response.status != 200:
error_text = await response.text() error_text = await response.text()
logger.error(f"Error getting file metadata: {error_text}") logger.error(f"Error getting file metadata: {error_text}")
@@ -135,7 +132,9 @@ class GeminiFileAPI:
file_info = await response.json() file_info = await response.json()
return file_info return file_info
async def list_files(self, page_size: int = 10, page_token: Optional[str] = None) -> Dict[str, Any]: async def list_files(
self, page_size: int = 10, page_token: Optional[str] = None
) -> Dict[str, Any]:
"""List uploaded files. """List uploaded files.
Args: Args:
@@ -145,19 +144,13 @@ class GeminiFileAPI:
Returns: Returns:
List of files and next page token if available List of files and next page token if available
""" """
params = { params = {"key": self.api_key, "pageSize": page_size}
"key": self.api_key,
"pageSize": page_size
}
if page_token: if page_token:
params["pageToken"] = page_token params["pageToken"] = page_token
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get( async with session.get(self.base_url, params=params) as response:
self.base_url,
params=params
) as response:
if response.status != 200: if response.status != 200:
error_text = await response.text() error_text = await response.text()
logger.error(f"Error listing files: {error_text}") logger.error(f"Error listing files: {error_text}")
@@ -176,13 +169,11 @@ class GeminiFileAPI:
True if deleted successfully True if deleted successfully
""" """
# Extract just the name part if a full path is provided # Extract just the name part if a full path is provided
if '/' in name: if "/" in name:
name = name.split('/')[-1] name = name.split("/")[-1]
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.delete( async with session.delete(f"{self.base_url}/{name}?key={self.api_key}") as response:
f"{self.base_url}/{name}?key={self.api_key}"
) as response:
if response.status != 200: if response.status != 200:
error_text = await response.text() error_text = await response.text()
logger.error(f"Error deleting file: {error_text}") logger.error(f"Error deleting file: {error_text}")

70
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -59,11 +59,8 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame, OpenAILLMContextFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame, LLMSearchResult
from pipecat.services.llm_service import LLMService
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai.llm import ( from pipecat.services.openai.llm import (
OpenAIAssistantContextAggregator, OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator, OpenAIUserContextAggregator,
@@ -75,7 +72,6 @@ from pipecat.utils.time import time_now_iso8601
from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt
from . import events from . import events
from .audio_transcriber import AudioTranscriber
from .file_api import GeminiFileAPI from .file_api import GeminiFileAPI
try: try:
@@ -240,7 +236,9 @@ class GeminiMultimodalLiveContext(OpenAILLMContext):
parts.append({"type": "text", "text": text}) parts.append({"type": "text", "text": text})
# Add file reference part # Add file reference part
parts.append({"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}}) parts.append(
{"type": "file_data", "file_data": {"mime_type": mime_type, "file_uri": file_uri}}
)
# Add to messages # Add to messages
message = {"role": "user", "content": parts} message = {"role": "user", "content": parts}
@@ -273,12 +271,14 @@ class GeminiMultimodalLiveContext(OpenAILLMContext):
parts.append({"text": part.get("text")}) parts.append({"text": part.get("text")})
elif part.get("type") == "file_data": elif part.get("type") == "file_data":
file_data = part.get("file_data", {}) file_data = part.get("file_data", {})
parts.append({ parts.append(
"fileData": { {
"mimeType": file_data.get("mime_type"), "fileData": {
"fileUri": file_data.get("file_uri") "mimeType": file_data.get("mime_type"),
"fileUri": file_data.get("file_uri"),
}
} }
}) )
else: else:
logger.warning(f"Unsupported content type: {str(part)[:80]}") logger.warning(f"Unsupported content type: {str(part)[:80]}")
else: else:
@@ -1015,12 +1015,14 @@ class GeminiMultimodalLiveLLMService(LLMService):
parts.append({"text": part.get("text")}) parts.append({"text": part.get("text")})
elif part.get("type") == "file_data": elif part.get("type") == "file_data":
file_data = part.get("file_data", {}) file_data = part.get("file_data", {})
parts.append({ parts.append(
"fileData": { {
"mimeType": file_data.get("mime_type"), "fileData": {
"fileUri": file_data.get("file_uri") "mimeType": file_data.get("mime_type"),
"fileUri": file_data.get("file_uri"),
}
} }
}) )
else: else:
logger.warning(f"Unsupported content type: {str(part)[:80]}") logger.warning(f"Unsupported content type: {str(part)[:80]}")
else: else:
@@ -1167,7 +1169,9 @@ class GeminiMultimodalLiveLLMService(LLMService):
# Process grounding metadata if we have accumulated any # Process grounding metadata if we have accumulated any
if self._accumulated_grounding_metadata: if self._accumulated_grounding_metadata:
logger.debug("Processing grounding metadata...") logger.debug("Processing grounding metadata...")
await self._process_grounding_metadata(self._accumulated_grounding_metadata, self._search_result_buffer) await self._process_grounding_metadata(
self._accumulated_grounding_metadata, self._search_result_buffer
)
else: else:
logger.debug("No grounding metadata to process") logger.debug("No grounding metadata to process")
@@ -1288,14 +1292,20 @@ class GeminiMultimodalLiveLLMService(LLMService):
if evt.serverContent and evt.serverContent.groundingMetadata: if evt.serverContent and evt.serverContent.groundingMetadata:
grounding_metadata = evt.serverContent.groundingMetadata grounding_metadata = evt.serverContent.groundingMetadata
logger.debug(f"Grounding data: {len(grounding_metadata.groundingChunks or [])} chunks, {len(grounding_metadata.groundingSupports or [])} supports") logger.debug(
f"Grounding data: {len(grounding_metadata.groundingChunks or [])} chunks, {len(grounding_metadata.groundingSupports or [])} supports"
)
# Process the grounding metadata immediately # Process the grounding metadata immediately
await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer) await self._process_grounding_metadata(grounding_metadata, self._search_result_buffer)
async def _process_grounding_metadata(self, grounding_metadata: events.GroundingMetadata, search_result: str = ""): async def _process_grounding_metadata(
self, grounding_metadata: events.GroundingMetadata, search_result: str = ""
):
"""Process grounding metadata and emit LLMSearchResponseFrame.""" """Process grounding metadata and emit LLMSearchResponseFrame."""
logger.debug(f"Processing grounding metadata. Search result text length: {len(search_result)}") logger.debug(
f"Processing grounding metadata. Search result text length: {len(search_result)}"
)
if not grounding_metadata: if not grounding_metadata:
logger.warning("No grounding metadata provided to _process_grounding_metadata") logger.warning("No grounding metadata provided to _process_grounding_metadata")
return return
@@ -1304,7 +1314,10 @@ class GeminiMultimodalLiveLLMService(LLMService):
# Extract rendered content for search suggestions # Extract rendered content for search suggestions
rendered_content = None rendered_content = None
if grounding_metadata.searchEntryPoint and grounding_metadata.searchEntryPoint.renderedContent: if (
grounding_metadata.searchEntryPoint
and grounding_metadata.searchEntryPoint.renderedContent
):
rendered_content = grounding_metadata.searchEntryPoint.renderedContent rendered_content = grounding_metadata.searchEntryPoint.renderedContent
# Convert grounding chunks and supports to LLMSearchOrigin format # Convert grounding chunks and supports to LLMSearchOrigin format
@@ -1317,9 +1330,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
for index, chunk in enumerate(grounding_metadata.groundingChunks): for index, chunk in enumerate(grounding_metadata.groundingChunks):
if chunk.web: if chunk.web:
origin = LLMSearchOrigin( origin = LLMSearchOrigin(
site_uri=chunk.web.uri, site_uri=chunk.web.uri, site_title=chunk.web.title, results=[]
site_title=chunk.web.title,
results=[]
) )
chunk_to_origin[index] = origin chunk_to_origin[index] = origin
origins.append(origin) origins.append(origin)
@@ -1333,20 +1344,17 @@ class GeminiMultimodalLiveLLMService(LLMService):
# Add this result to all origins referenced by this support # Add this result to all origins referenced by this support
for chunk_index in support.groundingChunkIndices: for chunk_index in support.groundingChunkIndices:
if chunk_index in chunk_to_origin: if chunk_index in chunk_to_origin:
result = LLMSearchResult( result = LLMSearchResult(text=text, confidence=confidence_scores)
text=text,
confidence=confidence_scores
)
chunk_to_origin[chunk_index].results.append(result) chunk_to_origin[chunk_index].results.append(result)
# Create and push the search response frame # Create and push the search response frame
search_frame = LLMSearchResponseFrame( search_frame = LLMSearchResponseFrame(
search_result=search_result, search_result=search_result, origins=origins, rendered_content=rendered_content
origins=origins,
rendered_content=rendered_content
) )
logger.debug(f"Emitting LLMSearchResponseFrame with {len(origins)} origins, rendered_content available: {rendered_content is not None}") logger.debug(
f"Emitting LLMSearchResponseFrame with {len(origins)} origins, rendered_content available: {rendered_content is not None}"
)
await self.push_frame(search_frame) await self.push_frame(search_frame)
def create_context_aggregator( def create_context_aggregator(