From fa7da8f5f6b8acaaf6daeee205e9606cbc936bbb Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Sat, 1 Mar 2025 00:20:16 +0530 Subject: [PATCH 1/3] adding vertex llm --- CHANGELOG.md | 3 ++ src/pipecat/services/google/google.py | 73 +++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47f1010a..b6001f82e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added foundational example `19a-azure-realtime-beta.py`. +- Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI + Gemini models. + ### Changed - Updated the default mode for `CartesiaTTSService` and diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 9d4a0a8a8..44a677136 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -71,6 +71,7 @@ try: import google.generativeai as gai from google import genai from google.api_core.client_options import ClientOptions + from google.auth.transport.requests import Request from google.cloud import speech_v2, texttospeech_v1 from google.cloud.speech_v2.types import cloud_speech from google.genai import types @@ -1333,6 +1334,78 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): ) +class GoogleVertexAIService(OpenAILLMService): + """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. + Reference: + https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library + """ + + class InputParams(OpenAILLMService.InputParams): + """Input parameters specific to Vertex AI.""" + + project_id: str + location: str + + def __init__( + self, + *, + credentials: Optional[str] = None, + credentials_path: Optional[str] = None, + model: str = "google/gemini-1.5-flash", + params: InputParams = OpenAILLMService.InputParams(), + **kwargs, + ): + """Initializes the VertexLLMService. + Args: + credentials (Optional[str]): JSON string of service account credentials. + credentials_path (Optional[str]): Path to the service account JSON file. + model (str): Model identifier. Defaults to "google/gemini-1.5-flash". + params (InputParams): Vertex AI input parameters. + **kwargs: Additional arguments for OpenAILLMService. + """ + base_url = self._get_base_url(params) + self._api_key = self._get_api_token(credentials, credentials_path) + + super().__init__(api_key=self._api_key, base_url=base_url, model=model, **kwargs) + + @staticmethod + def _get_base_url(params: InputParams) -> str: + """Constructs the base URL for Vertex AI API.""" + return ( + f"https://{params.location}-aiplatform.googleapis.com/v1/" + f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" + ) + + @staticmethod + def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: + """Retrieves an authentication token using Google service account credentials. + Args: + credentials (Optional[str]): JSON string of service account credentials. + credentials_path (Optional[str]): Path to the service account JSON file. + Returns: + str: OAuth token for API authentication. + """ + creds: Optional[service_account.Credentials] = None + + if credentials: + # Parse and load credentials from JSON string + creds = service_account.Credentials.from_service_account_info( + json.loads(credentials), scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + elif credentials_path: + # Load credentials from JSON file + creds = service_account.Credentials.from_service_account_file( + credentials_path, scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + + if not creds: + raise ValueError("No valid credentials provided.") + + creds.refresh(Request()) # Ensure token is up-to-date, lifetime is 1 hour. + + return creds.token + + class GoogleTTSService(TTSService): class InputParams(BaseModel): pitch: Optional[str] = None From 5f000efc611a3e7514df53e190abd6414296122d Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Sat, 15 Mar 2025 10:36:26 +0530 Subject: [PATCH 2/3] adding example --- CHANGELOG.md | 3 +- .../14p-function-calling-gemini-vertex-ai.py | 137 ++++++++++++++++++ src/pipecat/services/google/google.py | 7 +- 3 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 examples/foundational/14p-function-calling-gemini-vertex-ai.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b6001f82e..b3ee2646b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,7 +89,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 foundational example `19a-azure-realtime-beta.py`. - Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI - Gemini models. + Gemini models. Added foundational example + `14p-function-calling-gemini-vertex-ai.py`. ### Changed diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py new file mode 100644 index 000000000..68608e932 --- /dev/null +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.google import GoogleVertexAIService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) + + llm = GoogleVertexAIService( + # credentials="", + params=GoogleVertexAIService.InputParams( + project_id="", + ) + ) + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function( + "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather + ) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "user", + "content": "Start a conversation with 'Hey there' to get the current weather.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 44a677136..f741154b9 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1343,15 +1343,16 @@ class GoogleVertexAIService(OpenAILLMService): class InputParams(OpenAILLMService.InputParams): """Input parameters specific to Vertex AI.""" + # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations + location: str = "us-east4" project_id: str - location: str def __init__( self, *, credentials: Optional[str] = None, credentials_path: Optional[str] = None, - model: str = "google/gemini-1.5-flash", + model: str = "google/gemini-2.0-flash-001", params: InputParams = OpenAILLMService.InputParams(), **kwargs, ): @@ -1359,7 +1360,7 @@ class GoogleVertexAIService(OpenAILLMService): Args: credentials (Optional[str]): JSON string of service account credentials. credentials_path (Optional[str]): Path to the service account JSON file. - model (str): Model identifier. Defaults to "google/gemini-1.5-flash". + model (str): Model identifier. Defaults to "google/gemini-2.0-flash-001". params (InputParams): Vertex AI input parameters. **kwargs: Additional arguments for OpenAILLMService. """ From 4303ed4991a930bb4c4d3d268ea7991db3f78e56 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 18 Mar 2025 20:58:21 +0530 Subject: [PATCH 3/3] rename service --- CHANGELOG.md | 2 +- .../foundational/14p-function-calling-gemini-vertex-ai.py | 6 +++--- src/pipecat/services/google/google.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ee2646b..74cd0b15f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,7 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added foundational example `19a-azure-realtime-beta.py`. -- Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI +- Introduced `GoogleVertexLLMService`, a new class for integrating with Vertex AI Gemini models. Added foundational example `14p-function-calling-gemini-vertex-ai.py`. diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index 68608e932..d64bae89d 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -21,7 +21,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.google import GoogleVertexAIService +from pipecat.services.google import GoogleVertexLLMService from pipecat.services.openai import OpenAILLMContext from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -62,9 +62,9 @@ async def main(): voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), ) - llm = GoogleVertexAIService( + llm = GoogleVertexLLMService( # credentials="", - params=GoogleVertexAIService.InputParams( + params=GoogleVertexLLMService.InputParams( project_id="", ) ) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index f741154b9..f9a8d4894 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1334,7 +1334,7 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): ) -class GoogleVertexAIService(OpenAILLMService): +class GoogleVertexLLMService(OpenAILLMService): """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library