From bb89a036e5bc95caa21fb711a0c49eccbb15a8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 22:26:30 -0800 Subject: [PATCH 1/3] google: always send text part when sending inline audio --- CHANGELOG.md | 3 +++ src/pipecat/services/google/google.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afba77a64..3d41cb420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed a `GoogleLLMService` that was causing an exception when sending inline + audio in some cases. + - Fixed an `AudioContextWordTTSService` issue that would cause an `EndFrame` to disconnect from the TTS service before audio from all the contexts was received. This affected services like Cartesia and Rime. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 1dafc92bc..c0941ee33 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -722,7 +722,9 @@ class GoogleLLMContext(OpenAILLMContext): self.add_message(glm.Content(role="user", parts=parts)) - def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None): + def add_audio_frames_message( + self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" + ): if not audio_frames: return @@ -731,8 +733,9 @@ class GoogleLLMContext(OpenAILLMContext): parts = [] data = b"".join(frame.audio for frame in audio_frames) - if text: - parts.append(glm.Part(text=text)) + # NOTE(aleix): According to the docs only text or inline_data should be needed. + # (see https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference) + parts.append(glm.Part(text=text)) parts.append( glm.Part( inline_data=glm.Blob( From f83c89c20235b5aa83cf4b365c1fa41358fb3cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 22:28:02 -0800 Subject: [PATCH 2/3] examples: update google examples --- .../22d-natural-conversation-gemini-audio.py | 2 +- examples/phone-chatbot/bot_daily_gemini.py | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 9be9ec458..6313f951d 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -389,7 +389,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = GoogleLLMContext() - context.add_audio_frames_message(text="Audio follows", audio_frames=self._audio_frames) + context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(OpenAILLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest diff --git a/examples/phone-chatbot/bot_daily_gemini.py b/examples/phone-chatbot/bot_daily_gemini.py index 96f6506b6..8ada81a9c 100644 --- a/examples/phone-chatbot/bot_daily_gemini.py +++ b/examples/phone-chatbot/bot_daily_gemini.py @@ -7,20 +7,15 @@ import argparse import asyncio import os import sys -from dataclasses import dataclass from typing import Optional -import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, EndTaskFrame, - Frame, InputAudioRawFrame, - SystemFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, @@ -28,11 +23,11 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.ai_services import LLMService from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.google import GoogleLLMContext, GoogleLLMService +from pipecat.services.google import GoogleLLMService +from pipecat.services.google.google import GoogleLLMContext from pipecat.transports.services.daily import DailyDialinSettings, DailyParams, DailyTransport load_dotenv(override=True) @@ -240,7 +235,7 @@ If it sounds like a human (saying hello, asking questions, etc.), call the funct DO NOT say anything until you've determined if this is a voicemail or human.""" llm = GoogleLLMService( - model="models/gemini-2.0-flash-lite-preview-02-05", + model="models/gemini-2.0-flash-lite", api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, tools=tools, From 4f8b036abec64dcc54a44ff6252dbc18fee09cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 22:28:21 -0800 Subject: [PATCH 3/3] pyproject: remote httpx old dependency and upgrade anthropic/google-genai --- pyproject.toml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 768c9c816..1abfff495 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,6 @@ classifiers = [ dependencies = [ "aiohttp~=3.11.11", "audioop-lts~=0.2.1; python_version>='3.13'", - # We need an older version of `httpx` that doesn't remove the deprecated - # `proxies` argument. This is necessary for Azure and Anthropic clients. - "httpx~=0.27.2", "loguru~=0.7.3", "Markdown~=3.7", "numpy~=1.26.4", @@ -42,7 +39,7 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -anthropic = [ "anthropic~=0.45.2" ] +anthropic = [ "anthropic~=0.47.2" ] assemblyai = [ "assemblyai~=0.36.0" ] aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] @@ -56,7 +53,7 @@ elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.6" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] -google = [ "google-cloud-speech~=2.31.0", "google-cloud-texttospeech~=2.25.0", "google-genai~=1.2.0", "google-generativeai~=0.8.4" ] +google = [ "google-cloud-speech~=2.31.0", "google-cloud-texttospeech~=2.25.0", "google-genai~=1.3.0", "google-generativeai~=0.8.4" ] grok = [] groq = [] gstreamer = [ "pygobject~=3.50.0" ]