From 7634058f97d55cedd679f0ad8d7c6d6a2b5c1676 Mon Sep 17 00:00:00 2001 From: Mert Sefa AKGUN Date: Thu, 9 Jan 2025 02:18:52 +0300 Subject: [PATCH 1/4] fix(examples): correct TTS service import and setup - Update import to use CartesiaTTSService instead of CartesiaMultiLingualTTSService. - Adjust GeminiMultimodalLiveLLMService setup to use set_model_modalities with TEXT modality. --- .../foundational/26d-gemini-multimodal-live-text.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index 760af39ce..deca5dbf1 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -9,7 +9,6 @@ import os import sys import aiohttp -from agent.services.tts.cartesia_multilingual import CartesiaMultiLingualTTSService from dotenv import load_dotenv from loguru import logger from runner import configure @@ -19,7 +18,11 @@ from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.gemini_multimodal_live.gemini import ( + GeminiMultimodalLiveLLMService, + GeminiMultimodalModalities, +) from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -54,9 +57,11 @@ async def main(): api_key=os.getenv("GOOGLE_API_KEY"), # system_instruction="Talk like a pirate." ) - llm.set_model_only_text() # This forces model to produce text only responses + llm.set_model_modalities( + GeminiMultimodalModalities.TEXT + ) # This forces model to produce text only responses - tts = CartesiaMultiLingualTTSService(api_key=os.getenv("CARTESIA_API_KEY")) + tts = CartesiaTTSService(api_key=os.getenv("CARTESIA_API_KEY")) pipeline = Pipeline( [ From 6664c492ac6019631b15c1fb8724bf577fc42719 Mon Sep 17 00:00:00 2001 From: Mert Sefa AKGUN Date: Thu, 9 Jan 2025 15:38:33 +0300 Subject: [PATCH 2/4] feat(gemini): enable audio transcription in live text example Add options to transcribe both user and model audio during the GeminiMultimodalLiveLLMService setup in the 26d-gemini-multimodal-live-text.py example. --- examples/foundational/26d-gemini-multimodal-live-text.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index deca5dbf1..f70ba4eca 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -56,6 +56,8 @@ async def main(): llm = GeminiMultimodalLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), # system_instruction="Talk like a pirate." + transcribe_user_audio=True, + transcribe_model_audio=True, ) llm.set_model_modalities( GeminiMultimodalModalities.TEXT From 21c274944eed53bea6f5350df53cf7db4bfd3f6a Mon Sep 17 00:00:00 2001 From: Mert Sefa AKGUN Date: Fri, 10 Jan 2025 12:28:13 +0300 Subject: [PATCH 3/4] Update examples/foundational/26d-gemini-multimodal-live-text.py Co-authored-by: Vanessa Pyne --- examples/foundational/26d-gemini-multimodal-live-text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index f70ba4eca..06ba825d6 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -63,7 +63,7 @@ async def main(): GeminiMultimodalModalities.TEXT ) # This forces model to produce text only responses - tts = CartesiaTTSService(api_key=os.getenv("CARTESIA_API_KEY")) + tts = CartesiaTTSService(api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22") pipeline = Pipeline( [ From 67af4e619b2c19117464d8556b7e129872df96b1 Mon Sep 17 00:00:00 2001 From: Mert Sefa AKGUN Date: Fri, 10 Jan 2025 12:32:24 +0300 Subject: [PATCH 4/4] style(examples): fix ruff formatting in Gemini text example Refactor `CartesiaTTSService` instantiation to comply with line length requirements from the ruff linter. --- examples/foundational/26d-gemini-multimodal-live-text.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index 06ba825d6..493c2983a 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -63,7 +63,9 @@ async def main(): GeminiMultimodalModalities.TEXT ) # This forces model to produce text only responses - tts = CartesiaTTSService(api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22") + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + ) pipeline = Pipeline( [