From 1a662376fc87cd04fdeaa666c553876d83b09b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 19 Aug 2024 15:31:34 -0700 Subject: [PATCH] examples: use Cartesia TTS in most examples --- CHANGELOG.md | 2 ++ examples/foundational/01-say-one-thing.py | 13 +++++------ examples/foundational/01a-local-audio.py | 13 +++++------ examples/foundational/02-llm-say-one-thing.py | 13 +++++------ .../foundational/03a-local-still-frame.py | 2 +- .../foundational/05-sync-speech-and-image.py | 3 --- .../05a-local-sync-speech-and-image.py | 7 ++---- .../foundational/06-listen-and-respond.py | 10 ++++---- examples/foundational/07-interruptible.py | 9 ++++---- .../07a-interruptible-anthropic.py | 9 ++++---- .../07b-interruptible-langchain.py | 8 +++---- .../07h-interruptible-openpipe.py | 9 ++++---- .../foundational/07j-interruptible-gladia.py | 10 ++++---- examples/foundational/10-wake-phrase.py | 9 ++++---- examples/foundational/12-describe-video.py | 15 ++++-------- .../12a-describe-video-gemini-flash.py | 9 ++++---- .../foundational/12b-describe-video-gpt-4o.py | 9 ++++---- examples/foundational/14-function-calling.py | 9 ++++---- examples/foundational/15a-switch-languages.py | 17 ++++++-------- examples/foundational/17-detect-user-idle.py | 10 ++++---- examples/foundational/19a-tools-anthropic.py | 6 +---- .../foundational/19b-tools-video-anthropic.py | 10 ++------ examples/foundational/19c-tools-togetherai.py | 9 ++------ examples/moondream-chatbot/bot.py | 9 ++++---- examples/patient-intake/bot.py | 23 ++++++++----------- examples/simple-chatbot/bot.py | 2 +- examples/twilio-chatbot/bot.py | 9 ++++---- examples/websocket-server/bot.py | 9 ++++---- src/pipecat/pipeline/parallel_task.py | 11 +++++---- src/pipecat/processors/aggregators/gated.py | 19 ++++++++++----- 30 files changed, 124 insertions(+), 169 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7057d0821..9f4417350 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Most examples now use Cartesia. + - Added examples `foundational/19a-tools-anthropic.py`, `foundational/19b-tools-video-anthropic.py` and `foundational/19a-tools-togetherai.py`. diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index f7330e1c1..8102518f7 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -9,11 +9,11 @@ import aiohttp import os import sys -from pipecat.frames.frames import EndFrame, TextFrame +from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.runner import PipelineRunner -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport from runner import configure @@ -34,10 +34,9 @@ async def main(): transport = DailyTransport( room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True)) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) runner = PipelineRunner() @@ -49,7 +48,7 @@ async def main(): @transport.event_handler("on_participant_joined") async def on_new_participant_joined(transport, participant): participant_name = participant["info"]["userName"] or '' - await task.queue_frames([TextFrame(f"Hello there, {participant_name}!"), EndFrame()]) + await task.queue_frame(TextFrame(f"Hello there, {participant_name}!")) await runner.run(task) diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index 789a98a80..df63bca99 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -9,11 +9,11 @@ import aiohttp import os import sys -from pipecat.frames.frames import EndFrame, TextFrame +from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.audio import LocalAudioTransport @@ -30,10 +30,9 @@ async def main(): async with aiohttp.ClientSession() as session: transport = LocalAudioTransport(TransportParams(audio_out_enabled=True)) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) pipeline = Pipeline([tts, transport.output()]) @@ -42,7 +41,7 @@ async def main(): async def say_something(): await asyncio.sleep(1) - await task.queue_frames([TextFrame("Hello there!"), EndFrame()]) + await task.queue_frame(TextFrame("Hello there!")) runner = PipelineRunner() diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index 14086ad2e..e53aee9ae 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -9,11 +9,11 @@ import aiohttp import os import sys -from pipecat.frames.frames import EndFrame, LLMMessagesFrame +from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -38,10 +38,9 @@ async def main(): "Say One Thing From an LLM", DailyParams(audio_out_enabled=True)) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( @@ -60,7 +59,7 @@ async def main(): @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): - await task.queue_frames([LLMMessagesFrame(messages), EndFrame()]) + await task.queue_frame(LLMMessagesFrame(messages)) await runner.run(task) diff --git a/examples/foundational/03a-local-still-frame.py b/examples/foundational/03a-local-still-frame.py index d645f8b95..14e092508 100644 --- a/examples/foundational/03a-local-still-frame.py +++ b/examples/foundational/03a-local-still-frame.py @@ -56,7 +56,7 @@ async def main(): runner = PipelineRunner() async def run_tk(): - while runner.is_active(): + while not task.has_finished(): tk_root.update() tk_root.update_idletasks() await asyncio.sleep(0.1) diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 1eb05b96b..e3965e857 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -13,7 +13,6 @@ from dataclasses import dataclass from pipecat.frames.frames import ( AppFrame, - EndFrame, Frame, ImageRawFrame, LLMFullResponseStartFrame, @@ -152,8 +151,6 @@ async def main(): frames.append(MonthFrame(month=month)) frames.append(LLMMessagesFrame(messages)) - frames.append(EndFrame()) - runner = PipelineRunner() task = PipelineTask(pipeline) diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 7db6dc99f..5decffcb5 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -137,14 +137,11 @@ async def main(): task = PipelineTask(pipeline) - # We only specify 5 months as we create tasks all at once and we might - # get rate limited otherwise. + # We only specify a few months as we create tasks all at once and we + # might get rate limited otherwise. months: list[str] = [ "January", "February", - # "March", - # "April", - # "May", ] # We create one task per month. This will be executed concurrently. diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index b659615e0..35965185f 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -18,8 +18,7 @@ from pipecat.processors.aggregators.llm_response import ( LLMUserResponseAggregator, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.processors.logger import FrameLogger -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -59,10 +58,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 6aab95b31..90c10c76d 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -15,7 +15,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator) -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -47,10 +47,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index 6156b3a48..a8d90f087 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -15,7 +15,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator) -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -47,10 +47,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 6e691f62c..c517ff27a 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -17,6 +17,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator) from pipecat.processors.frameworks.langchain import LangchainProcessor +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -63,10 +64,9 @@ async def main(): ), ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) prompt = ChatPromptTemplate.from_messages( diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index e48f0517f..489015f21 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -17,7 +17,7 @@ from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator, ) -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openpipe import OpenPipeLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -50,10 +50,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) timestamp = int(time.time()) diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 23025dcfd..aff975e29 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -15,10 +15,9 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator) -from pipecat.services.deepgram import DeepgramSTTService, DeepgramTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.gladia import GladiaSTTService from pipecat.services.openai import OpenAILLMService -from pipecat.services.xtts import XTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -53,10 +52,9 @@ async def main(): api_key=os.getenv("GLADIA_API_KEY"), ) - tts = DeepgramTTSService( - aiohttp_session=session, - api_key=os.getenv("DEEPGRAM_API_KEY"), - voice="aura-helios-en" + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index 6002d8f09..6e9e106b8 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -15,7 +15,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator) -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -47,10 +47,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index f574aa50f..11240e8de 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -16,7 +16,7 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.moondream import MoondreamService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -65,12 +65,6 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), - ) - user_response = UserResponseAggregator() image_requester = UserImageRequester() @@ -80,10 +74,9 @@ async def main(): # If you run into weird description, try with use_cpu=True moondream = MoondreamService() - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index d399e0814..395abdbdc 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -16,7 +16,7 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.google import GoogleLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -76,10 +76,9 @@ async def main(): model="gemini-1.5-flash-latest", api_key=os.getenv("GOOGLE_API_KEY")) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index dc5bb4846..384c9aa0c 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -16,7 +16,7 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -76,10 +76,9 @@ async def main(): model="gpt-4o" ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 1cdd364c6..e4bdd5797 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -14,7 +14,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.logger import FrameLogger -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -56,10 +56,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index 811cbbaeb..0dde985ef 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -16,7 +16,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.filters.function_filter import FunctionFilter -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.services.whisper import Model, WhisperSTTService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -70,17 +70,14 @@ async def main(): stt = WhisperSTTService(model=Model.LARGE) - english_tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id="pNInz6obpgDQGcFmaJgB", + english_tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) - spanish_tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - model="eleven_multilingual_v2", - voice_id="9F4C8ztpNUmXkdDDbz3J", + spanish_tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="846d6cb0-2301-48b6-9683-48f5618ea2f6", # Spanish-speaking Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 80bdbb32e..903f35f4e 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -15,9 +15,8 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator) -from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.user_idle_processor import UserIdleProcessor -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -49,10 +48,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/19a-tools-anthropic.py b/examples/foundational/19a-tools-anthropic.py index 728265cbb..4cf42c2a2 100644 --- a/examples/foundational/19a-tools-anthropic.py +++ b/examples/foundational/19a-tools-anthropic.py @@ -12,15 +12,12 @@ import sys from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia import CartesiaTTSService - from pipecat.services.anthropic import AnthropicLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext - - from runner import configure from loguru import logger @@ -56,7 +53,6 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady - sample_rate=16000, ) llm = AnthropicLLMService( diff --git a/examples/foundational/19b-tools-video-anthropic.py b/examples/foundational/19b-tools-video-anthropic.py index 9721ad2ae..d9446d8e2 100644 --- a/examples/foundational/19b-tools-video-anthropic.py +++ b/examples/foundational/19b-tools-video-anthropic.py @@ -9,20 +9,15 @@ import aiohttp import os import sys -from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia import CartesiaTTSService - -from pipecat.services.anthropic import AnthropicLLMService, AnthropicUserContextAggregator, AnthropicAssistantContextAggregator +from pipecat.services.anthropic import AnthropicLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor - - from runner import configure from loguru import logger @@ -67,7 +62,6 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady - sample_rate=16000, ) llm = AnthropicLLMService( diff --git a/examples/foundational/19c-tools-togetherai.py b/examples/foundational/19c-tools-togetherai.py index c9a633b47..329ecce68 100644 --- a/examples/foundational/19c-tools-togetherai.py +++ b/examples/foundational/19c-tools-togetherai.py @@ -14,16 +14,12 @@ from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia import CartesiaTTSService - -from pipecat.services.together import TogetherLLMService, TogetherContextAggregatorPair +from pipecat.services.together import TogetherLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor - - from runner import configure from loguru import logger @@ -66,7 +62,6 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady - sample_rate=16000, ) llm = TogetherLLMService( diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index e29ce43e7..32bbf7e6b 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -31,7 +31,7 @@ from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregato from pipecat.processors.aggregators.sentence import SentenceAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.moondream import MoondreamService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -153,10 +153,9 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id="pNInz6obpgDQGcFmaJgB", + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) llm = OpenAILLMService( diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index bfa1de9b8..7dc404c52 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -16,7 +16,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.logger import FrameLogger from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -263,21 +263,16 @@ async def main(): ) ) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - # - # English - # - voice_id="pNInz6obpgDQGcFmaJgB", - - # - # Spanish - # - # model="eleven_multilingual_v2", - # voice_id="gD1IexrzCvsXPHUuT0s3", + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) + # tts = CartesiaTTSService( + # api_key=os.getenv("CARTESIA_API_KEY"), + # voice_id="846d6cb0-2301-48b6-9683-48f5618ea2f6", # Spanish-speaking Lady + # ) + llm = OpenAILLMService( api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/simple-chatbot/bot.py b/examples/simple-chatbot/bot.py index bb4381eba..d00f6acd1 100644 --- a/examples/simple-chatbot/bot.py +++ b/examples/simple-chatbot/bot.py @@ -26,7 +26,7 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTranscriptionSettings, DailyTransport +from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer from runner import configure diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index 7d523ca43..b7376e084 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -10,9 +10,9 @@ from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator ) +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.services.deepgram import DeepgramSTTService -from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketTransport, FastAPIWebsocketParams from pipecat.vad.silero import SileroVADAnalyzer from pipecat.serializers.twilio import TwilioFrameSerializer @@ -46,10 +46,9 @@ async def run_bot(websocket_client, stream_sid): stt = DeepgramSTTService(api_key=os.getenv('DEEPGRAM_API_KEY')) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) messages = [ diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index b05d644e3..29a99614f 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -17,8 +17,8 @@ from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator ) +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.deepgram import DeepgramSTTService -from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.network.websocket_server import WebsocketServerParams, WebsocketServerTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -50,10 +50,9 @@ async def main(): stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = ElevenLabsTTSService( - aiohttp_session=session, - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) messages = [ diff --git a/src/pipecat/pipeline/parallel_task.py b/src/pipecat/pipeline/parallel_task.py index 306a772b7..724183b3b 100644 --- a/src/pipecat/pipeline/parallel_task.py +++ b/src/pipecat/pipeline/parallel_task.py @@ -57,6 +57,7 @@ class ParallelTask(BasePipeline): raise Exception(f"ParallelTask needs at least one argument") self._sinks = [] + self._sources = [] self._pipelines = [] self._up_queue = asyncio.Queue() @@ -70,10 +71,10 @@ class ParallelTask(BasePipeline): # We add a source at the beginning of the pipeline and a sink at the end. source = Source(self._up_queue) sink = Sink(self._down_queue) - processors: List[FrameProcessor] = [source] + processors - processors.append(sink) + processors: List[FrameProcessor] = [source] + processors + [sink] - # Keep track of sinks. We access the source through the pipeline. + # Keep track of sources and sinks. + self._sources.append(source) self._sinks.append(sink) # Create pipeline @@ -99,8 +100,8 @@ class ParallelTask(BasePipeline): # If we get an upstream frame we process it in each sink. await asyncio.gather(*[s.process_frame(frame, direction) for s in self._sinks]) elif direction == FrameDirection.DOWNSTREAM: - # If we get a downstream frame we process it in each source (using the pipeline). - await asyncio.gather(*[p.process_frame(frame, direction) for p in self._pipelines]) + # If we get a downstream frame we process it in each source. + await asyncio.gather(*[s.process_frame(frame, direction) for s in self._sources]) seen_ids = set() while not self._up_queue.empty(): diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index 16a7e3076..aaeedb592 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import List +from typing import List, Tuple from pipecat.frames.frames import Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -40,12 +40,14 @@ class GatedAggregator(FrameProcessor): Goodbye. """ - def __init__(self, gate_open_fn, gate_close_fn, start_open): + def __init__(self, gate_open_fn, gate_close_fn, start_open, + direction: FrameDirection = FrameDirection.DOWNSTREAM): super().__init__() self._gate_open_fn = gate_open_fn self._gate_close_fn = gate_close_fn self._gate_open = start_open - self._accumulator: List[Frame] = [] + self._direction = direction + self._accumulator: List[Tuple[Frame, FrameDirection]] = [] async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -55,6 +57,11 @@ class GatedAggregator(FrameProcessor): await self.push_frame(frame, direction) return + # Ignore frames that are not following the direction of this gate. + if direction != self._direction: + await self.push_frame(frame, direction) + return + old_state = self._gate_open if self._gate_open: self._gate_open = not self._gate_close_fn(frame) @@ -67,8 +74,8 @@ class GatedAggregator(FrameProcessor): if self._gate_open: await self.push_frame(frame, direction) - for frame in self._accumulator: - await self.push_frame(frame, direction) + for (f, d) in self._accumulator: + await self.push_frame(f, d) self._accumulator = [] else: - self._accumulator.append(frame) + self._accumulator.append((frame, direction))