From 184f2cdb556a1553fcf7c8bc8c0fe9f5ebbffa3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 31 Oct 2024 15:47:32 -0700 Subject: [PATCH] examples: add bot background sound example --- CHANGELOG.md | 2 + .../11a-sound-effects-background-twilio.py | 100 ---------------- .../foundational/23-bot-background-sound.py | 109 ++++++++++++++++++ 3 files changed, 111 insertions(+), 100 deletions(-) delete mode 100644 examples/foundational/11a-sound-effects-background-twilio.py create mode 100644 examples/foundational/23-bot-background-sound.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c8fbf1515..05e393250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Add `23-bot-background-sound.py` foundational example. + - Added a new foundational example 22-natural-conversation.py. This examples shows how to achieve a more natural conversation detecting when the user ends statement. diff --git a/examples/foundational/11a-sound-effects-background-twilio.py b/examples/foundational/11a-sound-effects-background-twilio.py deleted file mode 100644 index 79e7aea2c..000000000 --- a/examples/foundational/11a-sound-effects-background-twilio.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) 2024, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os -import sys - -from pipecat.frames.frames import EndFrame, LLMMessagesFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantResponseAggregator, - LLMUserResponseAggregator, -) -from pipecat.services.background_noise import BackgroundNoiseEffect -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.openai import OpenAILLMService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.transports.network.fastapi_websocket import ( - FastAPIWebsocketTransport, - FastAPIWebsocketParams, -) -from pipecat.vad.silero import SileroVADAnalyzer -from pipecat.serializers.twilio import TwilioFrameSerializer - -from loguru import logger - -from dotenv import load_dotenv - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def run_bot(websocket_client, stream_sid): - transport = FastAPIWebsocketTransport( - websocket=websocket_client, - params=FastAPIWebsocketParams( - audio_out_enabled=True, - add_wav_header=False, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - serializer=TwilioFrameSerializer(stream_sid), - ), - ) - - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in an audio call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - tma_in = LLMUserResponseAggregator(messages) - tma_out = LLMAssistantResponseAggregator(messages) - background_noise = BackgroundNoiseEffect(websocket_client, stream_sid, "your_path_to_audio_in_format_pcm16000") - - pipeline = Pipeline( - [ - transport.input(), # Websocket input from client - stt, # Speech-To-Text - tma_in, # User responses - llm, # LLM - tts, # Text-To-Speech - background_noise, - transport.output(), # Websocket output to client - tma_out, # LLM responses - ] - ) - - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([LLMMessagesFrame(messages)]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - await task.queue_frames([EndFrame()]) - - runner = PipelineRunner(handle_sigint=False) - - await runner.run(task) - diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py new file mode 100644 index 000000000..8623e6ab9 --- /dev/null +++ b/examples/foundational/23-bot-background-sound.py @@ -0,0 +1,109 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import asyncio +import aiohttp +import os +import sys + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.audio.background_sound import BotBackgroundSound +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +from runner import configure_with_args + +from loguru import logger + +from dotenv import load_dotenv + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + parser = argparse.ArgumentParser(description="Bot Background Sound") + parser.add_argument("-i", "--input", type=str, required=True, help="Input audio file") + + (room_url, token, args) = await configure_with_args(session, parser) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + background_sound = BotBackgroundSound(file_name=args.input) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + background_sound, # Bot background sound + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main())