diff --git a/examples/foundational/01-say-one-thing-piper.py b/examples/foundational/01-say-one-thing-piper.py index 0ae6b4b66..209f8805f 100644 --- a/examples/foundational/01-say-one-thing-piper.py +++ b/examples/foundational/01-say-one-thing-piper.py @@ -4,54 +4,54 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.piper.tts import PiperTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_out_enabled=True, + ), + ) + + # Create an HTTP session async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) - - transport = DailyTransport( - room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True) - ) - tts = PiperTTSService( base_url=os.getenv("PIPER_BASE_URL"), aiohttp_session=session, sample_rate=24000 ) - runner = PipelineRunner() - task = PipelineTask(Pipeline([tts, transport.output()])) - # Register an event handler so we can play the audio when the - # participant joins. - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await task.queue_frames( - [TTSSpeakFrame(f"Hello there, how are you today ?"), EndFrame()] - ) + # Register an event handler so we can play the audio when the client joins + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) + + runner = PipelineRunner(handle_sigint=False) await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/01-say-one-thing-rime.py b/examples/foundational/01-say-one-thing-rime.py new file mode 100644 index 000000000..b13e28f45 --- /dev/null +++ b/examples/foundational/01-say-one-thing-rime.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.frames.frames import EndFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.services.rime.tts import RimeHttpTTSService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + + +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_out_enabled=True, + ), + ) + + # Create an HTTP session + async with aiohttp.ClientSession() as session: + tts = RimeHttpTTSService( + api_key=os.getenv("RIME_API_KEY", ""), + voice_id="rex", + aiohttp_session=session, + ) + + task = PipelineTask(Pipeline([tts, transport.output()])) + + # Register an event handler so we can play the audio when the client joins + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index b21a2bf83..14bbe6a44 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -4,56 +4,52 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_out_enabled=True, + ), + ) - transport = DailyTransport( - room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True) - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + task = PipelineTask(Pipeline([tts, transport.output()])) - runner = PipelineRunner() + # Register an event handler so we can play the audio when the client joins + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) - task = PipelineTask(Pipeline([tts, transport.output()])) + runner = PipelineRunner(handle_sigint=False) - # Register an event handler so we can play the audio when the - # participant joins. - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - participant_name = participant.get("info", {}).get("userName", "") - await task.queue_frames( - [TTSSpeakFrame(f"Hello there, {participant_name}!"), EndFrame()] - ) - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/01c-fastpitch.py b/examples/foundational/01c-fastpitch.py index 919ea0481..f3823637d 100644 --- a/examples/foundational/01c-fastpitch.py +++ b/examples/foundational/01c-fastpitch.py @@ -4,51 +4,49 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.riva.tts import FastPitchTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_out_enabled=True, + ), + ) - transport = DailyTransport( - room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True) - ) + tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) - tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + task = PipelineTask(Pipeline([tts, transport.output()])) - runner = PipelineRunner() + # Register an event handler so we can play the audio when the client joins + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) - task = PipelineTask(Pipeline([tts, transport.output()])) + runner = PipelineRunner(handle_sigint=False) - # Register an event handler so we can play the audio when the - # participant joins. - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - participant_name = participant.get("info", {}).get("userName", "") - await task.queue_frames([TTSSpeakFrame(f"Aloha, {participant_name}!"), EndFrame()]) - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index 326ea388a..510e9291d 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,46 +15,51 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_out_enabled=True, + ), + ) - transport = DailyTransport( - room_url, None, "Say One Thing From an LLM", DailyParams(audio_out_enabled=True) - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.", + } + ] - messages = [ - { - "role": "system", - "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.", - } - ] + task = PipelineTask(Pipeline([llm, tts, transport.output()])) - runner = PipelineRunner() + # Register an event handler so we can play the audio when the client joins + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + await task.queue_frames([LLMMessagesFrame(messages), EndFrame()]) - task = PipelineTask(Pipeline([llm, tts, transport.output()])) + runner = PipelineRunner(handle_sigint=False) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await task.queue_frames([LLMMessagesFrame(messages), EndFrame()]) - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/03-still-frame.py b/examples/foundational/03-still-frame.py index f758a3228..5a4d9e609 100644 --- a/examples/foundational/03-still-frame.py +++ b/examples/foundational/03-still-frame.py @@ -4,59 +4,67 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.fal.image import FalImageGenService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + camera_out_enabled=True, + camera_out_width=1024, + camera_out_height=1024, + ), + ) + + # Create an HTTP session async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Show a still frame image", - DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024), - ) - imagegen = FalImageGenService( params=FalImageGenService.InputParams(image_size="square_hd"), aiohttp_session=session, key=os.getenv("FAL_KEY"), ) - runner = PipelineRunner() - task = PipelineTask(Pipeline([imagegen, transport.output()])) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): + # Register an event handler so we can play the audio when the client joins + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): await task.queue_frame(TextFrame("a cat in the style of picasso")) - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") await task.cancel() + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/03b-still-frame-imagen.py b/examples/foundational/03b-still-frame-imagen.py index 3968be5c2..a9ae9ab53 100644 --- a/examples/foundational/03b-still-frame-imagen.py +++ b/examples/foundational/03b-still-frame-imagen.py @@ -4,62 +4,67 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure -from pipecat.frames.frames import EndFrame, TextFrame +from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.google.image import GoogleImageGenService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + camera_out_enabled=True, + camera_out_width=1024, + camera_out_height=1024, + ), + ) - transport = DailyTransport( - room_url, - None, - "Show a still frame image", - DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024), - ) + imagegen = GoogleImageGenService( + api_key=os.getenv("GOOGLE_API_KEY"), + ) - imagegen = GoogleImageGenService( - api_key=os.getenv("GOOGLE_API_KEY"), - ) + task = PipelineTask( + Pipeline([imagegen, transport.output()]), + params=PipelineParams(enable_metrics=True), + ) - runner = PipelineRunner() + # Register an event handler so we can play the audio when the client joins + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + await task.queue_frame(TextFrame("a cat in the style of picasso")) + await task.queue_frame(TextFrame("a dog in the style of picasso")) + await task.queue_frame(TextFrame("a fish in the style of picasso")) - task = PipelineTask( - Pipeline([imagegen, transport.output()]), - params=PipelineParams(enable_metrics=True), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await task.queue_frame(TextFrame("a cat in the style of picasso")) - await task.queue_frame(TextFrame("a dog in the style of picasso")) - await task.queue_frame(TextFrame("a fish in the style of picasso")) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.queue_frame(EndFrame()) + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/04-utterance-and-speech.py b/examples/foundational/04-utterance-and-speech.py index f2184b58f..f72bf5539 100644 --- a/examples/foundational/04-utterance-and-speech.py +++ b/examples/foundational/04-utterance-and-speech.py @@ -13,9 +13,9 @@ import os import sys import aiohttp +from daily_runner import configure from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame from pipecat.pipeline.merge_pipeline import SequentialMergePipeline diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 4e0202d3d..6778b6946 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -4,15 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from dataclasses import dataclass import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import ( DataFrame, @@ -30,13 +27,12 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaHttpTTSService from pipecat.services.fal.image import FalImageGenService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - @dataclass class MonthFrame(DataFrame): @@ -67,22 +63,28 @@ class MonthPrepender(FrameProcessor): await self.push_frame(frame, direction) -async def main(): +async def run_bot(webrtc_connection: SmallWebRTCConnection): + """Run the Calendar Month Narration bot using WebRTC transport. + + Args: + webrtc_connection: The WebRTC connection to use + room_name: Optional room name for display purposes + """ + logger.info(f"Starting bot") + + # Create a transport using the WebRTC connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_out_enabled=True, + camera_out_enabled=True, + camera_out_width=1024, + camera_out_height=1024, + ), + ) + + # Create an HTTP session for API calls async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Month Narration Bot", - DailyParams( - audio_out_enabled=True, - camera_out_enabled=True, - camera_out_width=1024, - camera_out_height=1024, - ), - ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") tts = CartesiaHttpTTSService( @@ -144,14 +146,30 @@ async def main(): frames.append(MonthFrame(month=month)) frames.append(LLMMessagesFrame(messages)) - runner = PipelineRunner() - task = PipelineTask(pipeline) - await task.queue_frames(frames) + # Set up transport event handlers + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start the month narration once connected + await task.queue_frames(frames) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + # Run the pipeline + runner = PipelineRunner(handle_sigint=False) await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index bc7160104..6df1b82b8 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, MetricsFrame @@ -27,14 +23,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class MetricsLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -56,76 +52,83 @@ class MetricsLogger(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - ml = MetricsLogger() + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + ml = MetricsLogger() + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + ml, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - ml, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index e8d21931c..1117ae940 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger from PIL import Image -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( @@ -20,7 +16,6 @@ from pipecat.frames.frames import ( BotStoppedSpeakingFrame, Frame, OutputImageRawFrame, - TextFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -28,14 +23,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class ImageSyncAggregator(FrameProcessor): def __init__(self, speaking_path: str, waiting_path: str): @@ -72,83 +67,90 @@ class ImageSyncAggregator(FrameProcessor): await self.push_frame(frame) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - camera_out_enabled=True, - camera_out_width=1024, - camera_out_height=1024, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_out_enabled=True, + camera_out_width=1024, + camera_out_height=1024, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + image_sync_aggregator = ImageSyncAggregator( + os.path.join(os.path.dirname(__file__), "assets", "speaking.png"), + os.path.join(os.path.dirname(__file__), "assets", "waiting.png"), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + image_sync_aggregator, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - image_sync_aggregator = ImageSyncAggregator( - os.path.join(os.path.dirname(__file__), "assets", "speaking.png"), - os.path.join(os.path.dirname(__file__), "assets", "waiting.png"), - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - image_sync_aggregator, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - participant_name = participant.get("info", {}).get("userName", "") - await transport.capture_participant_transcription(participant["id"]) - await task.queue_frames([TextFrame(f"Hi there {participant_name}!")]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py deleted file mode 100644 index 2e340ce40..000000000 --- a/examples/foundational/07-interruptible-vad.py +++ /dev/null @@ -1,104 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.audio.vad.silero import SileroVAD -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - transcription_enabled=True, - ), - ) - - vad = SileroVAD() - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), - vad, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 00589df97..9ec3b0215 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -19,84 +15,92 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py deleted file mode 100644 index 0f135b8fa..000000000 --- a/examples/foundational/07a-interruptible-anthropic.py +++ /dev/null @@ -1,106 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-opus-20240229" - ) - - # todo: think more about how to handle system prompts in a more general way. OpenAI, - # Google, and Anthropic all have slightly different approaches to providing a system - # prompt. - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/07a-interruptible-vad.py b/examples/foundational/07a-interruptible-vad.py new file mode 100644 index 000000000..ab70e5013 --- /dev/null +++ b/examples/foundational/07a-interruptible-vad.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.audio.vad.silero import SileroVAD +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + + +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + vad = SileroVAD() + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + vad, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index ba4f78adc..8f72375c7 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -4,11 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain_community.chat_message_histories import ChatMessageHistory @@ -16,7 +13,6 @@ from langchain_core.chat_history import BaseChatMessageHistory from langchain_core.runnables.history import RunnableWithMessageHistory from langchain_openai import ChatOpenAI from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -29,14 +25,14 @@ from pipecat.processors.aggregators.llm_response import ( ) from pipecat.processors.frameworks.langchain import LangchainProcessor from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - message_store = {} @@ -46,90 +42,97 @@ def get_session_history(session_id: str) -> BaseChatMessageHistory: return message_store[session_id] -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + "Be nice and helpful. Answer very briefly and without special characters like `#` or `*`. " + "Your response will be synthesized to voice and those characters will create unnatural sounds.", ), - ) + MessagesPlaceholder("chat_history"), + ("human", "{input}"), + ] + ) + chain = prompt | ChatOpenAI(model="gpt-4o", temperature=0.7) + history_chain = RunnableWithMessageHistory( + chain, + get_session_history, + history_messages_key="chat_history", + input_messages_key="input", + ) + lc = LangchainProcessor(history_chain) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tma_in = LLMUserResponseAggregator() + tma_out = LLMAssistantResponseAggregator() - prompt = ChatPromptTemplate.from_messages( - [ - ( - "system", - "Be nice and helpful. Answer very briefly and without special characters like `#` or `*`. " - "Your response will be synthesized to voice and those characters will create unnatural sounds.", - ), - MessagesPlaceholder("chat_history"), - ("human", "{input}"), - ] - ) - chain = prompt | ChatOpenAI(model="gpt-4o", temperature=0.7) - history_chain = RunnableWithMessageHistory( - chain, - get_session_history, - history_messages_key="chat_history", - input_messages_key="input", - ) - lc = LangchainProcessor(history_chain) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + tma_in, # User responses + lc, # Langchain + tts, # TTS + transport.output(), # Transport bot output + tma_out, # Assistant spoken responses + ] + ) - tma_in = LLMUserResponseAggregator() - tma_out = LLMAssistantResponseAggregator() + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - tma_in, # User responses - lc, # Langchain - tts, # TTS - transport.output(), # Transport bot output - tma_out, # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + # the `LLMMessagesFrame` will be picked up by the LangchainProcessor using + # only the content of the last message to inject it in the prompt defined + # above. So no role is required here. + messages = [({"content": "Please briefly introduce yourself to the user."})] + await task.queue_frames([LLMMessagesFrame(messages)]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - lc.set_participant_id(participant["id"]) - # Kick off the conversation. - # the `LLMMessagesFrame` will be picked up by the LangchainProcessor using - # only the content of the last message to inject it in the prompt defined - # above. So no role is required here. - messages = [({"content": "Please briefly introduce yourself to the user."})] - await task.queue_frames([LLMMessagesFrame(messages)]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index cdfa78b27..224002752 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from deepgram import LiveOptions from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import ( BotInterruptionFrame, @@ -27,91 +23,95 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + ) - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - ) + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + live_options=LiveOptions(vad_events=True, utterance_end_ms="1000"), + ) - stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - live_options=LiveOptions(vad_events=True, utterance_end_ms="1000"), - ) + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") - tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @stt.event_handler("on_speech_started") + async def on_speech_started(stt, *args, **kwargs): + await task.queue_frames([BotInterruptionFrame(), UserStartedSpeakingFrame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @stt.event_handler("on_utterance_end") + async def on_utterance_end(stt, *args, **kwargs): + await task.queue_frames([StopInterruptionFrame(), UserStoppedSpeakingFrame()]) - @stt.event_handler("on_speech_started") - async def on_speech_started(stt, *args, **kwargs): - await task.queue_frames([BotInterruptionFrame(), UserStartedSpeakingFrame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - @stt.event_handler("on_utterance_end") - async def on_utterance_end(stt, *args, **kwargs): - await task.queue_frames([StopInterruptionFrame(), UserStoppedSpeakingFrame()]) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index 6da37bdbc..7cd1ea140 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -21,82 +17,87 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") - tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index 13a60051a..c28361e8b 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -4,45 +4,44 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + # Create an HTTP session async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = ElevenLabsHttpTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), @@ -65,6 +64,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), # Transport user input + stt, context_aggregator.user(), # User responses llm, # LLM tts, # TTS @@ -83,21 +83,28 @@ async def main(): ), ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([context_aggregator.user().get_context_frame()]) - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index e900b6809..1d51e287b 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -4,99 +4,103 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY", ""), - voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), - ) + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07e-interruptible-playht-http.py b/examples/foundational/07e-interruptible-playht-http.py index a04fefaee..cc4f581ef 100644 --- a/examples/foundational/07e-interruptible-playht-http.py +++ b/examples/foundational/07e-interruptible-playht-http.py @@ -4,100 +4,104 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.playht.tts import PlayHTHttpTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = PlayHTHttpTTSService( - user_id=os.getenv("PLAYHT_USER_ID"), - api_key=os.getenv("PLAYHT_API_KEY"), - voice_url="s3://voice-cloning-zero-shot/d9ff78ba-d016-47f6-b0ef-dd630f59414e/female-cs/manifest.json", - ) + tts = PlayHTHttpTTSService( + user_id=os.getenv("PLAYHT_USER_ID"), + api_key=os.getenv("PLAYHT_API_KEY"), + voice_url="s3://voice-cloning-zero-shot/d9ff78ba-d016-47f6-b0ef-dd630f59414e/female-cs/manifest.json", + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 5539d9df4..cd8d70051 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -4,102 +4,106 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.playht.tts import PlayHTTTSService from pipecat.transcriptions.language import Language -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = PlayHTTTSService( - user_id=os.getenv("PLAYHT_USER_ID"), - api_key=os.getenv("PLAYHT_API_KEY"), - voice_url="s3://voice-cloning-zero-shot/e46b4027-b38d-4d24-b292-38fbca2be0ef/original/manifest.json", - params=PlayHTTTSService.InputParams(language=Language.EN), - ) + tts = PlayHTTTSService( + user_id=os.getenv("PLAYHT_USER_ID"), + api_key=os.getenv("PLAYHT_API_KEY"), + voice_url="s3://voice-cloning-zero-shot/e46b4027-b38d-4d24-b292-38fbca2be0ef/original/manifest.json", + params=PlayHTTTSService.InputParams(language=Language.EN), + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index 184d8eb7d..43eae060a 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -21,93 +17,97 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.azure.llm import AzureLLMService from pipecat.services.azure.stt import AzureSTTService from pipecat.services.azure.tts import AzureTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = AzureSTTService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), + ) - stt = AzureSTTService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), - ) + tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), + ) - tts = AzureTTSService( - api_key=os.getenv("AZURE_SPEECH_API_KEY"), - region=os.getenv("AZURE_SPEECH_REGION"), - ) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL"), + ) - llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), - model=os.getenv("AZURE_CHATGPT_MODEL"), - ) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index e73daa881..a55f249f0 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -21,95 +17,92 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.stt import OpenAISTTService from pipecat.services.openai.tts import OpenAITTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - audio_out_sample_rate=24000, - transcription_enabled=False, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = OpenAISTTService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o-transcribe-latest", + prompt="Expect words related to dogs, such as breed names.", + ) - # You can use the OpenAI compatible API like Groq. - # stt = OpenAISTTService( - # base_url="https://api.groq.com/openai/v1", - # api_key="gsk_***", - # model="whisper-large-v3", - # ) - stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), - model="gpt-4o-transcribe-latest", - prompt="Expect words related to dogs, such as breed names.", - ) + tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), voice="ballad") - tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), voice="ballad") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + audio_out_sample_rate=24000, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 38a71d75a..9335b4a20 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import time -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -20,90 +16,98 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openpipe.llm import OpenPipeLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - timestamp = int(time.time()) - llm = OpenPipeLLMService( - api_key=os.getenv("OPENAI_API_KEY"), - openpipe_api_key=os.getenv("OPENPIPE_API_KEY"), - model="gpt-4o", - tags={"conversation_id": f"pipecat-{timestamp}"}, - ) + timestamp = int(time.time()) + llm = OpenPipeLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + openpipe_api_key=os.getenv("OPENPIPE_API_KEY"), + model="gpt-4o", + tags={"conversation_id": f"pipecat-{timestamp}"}, + ) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 8504261b5..f6aaf80cb 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -4,45 +4,44 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.xtts.tts import XTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + # Create an HTTP session async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = XTTSService( aiohttp_session=session, @@ -65,6 +64,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), # Transport user input + stt, context_aggregator.user(), # User responses llm, # LLM tts, # TTS @@ -83,21 +83,28 @@ async def main(): ), ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([context_aggregator.user().get_context_frame()]) - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 2c7d86a79..3174c4bee 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -23,94 +19,96 @@ from pipecat.services.gladia.config import GladiaInputParams, LanguageConfig from pipecat.services.gladia.stt import GladiaSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transcriptions.language import Language -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = GladiaSTTService( + api_key=os.getenv("GLADIA_API_KEY", ""), + params=GladiaInputParams( + language_config=LanguageConfig( + languages=[Language.EN], + ) + ), + ) - stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY"), - params=GladiaInputParams( - language_config=LanguageConfig( - languages=[Language.EN], - ) - ), - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY", ""), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - # Register an event handler to exit the application when the user leaves. - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index ed08d6ee5..0c9bec12e 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -4,96 +4,100 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.lmnt.tts import LmntTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = LmntTTSService(api_key=os.getenv("LMNT_API_KEY"), voice_id="morgan") + tts = LmntTTSService(api_key=os.getenv("LMNT_API_KEY"), voice_id="morgan") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User respones + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User respones - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07l-interruptible-groq.py b/examples/foundational/07l-interruptible-groq.py new file mode 100644 index 000000000..c6f7960c0 --- /dev/null +++ b/examples/foundational/07l-interruptible-groq.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.groq.llm import GroqLLMService +from pipecat.services.groq.stt import GroqSTTService +from pipecat.services.groq.tts import GroqTTSService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + + +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) + + llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") + + tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py deleted file mode 100644 index 243fbe5f5..000000000 --- a/examples/foundational/07l-interruptible-together.py +++ /dev/null @@ -1,115 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.together.llm import TogetherLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = TogetherLLMService( - api_key=os.getenv("TOGETHER_API_KEY"), - model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - params=TogetherLLMService.InputParams( - temperature=1.0, - top_p=0.9, - top_k=40, - extra={ - "frequency_penalty": 2.0, - "presence_penalty": 0.0, - }, - ), - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond in plain language. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - user_aggregator = context_aggregator.user() - assistant_aggregator = context_aggregator.assistant() - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - user_aggregator, # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - assistant_aggregator, # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/07m-interruptible-polly.py b/examples/foundational/07m-interruptible-polly.py index d5d2ec887..d832a7a06 100644 --- a/examples/foundational/07m-interruptible-polly.py +++ b/examples/foundational/07m-interruptible-polly.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -21,89 +17,93 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.aws.tts import PollyTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = PollyTTSService( + api_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + region=os.getenv("AWS_REGION"), + voice_id="Amy", + params=PollyTTSService.InputParams(engine="neural", language="en-GB", rate="1.05"), + ) - tts = PollyTTSService( - api_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - region=os.getenv("AWS_REGION"), - voice_id="Amy", - params=PollyTTSService.InputParams(engine="neural", language="en-GB", rate="1.05"), - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index 0b974a691..acfaa8cd4 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -22,88 +18,94 @@ from pipecat.services.google.llm import GoogleLLMService from pipecat.services.google.stt import GoogleSTTService from pipecat.services.google.tts import GoogleTTSService from pipecat.transcriptions.language import Language -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = GoogleSTTService( + params=GoogleSTTService.InputParams(languages=Language.EN_US), + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + ) - stt = GoogleSTTService( - params=GoogleSTTService.InputParams(languages=Language.EN_US), - ) + tts = GoogleTTSService( + voice_id="en-US-Chirp3-HD-Charon", + params=GoogleTTSService.InputParams(language=Language.EN_US), + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + ) - tts = GoogleTTSService( - voice_id="en-US-Journey-F", - params=GoogleTTSService.InputParams(language=Language.EN_US), - ) + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User respones + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User respones - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 3c1ac6039..bc132860e 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -21,88 +17,92 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.assemblyai.stt import AssemblyAISTTService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = AssemblyAISTTService( + api_key=os.getenv("ASSEMBLYAI_API_KEY"), + ) - stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 7ca7e2fb6..ab8e4617d 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.filters.krisp_filter import KrispFilter from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -22,83 +18,88 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + audio_in_filter=KrispFilter(), + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - audio_in_filter=KrispFilter(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") - tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07q-interruptible-rime-http.py b/examples/foundational/07q-interruptible-rime-http.py index 3b0c99a21..b6ee068c0 100644 --- a/examples/foundational/07q-interruptible-rime-http.py +++ b/examples/foundational/07q-interruptible-rime-http.py @@ -4,45 +4,44 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.rime.tts import RimeHttpTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + # Create an HTTP session async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = RimeHttpTTSService( api_key=os.getenv("RIME_API_KEY", ""), @@ -65,6 +64,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), # Transport user input + stt, context_aggregator.user(), # User responses llm, # LLM tts, # TTS @@ -83,21 +83,28 @@ async def main(): ), ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([context_aggregator.user().get_context_frame()]) - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index febb30a4e..b297177f3 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -4,99 +4,103 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.rime.tts import RimeTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = RimeTTSService( - api_key=os.getenv("RIME_API_KEY", ""), - voice_id="rex", - ) + tts = RimeTTSService( + api_key=os.getenv("RIME_API_KEY", ""), + voice_id="rex", + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index 411a6ec80..65adbd7e4 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -21,76 +17,87 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.nim.llm import NimLLMService from pipecat.services.riva.stt import ParakeetSTTService from pipecat.services.riva.tts import FastPitchTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY")) - stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY")) + llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct") - llm = NimLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct" - ) + tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) - tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 69bae75fc..92e6f7f7b 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -4,16 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from dataclasses import dataclass -import aiohttp import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( @@ -32,14 +28,15 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameProcessor -from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.google.llm import GoogleLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.google.tts import GoogleTTSService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") marker = "|----|" system_message = f""" @@ -193,85 +190,92 @@ class TanscriptionContextFixup(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - # No transcription at all. just audio input to Gemini! - # transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # No transcription at all. just audio input to Gemini! + # transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") + tts = GoogleTTSService( + voice_id="en-US-Chirp3-HD-Charon", + params=GoogleTTSService.InputParams(language=Language.EN_US), + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), + ) - messages = [ - { - "role": "system", - "content": system_message, - }, - { - "role": "user", - "content": "Start by saying hello.", - }, + messages = [ + { + "role": "system", + "content": system_message, + }, + { + "role": "user", + "content": "Start by saying hello.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + audio_collector = UserAudioCollector(context, context_aggregator.user()) + pull_transcript_out_of_llm_output = TranscriptExtractor(context) + fixup_context_messages = TanscriptionContextFixup(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + audio_collector, + context_aggregator.user(), # User responses + llm, # LLM + pull_transcript_out_of_llm_output, + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + fixup_context_messages, ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - audio_collector = UserAudioCollector(context, context_aggregator.user()) - pull_transcript_out_of_llm_output = TranscriptExtractor(context) - fixup_context_messages = TanscriptionContextFixup(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - audio_collector, - context_aggregator.user(), # User responses - llm, # LLM - pull_transcript_out_of_llm_output, - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - fixup_context_messages, - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index 567e57ecf..db00b1095 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -4,99 +4,103 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.fish.tts import FishAudioTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = FishAudioTTSService( - api_key=os.getenv("FISH_API_KEY"), - model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama - ) + tts = FishAudioTTSService( + api_key=os.getenv("FISH_API_KEY"), + model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py index 0c9821196..966a7b211 100644 --- a/examples/foundational/07u-interruptible-ultravox.py +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -20,7 +16,9 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.ultravox.stt import UltravoxSTTService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) @@ -28,8 +26,6 @@ load_dotenv(override=True) # The Ultravox model is compute-intensive and performs best with GPU acceleration. # This can be deployed on cloud GPU providers like Cerebrium.ai for optimal performance. -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") # Want to initialize the ultravox processor since it takes time to load the model and dont # want to load it every time the pipeline is run @@ -39,53 +35,61 @@ ultravox_processor = UltravoxSTTService( ) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=False, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.environ.get("CARTESIA_API_KEY"), - voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a", - ) + tts = CartesiaTTSService( + api_key=os.environ.get("CARTESIA_API_KEY"), + voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a", + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - ultravox_processor, - tts, # TTS - transport.output(), # Transport bot output - ] - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + ultravox_processor, + tts, # TTS + transport.output(), # Transport bot output + ] + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07v-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py index b89315ec5..66170bcf9 100644 --- a/examples/foundational/07v-interruptible-neuphonic-http.py +++ b/examples/foundational/07v-interruptible-neuphonic-http.py @@ -4,99 +4,103 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.neuphonic.tts import NeuphonicHttpTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = NeuphonicHttpTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), - voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily - ) + tts = NeuphonicHttpTTSService( + api_key=os.getenv("NEUPHONIC_API_KEY"), + voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07v-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py index 002b3f6cc..d72b9327d 100644 --- a/examples/foundational/07v-interruptible-neuphonic.py +++ b/examples/foundational/07v-interruptible-neuphonic.py @@ -4,99 +4,103 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.neuphonic.tts import NeuphonicTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = NeuphonicTTSService( - api_key=os.getenv("NEUPHONIC_API_KEY"), - voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily - ) + tts = NeuphonicTTSService( + api_key=os.getenv("NEUPHONIC_API_KEY"), + voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py index f202d7245..2b8df2aaf 100644 --- a/examples/foundational/07w-interruptible-fal.py +++ b/examples/foundational/07w-interruptible-fal.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -21,89 +17,92 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.fal.stt import FalSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + stt = FalSTTService( + api_key=os.getenv("FAL_KEY"), + ) - stt = FalSTTService( - api_key=os.getenv("FAL_KEY"), - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - # Register an event handler to exit the application when the user leaves. - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/07y-interruptible-groq.py b/examples/foundational/07y-interruptible-groq.py deleted file mode 100644 index f0639412e..000000000 --- a/examples/foundational/07y-interruptible-groq.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.groq.llm import GroqLLMService -from pipecat.services.groq.stt import GroqSTTService -from pipecat.services.groq.tts import GroqTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - # transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) - - stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) - - llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") - - tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py index a0472546e..ffb642d64 100644 --- a/examples/foundational/08-bots-arguing.py +++ b/examples/foundational/08-bots-arguing.py @@ -4,8 +4,8 @@ import os from typing import Tuple import aiohttp +from daily_runner import configure from dotenv import load_dotenv -from runner import configure from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline @@ -72,7 +72,8 @@ async def main(): async def get_text_and_audio(messages) -> Tuple[str, bytearray]: """This function streams text from the LLM and uses the TTS service to convert - that text to speech as it's received.""" + that text to speech as it's received. + """ source_queue = asyncio.Queue() sink_queue = asyncio.Queue() sentence_aggregator = SentenceAggregator() diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index dc060ccc4..46245344f 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -4,13 +4,9 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import ( Frame, @@ -23,13 +19,12 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class MirrorProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -44,6 +39,7 @@ class MirrorProcessor(FrameProcessor): ) ) elif isinstance(frame, InputImageRawFrame): + print(f"Received image frame: {frame.size} {frame.format}") await self.push_frame( OutputImageRawFrame(image=frame.image, size=frame.size, format=frame.format) ) @@ -51,42 +47,48 @@ class MirrorProcessor(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Test", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - camera_out_enabled=True, - camera_out_is_live=True, - camera_out_width=1280, - camera_out_height=720, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, + camera_out_enabled=True, + camera_out_is_live=True, + camera_out_width=1280, + camera_out_height=720, + ), + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_video(participant["id"]) + pipeline = Pipeline([transport.input(), MirrorProcessor(), transport.output()]) - pipeline = Pipeline([transport.input(), MirrorProcessor(), transport.output()]) + task = PipelineTask( + pipeline, + params=PipelineParams(), + ) - runner = PipelineRunner() + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") - task = PipelineTask( - pipeline, - params=PipelineParams( - audio_in_sample_rate=24000, - audio_out_sample_rate=24000, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index eb6739f49..b053beb7a 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -5,13 +5,10 @@ # import asyncio -import sys import tkinter as tk -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import ( Frame, @@ -24,14 +21,13 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class MirrorProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -53,52 +49,59 @@ class MirrorProcessor(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - tk_root = tk.Tk() - tk_root.title("Local Mirror") + p2p_transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, + camera_out_enabled=True, + camera_out_is_live=True, + camera_out_width=1280, + camera_out_height=720, + ), + ) - daily_transport = DailyTransport( - room_url, token, "Test", DailyParams(audio_in_enabled=True) - ) + tk_root = tk.Tk() + tk_root.title("Local Mirror") - tk_transport = TkLocalTransport( - tk_root, - TkTransportParams( - audio_out_enabled=True, - camera_out_enabled=True, - camera_out_is_live=True, - camera_out_width=1280, - camera_out_height=720, - ), - ) + tk_transport = TkLocalTransport( + tk_root, + TkTransportParams( + audio_out_enabled=True, + camera_out_enabled=True, + camera_out_is_live=True, + camera_out_width=1280, + camera_out_height=720, + ), + ) - @daily_transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_video(participant["id"]) + @p2p_transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") - pipeline = Pipeline([daily_transport.input(), MirrorProcessor(), tk_transport.output()]) + pipeline = Pipeline([p2p_transport.input(), MirrorProcessor(), tk_transport.output()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - audio_in_sample_rate=24000, - audio_out_sample_rate=24000, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams(), + ) - async def run_tk(): - while not task.has_finished(): - tk_root.update() - tk_root.update_idletasks() - await asyncio.sleep(0.1) + async def run_tk(): + while not task.has_finished(): + tk_root.update() + tk_root.update_idletasks() + await asyncio.sleep(0.1) - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await asyncio.gather(runner.run(task), run_tk()) + await asyncio.gather(runner.run(task), run_tk()) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index a89d0a0db..24397bfa5 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -4,89 +4,99 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.filters.wake_check_filter import WakeCheckFilter from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Robot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful assistant. Respond to what the user said in a creative and helpful way. Keep your responses brief.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful assistant. Respond to what the user said in a creative and helpful way. Keep your responses brief.", + }, + ] + + hey_robot_filter = WakeCheckFilter(["hey robot", "hey, robot"]) + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + hey_robot_filter, # Filter out speech not directed at the robot + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - hey_robot_filter = WakeCheckFilter(["hey robot", "hey, robot"]) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frame(TTSSpeakFrame("Hi! If you want to talk to me, just say 'Hey Robot'")) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - hey_robot_filter, # Filter out speech not directed at the robot - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.") + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 5c25e4c33..f9d179867 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -4,21 +4,18 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import wave -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( Frame, LLMFullResponseEndFrame, OutputAudioRawFrame, + TTSSpeakFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -30,14 +27,14 @@ from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.logger import FrameLogger from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - sounds = {} sound_files = ["ding1.wav", "ding2.wav"] @@ -80,70 +77,83 @@ class InboundSoundEffectWrapper(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.", - }, + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + out_sound = OutboundSoundEffectWrapper() + in_sound = InboundSoundEffectWrapper() + fl = FrameLogger("LLM Out") + fl2 = FrameLogger("Transcription In") + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + in_sound, + fl2, + llm, + fl, + tts, + out_sound, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - out_sound = OutboundSoundEffectWrapper() - in_sound = InboundSoundEffectWrapper() - fl = FrameLogger("LLM Out") - fl2 = FrameLogger("Transcription In") + task = PipelineTask(pipeline) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - in_sound, - fl2, - llm, - fl, - tts, - out_sound, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frame(TTSSpeakFrame("Hi, I'm listening!")) + await transport.send_audio(sounds["ding1.wav"]) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await tts.say("Hi, I'm listening!") - await transport.send_audio(sounds["ding1.wav"]) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - runner = PipelineRunner() + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - task = PipelineTask(pipeline) + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index a0977167d..ce614dada 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from typing import Optional -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame @@ -23,14 +19,14 @@ from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.moondream.vision import MoondreamService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class UserImageRequester(FrameProcessor): def __init__(self, participant_id: Optional[str] = None): @@ -50,61 +46,81 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + # Get WebRTC peer connection ID + webrtc_peer_id = webrtc_connection.pc_id - transport = DailyTransport( - room_url, - token, - "Describe participant video", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - user_response = UserResponseAggregator() + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - image_requester = UserImageRequester() + user_response = UserResponseAggregator() - vision_aggregator = VisionImageFrameAggregator() + # Initialize the image requester without setting the participant ID yet + image_requester = UserImageRequester() - # If you run into weird description, try with use_cpu=True - moondream = MoondreamService() + vision_aggregator = VisionImageFrameAggregator() - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + # If you run into weird description, try with use_cpu=True + moondream = MoondreamService() - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await tts.say("Hi there! Feel free to ask me what I see.") - await transport.capture_participant_video(participant["id"], framerate=0) - await transport.capture_participant_transcription(participant["id"]) - image_requester.set_participant_id(participant["id"]) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - pipeline = Pipeline( - [ - transport.input(), - user_response, - image_requester, - vision_aggregator, - moondream, - tts, - transport.output(), - ] - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - task = PipelineTask(pipeline) + pipeline = Pipeline( + [ + transport.input(), + stt, + user_response, + image_requester, + vision_aggregator, + moondream, + tts, + transport.output(), + ] + ) - runner = PipelineRunner() + task = PipelineTask(pipeline) - await runner.run(task) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") + + # Set the participant ID in the image requester + image_requester.set_participant_id(webrtc_peer_id) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index 131196e4f..2ed55802a 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -4,33 +4,29 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from typing import Optional -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineTask +from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class UserImageRequester(FrameProcessor): def __init__(self, participant_id: Optional[str] = None): @@ -50,61 +46,84 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + # Get WebRTC peer connection ID + webrtc_peer_id = webrtc_connection.pc_id - transport = DailyTransport( - room_url, - token, - "Describe participant video", - DailyParams( - audio_in_enabled=True, # This is so Silero VAD can get audio data - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - user_response = UserResponseAggregator() + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - image_requester = UserImageRequester() + user_response = UserResponseAggregator() - vision_aggregator = VisionImageFrameAggregator() + # Initialize the image requester without setting the participant ID yet + image_requester = UserImageRequester() - google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) + vision_aggregator = VisionImageFrameAggregator() - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await tts.say("Hi there! Feel free to ask me what I see.") - await transport.capture_participant_video(participant["id"], framerate=0) - await transport.capture_participant_transcription(participant["id"]) - image_requester.set_participant_id(participant["id"]) + # Google Gemini model for vision analysis + google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) - pipeline = Pipeline( - [ - transport.input(), - user_response, - image_requester, - vision_aggregator, - google, - tts, - transport.output(), - ] - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - task = PipelineTask(pipeline) + pipeline = Pipeline( + [ + transport.input(), + stt, + user_response, + image_requester, + vision_aggregator, + google, + tts, + transport.output(), + ] + ) - runner = PipelineRunner() + task = PipelineTask( + pipeline, + params=PipelineParams(allow_interruptions=True), + ) - await runner.run(task) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") + + # Set the participant ID in the image requester + image_requester.set_participant_id(webrtc_peer_id) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index a8eed5a1a..8e1dc99dc 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -4,33 +4,29 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from typing import Optional -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineTask +from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class UserImageRequester(FrameProcessor): def __init__(self, participant_id: Optional[str] = None): @@ -50,60 +46,84 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + # Get WebRTC peer connection ID + webrtc_peer_id = webrtc_connection.pc_id - transport = DailyTransport( - room_url, - token, - "Describe participant video", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - user_response = UserResponseAggregator() + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - image_requester = UserImageRequester() + user_response = UserResponseAggregator() - vision_aggregator = VisionImageFrameAggregator() + # Initialize the image requester without setting the participant ID yet + image_requester = UserImageRequester() - openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + vision_aggregator = VisionImageFrameAggregator() - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await tts.say("Hi there! Feel free to ask me what I see.") - await transport.capture_participant_video(participant["id"], framerate=0) - await transport.capture_participant_transcription(participant["id"]) - image_requester.set_participant_id(participant["id"]) + # OpenAI GPT-4o for vision analysis + openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - pipeline = Pipeline( - [ - transport.input(), - user_response, - image_requester, - vision_aggregator, - openai, - tts, - transport.output(), - ] - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - task = PipelineTask(pipeline) + pipeline = Pipeline( + [ + transport.input(), + stt, + user_response, + image_requester, + vision_aggregator, + openai, + tts, + transport.output(), + ] + ) - runner = PipelineRunner() + task = PipelineTask( + pipeline, + params=PipelineParams(allow_interruptions=True), + ) - await runner.run(task) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") + + # Set the participant ID in the image requester + image_requester.set_participant_id(webrtc_peer_id) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index e53908357..28620bbf1 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -4,33 +4,29 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from typing import Optional -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineTask +from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class UserImageRequester(FrameProcessor): def __init__(self, participant_id: Optional[str] = None): @@ -50,60 +46,84 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + # Get WebRTC peer connection ID + webrtc_peer_id = webrtc_connection.pc_id - transport = DailyTransport( - room_url, - token, - "Describe participant video", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - user_response = UserResponseAggregator() + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - image_requester = UserImageRequester() + user_response = UserResponseAggregator() - vision_aggregator = VisionImageFrameAggregator() + # Initialize the image requester without setting the participant ID yet + image_requester = UserImageRequester() - anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) + vision_aggregator = VisionImageFrameAggregator() - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await tts.say("Hi there! Feel free to ask me what I see.") - await transport.capture_participant_video(participant["id"], framerate=0) - await transport.capture_participant_transcription(participant["id"]) - image_requester.set_participant_id(participant["id"]) + # Anthropic for vision analysis + anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) - pipeline = Pipeline( - [ - transport.input(), - user_response, - image_requester, - vision_aggregator, - anthropic, - tts, - transport.output(), - ] - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - task = PipelineTask(pipeline) + pipeline = Pipeline( + [ + transport.input(), + stt, + user_response, + image_requester, + vision_aggregator, + anthropic, + tts, + transport.output(), + ] + ) - runner = PipelineRunner() + task = PipelineTask( + pipeline, + params=PipelineParams(allow_interruptions=True), + ) - await runner.run(task) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") + + # Set the participant ID in the image requester + image_requester.set_participant_id(webrtc_peer_id) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/13-whisper-transcription.py b/examples/foundational/13-whisper-transcription.py index e9fe7400b..5df7ff854 100644 --- a/examples/foundational/13-whisper-transcription.py +++ b/examples/foundational/13-whisper-transcription.py @@ -4,13 +4,9 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TranscriptionFrame @@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper.stt import WhisperSTTService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -35,34 +30,42 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - None, - "Transcription bot", - DailyParams( - audio_in_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = WhisperSTTService() + stt = WhisperSTTService() - tl = TranscriptionLogger() + tl = TranscriptionLogger() - pipeline = Pipeline([transport.input(), stt, tl]) + pipeline = Pipeline([transport.input(), stt, tl]) - task = PipelineTask(pipeline) + task = PipelineTask(pipeline) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/13b-deepgram-transcription.py b/examples/foundational/13b-deepgram-transcription.py index 32452d9f2..e76ffb45b 100644 --- a/examples/foundational/13b-deepgram-transcription.py +++ b/examples/foundational/13b-deepgram-transcription.py @@ -4,14 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio -import os -import sys -import aiohttp +import os + from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,13 +16,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -35,29 +31,40 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True) - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams(audio_in_enabled=True), + ) - stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - # live_options=LiveOptions(language=Language.FR), - ) + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + live_options=LiveOptions(language=Language.EN), + ) - tl = TranscriptionLogger() + tl = TranscriptionLogger() - pipeline = Pipeline([transport.input(), stt, tl]) + pipeline = Pipeline([transport.input(), stt, tl]) - task = PipelineTask(pipeline) + task = PipelineTask(pipeline) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/13c-gladia-transcription.py b/examples/foundational/13c-gladia-transcription.py index d982c20d1..76299f837 100644 --- a/examples/foundational/13c-gladia-transcription.py +++ b/examples/foundational/13c-gladia-transcription.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.gladia import GladiaSTTService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -35,29 +30,40 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True) - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams(audio_in_enabled=True), + ) - stt = GladiaSTTService( - api_key=os.getenv("GLADIA_API_KEY"), - # live_options=LiveOptions(language=Language.FR), - ) + stt = GladiaSTTService( + api_key=os.getenv("GLADIA_API_KEY"), + # live_options=LiveOptions(language=Language.FR), + ) - tl = TranscriptionLogger() + tl = TranscriptionLogger() - pipeline = Pipeline([transport.input(), stt, tl]) + pipeline = Pipeline([transport.input(), stt, tl]) - task = PipelineTask(pipeline) + task = PipelineTask(pipeline) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/13d-assemblyai-transcription.py b/examples/foundational/13d-assemblyai-transcription.py index 18ba8d44f..a241a88f1 100644 --- a/examples/foundational/13d-assemblyai-transcription.py +++ b/examples/foundational/13d-assemblyai-transcription.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.assemblyai.stt import AssemblyAISTTService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -35,28 +30,39 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True) - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams(audio_in_enabled=True), + ) - stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), - ) + stt = AssemblyAISTTService( + api_key=os.getenv("ASSEMBLYAI_API_KEY"), + ) - tl = TranscriptionLogger() + tl = TranscriptionLogger() - pipeline = Pipeline([transport.input(), stt, tl]) + pipeline = Pipeline([transport.input(), stt, tl]) - task = PipelineTask(pipeline) + task = PipelineTask(pipeline) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/13e-whisper-mlx.py b/examples/foundational/13e-whisper-mlx.py index 4ed2bb2c4..8470dce61 100644 --- a/examples/foundational/13e-whisper-mlx.py +++ b/examples/foundational/13e-whisper-mlx.py @@ -4,14 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio -import sys + import time -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -21,13 +18,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper.stt import MLXModel, WhisperSTTServiceMLX -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - STOP_SECS = 2.0 @@ -56,40 +52,48 @@ class TranscriptionLogger(FrameProcessor): self._last_transcription_time = time.time() -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - None, - "Transcription bot", - DailyParams( - audio_in_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)), + vad_audio_passthrough=True, + ), + ) - stt = WhisperSTTServiceMLX(model=MLXModel.LARGE_V3_TURBO) + stt = WhisperSTTServiceMLX(model=MLXModel.LARGE_V3_TURBO) - tl = TranscriptionLogger() + tl = TranscriptionLogger() - pipeline = Pipeline([transport.input(), stt, tl]) + pipeline = Pipeline([transport.input(), stt, tl]) - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - report_only_initial_ttfb=False, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + report_only_initial_ttfb=False, + ), + ) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 942995248..70d3c1667 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,106 +18,118 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index e7fcc6a34..be6597027 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,99 +18,111 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): location = arguments["location"] await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" - ) - llm.register_function("get_weather", get_weather) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" + ) + llm.register_function("get_weather", get_weather) + + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) - # todo: test with very short initial user message + # todo: test with very short initial user message - # messages = [{"role": "system", - # "content": "You are a helpful assistant who can report the weather in any location in the universe. Respond concisely. Your response will be turned into speech so use only simple words and punctuation."}, - # {"role": "user", - # "content": " Start the conversation by introducing yourself."}] + # messages = [{"role": "system", + # "content": "You are a helpful assistant who can report the weather in any location in the universe. Respond concisely. Your response will be turned into speech so use only simple words and punctuation."}, + # {"role": "user", + # "content": " Start the conversation by introducing yourself."}] - messages = [{"role": "user", "content": "Say 'hello' to start the conversation."}] + messages = [{"role": "user", "content": "Say 'hello' to start the conversation."}] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User spoken responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User spoken responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses and tool context + ] + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 92d570c1e..995d39643 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -6,12 +6,9 @@ import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,14 +19,16 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") -video_participant_id = None +# Global variable to store the peer connection ID +webrtc_peer_id = None async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): @@ -39,72 +38,83 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): question = arguments["question"] + logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + + # Request the image frame await llm.request_image_frame( - user_id=video_participant_id, + user_id=webrtc_peer_id, function_name=function_name, tool_call_id=tool_call_id, text_content=question, ) + # Wait a short time for the frame to be processed + await asyncio.sleep(0.5) -async def main(): - global llm + # Return a result to complete the function call + await result_callback( + f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" + ) - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + global webrtc_peer_id + webrtc_peer_id = webrtc_connection.pc_id - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - model="claude-3-7-sonnet-latest", - enable_prompt_caching_beta=True, - ) - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, # Make sure camera input is enabled + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), + model="claude-3-7-sonnet-latest", + enable_prompt_caching_beta=True, + ) + llm.register_function("get_weather", get_weather) + llm.register_function("get_image", get_image) + + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) + }, + required=["location"], + ) + get_image_function = FunctionSchema( + name="get_image", + description="Get an image from the video stream.", + properties={ + "question": { + "type": "string", + "description": "The question that the user is asking about the image.", + } + }, + required=["question"], + ) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - # todo: test with very short initial user message - - system_prompt = """\ + system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. Your response will be turned into speech so use only simple words and punctuation. @@ -115,63 +125,73 @@ You can respond to questions about the weather using the get_weather tool. You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ indicate you should use the get_image tool are: - - What do you see? - - What's in the video? - - Can you describe the video? - - Tell me about what you see. - - Tell me something interesting about what you see. - - What's happening in the video? +- What do you see? +- What's in the video? +- Can you describe the video? +- Tell me about what you see. +- Tell me something interesting about what you see. +- What's happening in the video? If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise. - """ + """ - messages = [ - { - "role": "system", - "content": [ - { - "type": "text", - "text": system_prompt, - } - ], - }, - {"role": "user", "content": "Start the conversation by introducing yourself."}, + messages = [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": system_prompt, + } + ], + }, + {"role": "user", "content": "Start the conversation by introducing yourself."}, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User speech to text + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses and tool context ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User speech to text - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - global video_participant_id - video_participant_id = participant["id"] - await transport.capture_participant_transcription(video_participant_id) - await transport.capture_participant_video(video_participant_id, framerate=0) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index dcceb973b..4eb7883bd 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,99 +18,111 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.together.llm import TogetherLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = TogetherLLMService( - api_key=os.getenv("TOGETHER_API_KEY"), - model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = TogetherLLMService( + api_key=os.getenv("TOGETHER_API_KEY"), + model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", + ) + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask(pipeline) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask(pipeline) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index e50ea5b31..f8c86b8e7 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -6,12 +6,9 @@ import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -21,15 +18,17 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") -video_participant_id = None +# Global variable to store the peer connection ID +webrtc_peer_id = None async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): @@ -38,71 +37,85 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): - logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}") question = arguments["question"] + logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + + # Request the image frame await llm.request_image_frame( - user_id=video_participant_id, + user_id=webrtc_peer_id, function_name=function_name, tool_call_id=tool_call_id, text_content=question, ) + # Wait a short time for the frame to be processed + await asyncio.sleep(0.5) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + # Return a result to complete the function call + await result_callback( + f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + global webrtc_peer_id + webrtc_peer_id = webrtc_connection.pc_id - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, # Make sure camera input is enabled + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm.register_function("get_weather", get_weather) + llm.register_function("get_image", get_image) + + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) + }, + required=["location"], + ) + get_image_function = FunctionSchema( + name="get_image", + description="Get an image from the video stream.", + properties={ + "question": { + "type": "string", + "description": "The question that the user is asking about the image.", + } + }, + required=["question"], + ) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - system_prompt = """\ + system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. Your response will be turned into speech so use only simple words and punctuation. @@ -113,46 +126,55 @@ You can respond to questions about the weather using the get_weather tool. You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ indicate you should use the get_image tool are: - - What do you see? - - What's in the video? - - Can you describe the video? - - Tell me about what you see. - - Tell me something interesting about what you see. - - What's happening in the video? +- What do you see? +- What's in the video? +- Can you describe the video? +- Tell me about what you see. +- Tell me something interesting about what you see. +- What's happening in the video? """ - messages = [ - {"role": "system", "content": system_prompt}, + messages = [ + {"role": "system", "content": system_prompt}, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask(pipeline) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask(pipeline) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - global video_participant_id - video_participant_id = participant["id"] - await transport.capture_participant_transcription(participant["id"]) - await transport.capture_participant_video(video_participant_id, framerate=0) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 9853c07f6..ccd2efc82 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -6,12 +6,9 @@ import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,15 +19,17 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") -video_participant_id = None +# Global variable to store the peer connection ID +webrtc_peer_id = None async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): @@ -40,71 +39,85 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): - logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}") question = arguments["question"] + logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + + # Request the image frame await llm.request_image_frame( - user_id=video_participant_id, + user_id=webrtc_peer_id, function_name=function_name, tool_call_id=tool_call_id, text_content=question, ) + # Wait a short time for the frame to be processed + await asyncio.sleep(0.5) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + # Return a result to complete the function call + await result_callback( + f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + global webrtc_peer_id + webrtc_peer_id = webrtc_connection.pc_id - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, # Make sure camera input is enabled + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") + llm.register_function("get_weather", get_weather) + llm.register_function("get_image", get_image) + + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, - required=["question"], - ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) + }, + required=["location", "format"], + ) + get_image_function = FunctionSchema( + name="get_image", + description="Get an image from the video stream.", + properties={ + "question": { + "type": "string", + "description": "The question that the user is asking about the image.", + } + }, + required=["question"], + ) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - system_prompt = """\ + system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. Your response will be turned into speech so use only simple words and punctuation. @@ -115,54 +128,63 @@ You can respond to questions about the weather using the get_weather tool. You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ indicate you should use the get_image tool are: - - What do you see? - - What's in the video? - - Can you describe the video? - - Tell me about what you see. - - Tell me something interesting about what you see. - - What's happening in the video? +- What do you see? +- What's in the video? +- Can you describe the video? +- Tell me about what you see. +- Tell me something interesting about what you see. +- What's happening in the video? """ - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Say hello."}, + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "Say hello."}, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - global video_participant_id - video_participant_id = participant["id"] - await transport.capture_participant_transcription(participant["id"]) - await transport.capture_participant_video(video_participant_id, framerate=0) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 839c9460c..a4a58e23f 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -24,105 +20,113 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.groq.llm import GroqLLMService from pipecat.services.groq.stt import GroqSTTService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"), model="distil-whisper-large-v3-en") + stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"), model="distil-whisper-large-v3-en") - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index f287b8380..11cbdf991 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -21,102 +17,114 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.grok.llm import GrokLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index e10b4ac02..f52e42b80 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -23,106 +19,118 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.azure.llm import AzureLLMService from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), - model=os.getenv("AZURE_CHATGPT_MODEL"), - ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL"), + ) + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 3d2f1e876..61c2c72e6 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,106 +18,118 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.fireworks.llm import FireworksLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = FireworksLLMService( - api_key=os.getenv("FIREWORKS_API_KEY"), - model="accounts/fireworks/models/llama-v3p1-405b-instruct", - ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = FireworksLLMService( + api_key=os.getenv("FIREWORKS_API_KEY"), + model="accounts/fireworks/models/llama-v3p1-405b-instruct", + ) + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 174b92aa3..cd5ee8172 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,106 +18,116 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.nim.llm import NimLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - # text_filters=[MarkdownTextFilter()], - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = NimLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct" - ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + # text_filters=[MarkdownTextFilter()], + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct") + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index fbe27f35b..032d1c2f9 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -23,66 +19,66 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cerebras.llm import CerebrasLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have one functions available: @@ -92,44 +88,56 @@ Infer whether to use Fahrenheit or Celsius automatically based on the location, Start by asking me for my location. Then, use 'get_weather_current' to give me a forecast. - Respond to what the user said in a creative and helpful way.""", - }, + Respond to what the user said in a creative and helpful way.""", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index 674579b75..4e9b55f20 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,67 +18,67 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepseek.llm import DeepSeekLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have one functions available: @@ -92,44 +88,56 @@ Infer whether to use Fahrenheit or Celsius automatically based on the location, Start by asking me for my location. Then, use 'get_weather_current' to give me a forecast. - Respond to what the user said in a creative and helpful way.""", - }, + Respond to what the user said in a creative and helpful way.""", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index 6ae4c4e7a..624ff947e 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,108 +18,120 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.azure.tts import AzureTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openrouter.llm import OpenRouterLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = AzureTTSService( - api_key=os.getenv("AZURE_API_KEY"), - region="eastus", - voice="en-US-JennyNeural", - params=AzureTTSService.InputParams(language="en-US", rate="1.1", style="cheerful"), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = OpenRouterLLMService( - api_key=os.getenv("OPENROUTER_API_KEY"), model="openai/gpt-4o-2024-11-20" - ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = AzureTTSService( + api_key=os.getenv("AZURE_API_KEY"), + region="eastus", + voice="en-US-JennyNeural", + params=AzureTTSService.InputParams(language="en-US", rate="1.1", style="cheerful"), + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = OpenRouterLLMService( + api_key=os.getenv("OPENROUTER_API_KEY"), model="openai/gpt-4o-2024-11-20" + ) + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 47afe5918..c0e0e8127 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -11,14 +11,10 @@ currently support function calling. The example shows basic chat completion func using Perplexity's API while maintaining compatibility with the OpenAI interface. """ -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -26,79 +22,91 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.perplexity.llm import PerplexityLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY"), model="sonar") + llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY"), model="sonar") - messages = [ - { - "role": "user", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "user", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14o-function-calling-gemini-openai-format.py b/examples/foundational/14o-function-calling-gemini-openai-format.py index bbd205e41..00875c741 100644 --- a/examples/foundational/14o-function-calling-gemini-openai-format.py +++ b/examples/foundational/14o-function-calling-gemini-openai-format.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -21,104 +17,116 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.google.llm_openai import GoogleLLMOpenAIBetaService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY", ""), - voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = GoogleLLMOpenAIBetaService(api_key=os.getenv("GEMINI_API_KEY")) - # You can aslo register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + llm = GoogleLLMOpenAIBetaService(api_key=os.getenv("GEMINI_API_KEY")) + # You can aslo register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - messages = [ - { - "role": "user", - "content": "Start a conversation with 'Hey there' to get the current weather.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ + { + "role": "user", + "content": "Start a conversation with 'Hey there' to get the current weather.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index edb9bed5a..757001ffb 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -21,110 +17,122 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.google.llm_vertex import GoogleVertexLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) + + llm = GoogleVertexLLMService( + # credentials="", + params=GoogleVertexLLMService.InputParams( + project_id="", ) + ) + # You can aslo register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) - tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY", ""), - voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), - ) - - llm = GoogleVertexLLMService( - # credentials="", - params=GoogleVertexLLMService.InputParams( - project_id="", - ) - ) - # You can aslo register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) - - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - - messages = [ - { - "role": "user", - "content": "Start a conversation with 'Hey there' to get the current weather.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "user", + "content": "Start a conversation with 'Hey there' to get the current weather.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/14q-function-calling-qwen.py b/examples/foundational/14q-function-calling-qwen.py index 43b290040..ed514f5ee 100644 --- a/examples/foundational/14q-function-calling-qwen.py +++ b/examples/foundational/14q-function-calling-qwen.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -22,106 +18,118 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.qwen.llm import QwenLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = QwenLLMService(api_key=os.getenv("QWEN_API_KEY"), model="qwen2.5-72b-instruct") + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + llm = QwenLLMService(api_key=os.getenv("QWEN_API_KEY"), model="qwen2.5-72b-instruct") - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 8743f6aad..85e063f21 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger from openai.types.chat import ChatCompletionToolParam -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.parallel_pipeline import ParallelPipeline @@ -22,13 +18,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") current_voice = "News Lady" @@ -55,105 +52,117 @@ async def barbershop_man_filter(frame) -> bool: return current_voice == "Barbershop Man" -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Pipecat", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - news_lady = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - british_lady = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + news_lady = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady + ) - barbershop_man = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", # Barbershop Man - ) + british_lady = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm.register_function("switch_voice", switch_voice) + barbershop_man = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", # Barbershop Man + ) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "switch_voice", - "description": "Switch your voice only when the user asks you to", - "parameters": { - "type": "object", - "properties": { - "voice": { - "type": "string", - "description": "The voice the user wants you to use", - }, + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm.register_function("switch_voice", switch_voice) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "switch_voice", + "description": "Switch your voice only when the user asks you to", + "parameters": { + "type": "object", + "properties": { + "voice": { + "type": "string", + "description": "The voice the user wants you to use", }, - "required": ["voice"], }, + "required": ["voice"], }, - ) + }, + ) + ] + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities. Respond to what the user said in a creative and helpful way. Your output should not include non-alphanumeric characters. You can do the following voices: 'News Lady', 'British Lady' and 'Barbershop Man'.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + ParallelPipeline( # TTS (one of the following vocies) + [FunctionFilter(news_lady_filter), news_lady], # News Lady voice + [ + FunctionFilter(british_lady_filter), + british_lady, + ], # British Reading Lady voice + [FunctionFilter(barbershop_man_filter), barbershop_man], # Barbershop Man voice + ), + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] - messages = [ + ) + + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append( { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities. Respond to what the user said in a creative and helpful way. Your output should not include non-alphanumeric characters. You can do the following voices: 'News Lady', 'British Lady' and 'Barbershop Man'.", - }, - ] - - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - ParallelPipeline( # TTS (one of the following vocies) - [FunctionFilter(news_lady_filter), news_lady], # News Lady voice - [ - FunctionFilter(british_lady_filter), - british_lady, - ], # British Reading Lady voice - [FunctionFilter(barbershop_man_filter), barbershop_man], # Barbershop Man voice - ), - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] + "content": f"Please introduce yourself to the user and let them know the voices you can do. Your initial responses should be as if you were a {current_voice}.", + } ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append( - { - "role": "system", - "content": f"Please introduce yourself to the user and let them know the voices you can do. Your initial responses should be as if you were a {current_voice}.", - } - ) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index dd094f8ea..0e7cb7c3c 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -4,16 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from deepgram import LiveOptions from dotenv import load_dotenv from loguru import logger from openai.types.chat import ChatCompletionToolParam -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.parallel_pipeline import ParallelPipeline @@ -25,12 +21,12 @@ from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") current_language = "English" @@ -49,101 +45,110 @@ async def spanish_filter(frame) -> bool: return current_language == "Spanish" -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Pipecat", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), live_options=LiveOptions(language="multi") - ) + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), live_options=LiveOptions(language="multi") + ) - english_tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + english_tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - spanish_tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="846d6cb0-2301-48b6-9683-48f5618ea2f6", # Spanish-speaking Lady - ) + spanish_tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", # Spanish-speaking Lady + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm.register_function("switch_language", switch_language) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm.register_function("switch_language", switch_language) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "switch_language", - "description": "Switch to another language when the user asks you to", - "parameters": { - "type": "object", - "properties": { - "language": { - "type": "string", - "description": "The language the user wants you to speak", - }, + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "switch_language", + "description": "Switch to another language when the user asks you to", + "parameters": { + "type": "object", + "properties": { + "language": { + "type": "string", + "description": "The language the user wants you to speak", }, - "required": ["language"], }, + "required": ["language"], }, - ) + }, + ) + ] + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities. Respond to what the user said in a creative and helpful way. Your output should not include non-alphanumeric characters. You can speak the following languages: 'English' and 'Spanish'.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + ParallelPipeline( # TTS (bot will speak the chosen language) + [FunctionFilter(english_filter), english_tts], # English + [FunctionFilter(spanish_filter), spanish_tts], # Spanish + ), + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] - messages = [ + ) + + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append( { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities. Respond to what the user said in a creative and helpful way. Your output should not include non-alphanumeric characters. You can speak the following languages: 'English' and 'Spanish'.", - }, - ] - - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - ParallelPipeline( # TTS (bot will speak the chosen language) - [FunctionFilter(english_filter), english_tts], # English - [FunctionFilter(spanish_filter), spanish_tts], # Spanish - ), - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] + "content": f"Please introduce yourself to the user and let them know the languages you speak. Your initial responses should be in {current_language}.", + } ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append( - { - "role": "system", - "content": f"Please introduce yourself to the user and let them know the languages you speak. Your initial responses should be in {current_language}.", - } - ) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index 07afcfa7b..d23b29271 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -4,49 +4,45 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import ( - DailyParams, - DailyTransport, - DailyTransportMessageFrame, -) +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.services.daily import DailyTransportMessageFrame load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + # Create an HTTP session async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = DeepgramTTSService( aiohttp_session=session, @@ -77,6 +73,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), # Transport user input + stt, # STT context_aggregator.user(), llm, # LLM tts, # TTS @@ -93,15 +90,11 @@ async def main(): ), ) - # When a participant joins, start transcription for that participant so the - # bot can "hear" and respond to them. - @transport.event_handler("on_participant_joined") - async def on_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # When the first participant joins, the bot should introduce itself. - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([context_aggregator.user().get_context_frame()]) @@ -134,9 +127,21 @@ async def main(): except Exception as e: logger.debug(f"message handling error: {e} - {message}") - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 41cd84d67..d7630c798 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import EndFrame, LLMMessagesFrame, TTSSpeakFrame @@ -21,111 +17,123 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.user_idle_processor import UserIdleProcessor from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + async def handle_user_idle(user_idle: UserIdleProcessor, retry_count: int) -> bool: + if retry_count == 1: + # First attempt: Add a gentle prompt to the conversation + messages.append( + { + "role": "system", + "content": "The user has been quiet. Politely and briefly ask if they're still there.", + } + ) + await user_idle.push_frame(LLMMessagesFrame(messages)) + return True + elif retry_count == 2: + # Second attempt: More direct prompt + messages.append( + { + "role": "system", + "content": "The user is still inactive. Ask if they'd like to continue our conversation.", + } + ) + await user_idle.push_frame(LLMMessagesFrame(messages)) + return True + else: + # Third attempt: End the conversation + await user_idle.push_frame( + TTSSpeakFrame("It seems like you're busy right now. Have a nice day!") + ) + await task.queue_frame(EndFrame()) + return False + + user_idle = UserIdleProcessor(callback=handle_user_idle, timeout=5.0) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + user_idle, # Idle user check-in + context_aggregator.user(), + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + report_only_initial_ttfb=True, + ), + ) - async def handle_user_idle(user_idle: UserIdleProcessor, retry_count: int) -> bool: - if retry_count == 1: - # First attempt: Add a gentle prompt to the conversation - messages.append( - { - "role": "system", - "content": "The user has been quiet. Politely and briefly ask if they're still there.", - } - ) - await user_idle.push_frame(LLMMessagesFrame(messages)) - return True - elif retry_count == 2: - # Second attempt: More direct prompt - messages.append( - { - "role": "system", - "content": "The user is still inactive. Ask if they'd like to continue our conversation.", - } - ) - await user_idle.push_frame(LLMMessagesFrame(messages)) - return True - else: - # Third attempt: End the conversation - await user_idle.push_frame( - TTSSpeakFrame("It seems like you're busy right now. Have a nice day!") - ) - await task.queue_frame(EndFrame()) - return False + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - user_idle = UserIdleProcessor(callback=handle_user_idle, timeout=5.0) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - pipeline = Pipeline( - [ - transport.input(), # Transport user input - user_idle, # Idle user check-in - context_aggregator.user(), - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - report_only_initial_ttfb=True, - ), - ) + runner = PipelineRunner(handle_sigint=False) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/18-gstreamer-filesrc.py b/examples/foundational/18-gstreamer-filesrc.py index 12bec25bd..ed2b45699 100644 --- a/examples/foundational/18-gstreamer-filesrc.py +++ b/examples/foundational/18-gstreamer-filesrc.py @@ -5,67 +5,71 @@ # import argparse -import asyncio -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure_with_args from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") + +# Parse command line arguments +# This will be used to pass the input video file to the bot +# You can run the bot with a command like: +# python 18-gstreamer-filesrc.py -i path/to/video.mp4 +def parse_arguments(): + parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") + parser.add_argument("-i", "--input", type=str, required=True, help="Input video file") + return parser.parse_args() -async def main(): - async with aiohttp.ClientSession() as session: - parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") - parser.add_argument("-i", "--input", type=str, required=True, help="Input video file") +args = parse_arguments() - (room_url, _, args) = await configure_with_args(session, parser) - transport = DailyTransport( - room_url, - None, - "GStreamer", - DailyParams( - audio_out_enabled=True, - camera_out_enabled=True, - camera_out_width=1280, - camera_out_height=720, - camera_out_is_live=True, - ), - ) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot with video input: {args.input}") - gst = GStreamerPipelineSource( - pipeline=f"filesrc location={args.input}", - out_params=GStreamerPipelineSource.OutputParams( - video_width=1280, - video_height=720, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + camera_out_enabled=True, + camera_out_is_live=True, + camera_out_width=1280, + camera_out_height=720, + ), + ) - pipeline = Pipeline( - [ - gst, # GStreamer file source - transport.output(), # Transport bot output - ] - ) + gst = GStreamerPipelineSource( + pipeline=f"filesrc location={args.input}", + out_params=GStreamerPipelineSource.OutputParams( + video_width=1280, + video_height=720, + ), + ) - task = PipelineTask(pipeline) + pipeline = Pipeline( + [ + gst, # GStreamer file source + transport.output(), # Transport bot output + ] + ) - runner = PipelineRunner() + task = PipelineTask(pipeline) - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/18a-gstreamer-videotestsrc.py b/examples/foundational/18a-gstreamer-videotestsrc.py index b1139c12d..612701ec6 100644 --- a/examples/foundational/18a-gstreamer-videotestsrc.py +++ b/examples/foundational/18a-gstreamer-videotestsrc.py @@ -4,62 +4,57 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot with video test source") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + camera_out_enabled=True, + camera_out_is_live=True, + camera_out_width=1280, + camera_out_height=720, + ), + ) - transport = DailyTransport( - room_url, - None, - "GStreamer", - DailyParams( - camera_out_enabled=True, - camera_out_width=1280, - camera_out_height=720, - camera_out_is_live=True, - ), - ) + gst = GStreamerPipelineSource( + pipeline='videotestsrc ! capsfilter caps="video/x-raw,width=1280,height=720,framerate=30/1"', + out_params=GStreamerPipelineSource.OutputParams( + video_width=1280, video_height=720, clock_sync=False + ), + ) - gst = GStreamerPipelineSource( - pipeline='videotestsrc ! capsfilter caps="video/x-raw,width=1280,height=720,framerate=30/1"', - out_params=GStreamerPipelineSource.OutputParams( - video_width=1280, video_height=720, clock_sync=False - ), - ) + pipeline = Pipeline( + [ + gst, # GStreamer test source + transport.output(), # Transport bot output + ] + ) - pipeline = Pipeline( - [ - gst, # GStreamer file source - transport.output(), # Transport bot output - ] - ) + task = PipelineTask(pipeline) - task = PipelineTask(pipeline) + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 3aff14e65..9a7e8d8b5 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from datetime import datetime -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -29,13 +25,12 @@ from pipecat.services.openai_realtime_beta import ( SemanticTurnDetection, SessionProperties, ) -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): temperature = 75 if args["format"] == "fahrenheit" else 24 @@ -70,34 +65,30 @@ weather_function = FunctionSchema( tools = ToolsSchema(standard_tools=[weather_function]) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - transcription_enabled=False, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) - session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), - # Set openai TurnDetection parameters. Not setting this at all will turn it - # on by default - turn_detection=SemanticTurnDetection(), - # Or set to False to disable openai turn detection and use transport VAD - # turn_detection=False, - input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"), - # tools=tools, - instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + session_properties = SessionProperties( + input_audio_transcription=InputAudioTranscription(), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + turn_detection=SemanticTurnDetection(), + # Or set to False to disable openai turn detection and use transport VAD + # turn_detection=False, + input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"), + # tools=tools, + instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and @@ -111,68 +102,79 @@ You are participating in a voice conversation. Keep your responses concise, shor unless specifically asked to elaborate on a topic. Remember, your responses should be short. Just one or two sentences, usually.""", - ) + ) - llm = OpenAIRealtimeBetaLLMService( - api_key=os.getenv("OPENAI_API_KEY"), - session_properties=session_properties, - start_audio_paused=False, - ) + llm = OpenAIRealtimeBetaLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + session_properties=session_properties, + start_audio_paused=False, + ) - # you can either register a single function for all function calls, or specific functions - # llm.register_function(None, fetch_weather_from_api) - llm.register_function("get_current_weather", fetch_weather_from_api) + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) - # Create a standard OpenAI LLM context object using the normal messages format. The - # OpenAIRealtimeBetaLLMService will convert this internally to messages that the - # openai WebSocket API can understand. - context = OpenAILLMContext( - [{"role": "user", "content": "Say hello!"}], - # [{"role": "user", "content": [{"type": "text", "text": "Say hello!"}]}], - # [ - # { - # "role": "user", - # "content": [ - # {"type": "text", "text": "Say"}, - # {"type": "text", "text": "yo what's up!"}, - # ], - # } - # ], - tools, - ) + # Create a standard OpenAI LLM context object using the normal messages format. The + # OpenAIRealtimeBetaLLMService will convert this internally to messages that the + # openai WebSocket API can understand. + context = OpenAILLMContext( + [{"role": "user", "content": "Say hello!"}], + # [{"role": "user", "content": [{"type": "text", "text": "Say hello!"}]}], + # [ + # { + # "role": "user", + # "content": [ + # {"type": "text", "text": "Say"}, + # {"type": "text", "text": "yo what's up!"}, + # ], + # } + # ], + tools, + ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = llm.create_context_aggregator(context) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), - llm, # LLM - transport.output(), # Transport bot output - context_aggregator.assistant(), - ] - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), + llm, # LLM + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - # report_only_initial_ttfb=True, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 004192ca2..abde93a12 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from datetime import datetime -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -27,13 +23,12 @@ from pipecat.services.openai_realtime_beta import ( InputAudioTranscription, SessionProperties, ) -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): temperature = 75 if args["format"] == "fahrenheit" else 24 @@ -69,33 +64,29 @@ weather_function = FunctionSchema( tools = ToolsSchema(standard_tools=[weather_function]) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - transcription_enabled=False, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) - session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), - # Set openai TurnDetection parameters. Not setting this at all will turn it - # on by default - # turn_detection=TurnDetection(silence_duration_ms=1000), - # Or set to False to disable openai turn detection and use transport VAD - # turn_detection=False, - # tools=tools, - instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + session_properties = SessionProperties( + input_audio_transcription=InputAudioTranscription(model="whisper-1"), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + # turn_detection=TurnDetection(silence_duration_ms=1000), + # Or set to False to disable openai turn detection and use transport VAD + # turn_detection=False, + # tools=tools, + instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and @@ -109,69 +100,80 @@ You are participating in a voice conversation. Keep your responses concise, shor unless specifically asked to elaborate on a topic. Remember, your responses should be short. Just one or two sentences, usually.""", - ) + ) - llm = AzureRealtimeBetaLLMService( - api_key=os.getenv("AZURE_REALTIME_API_KEY"), - base_url=os.getenv("AZURE_REALTIME_BASE_URL"), - session_properties=session_properties, - start_audio_paused=False, - ) + llm = AzureRealtimeBetaLLMService( + api_key=os.getenv("AZURE_REALTIME_API_KEY"), + base_url=os.getenv("AZURE_REALTIME_BASE_URL"), + session_properties=session_properties, + start_audio_paused=False, + ) - # you can either register a single function for all function calls, or specific functions - # llm.register_function(None, fetch_weather_from_api) - llm.register_function("get_current_weather", fetch_weather_from_api) + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) - # Create a standard OpenAI LLM context object using the normal messages format. The - # OpenAIRealtimeBetaLLMService will convert this internally to messages that the - # openai WebSocket API can understand. - context = OpenAILLMContext( - [{"role": "user", "content": "Say hello!"}], - # [{"role": "user", "content": [{"type": "text", "text": "Say hello!"}]}], - # [ - # { - # "role": "user", - # "content": [ - # {"type": "text", "text": "Say"}, - # {"type": "text", "text": "yo what's up!"}, - # ], - # } - # ], - tools, - ) + # Create a standard OpenAI LLM context object using the normal messages format. The + # OpenAIRealtimeBetaLLMService will convert this internally to messages that the + # openai WebSocket API can understand. + context = OpenAILLMContext( + [{"role": "user", "content": "Say hello!"}], + # [{"role": "user", "content": [{"type": "text", "text": "Say hello!"}]}], + # [ + # { + # "role": "user", + # "content": [ + # {"type": "text", "text": "Say"}, + # {"type": "text", "text": "yo what's up!"}, + # ], + # } + # ], + tools, + ) - context_aggregator = llm.create_context_aggregator(context) + context_aggregator = llm.create_context_aggregator(context) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), - llm, # LLM - transport.output(), # Transport bot output - context_aggregator.assistant(), - ] - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), + llm, # LLM + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - # report_only_initial_ttfb=True, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 2c792baed..22950e0fa 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -4,17 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import glob import json import os -import sys from datetime import datetime -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -25,13 +21,14 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") BASE_FILENAME = "/tmp/pipecat_conversation_" tts = None @@ -165,71 +162,84 @@ tools = [ ] -async def main(): +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + global tts - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # you can either register a single function for all function calls, or specific functions - # llm.register_function(None, fetch_weather_from_api) - llm.register_function("get_current_weather", fetch_weather_from_api) - llm.register_function("save_conversation", save_conversation) - llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) - llm.register_function("load_conversation", load_conversation) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("save_conversation", save_conversation) + llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) + llm.register_function("load_conversation", load_conversation) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), - llm, # LLM - tts, - transport.output(), # Transport bot output - context_aggregator.assistant(), - ] - ) + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - # report_only_initial_ttfb=True, - ), - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), + llm, # LLM + tts, + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - runner = PipelineRunner() + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - await runner.run(task) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 14a35fe00..517aa29be 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -8,13 +8,10 @@ import asyncio import glob import json import os -import sys from datetime import datetime -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -24,19 +21,19 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai_realtime_beta import ( InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, ) -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - BASE_FILENAME = "/tmp/pipecat_conversation_" @@ -167,33 +164,31 @@ tools = [ ] -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - transcription_enabled=False, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) - session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), - # Set openai TurnDetection parameters. Not setting this at all will turn it - # on by default - turn_detection=TurnDetection(silence_duration_ms=1000), - # Or set to False to disable openai turn detection and use transport VAD - # turn_detection=False, - # tools=tools, - instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + session_properties = SessionProperties( + input_audio_transcription=InputAudioTranscription(), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + turn_detection=TurnDetection(silence_duration_ms=1000), + # Or set to False to disable openai turn detection and use transport VAD + # turn_detection=False, + # tools=tools, + instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and @@ -207,54 +202,66 @@ You are participating in a voice conversation. Keep your responses concise, shor unless specifically asked to elaborate on a topic. Remember, your responses should be short. Just one or two sentences, usually.""", - ) + ) - llm = OpenAIRealtimeBetaLLMService( - api_key=os.getenv("OPENAI_API_KEY"), - session_properties=session_properties, - start_audio_paused=False, - ) + llm = OpenAIRealtimeBetaLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + session_properties=session_properties, + start_audio_paused=False, + ) - # you can either register a single function for all function calls, or specific functions - # llm.register_function(None, fetch_weather_from_api) - llm.register_function("get_current_weather", fetch_weather_from_api) - llm.register_function("save_conversation", save_conversation) - llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) - llm.register_function("load_conversation", load_conversation) + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("save_conversation", save_conversation) + llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) + llm.register_function("load_conversation", load_conversation) - context = OpenAILLMContext([], tools) - context_aggregator = llm.create_context_aggregator(context) + context = OpenAILLMContext([], tools) + context_aggregator = llm.create_context_aggregator(context) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), - llm, # LLM - transport.output(), # Transport bot output - context_aggregator.assistant(), - ] - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), + llm, # LLM + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - # report_only_initial_ttfb=True, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index d9e45bab0..17c0479d8 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -4,17 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import glob import json import os -import sys from datetime import datetime -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -26,12 +22,13 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") BASE_FILENAME = "/tmp/pipecat_conversation_" tts = None @@ -160,73 +157,86 @@ tools = [ ] -async def main(): +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + global tts - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-latest" - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # you can either register a single function for all function calls, or specific functions - # llm.register_function(None, fetch_weather_from_api) - llm.register_function("get_current_weather", fetch_weather_from_api) - llm.register_function("save_conversation", save_conversation) - llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) - llm.register_function("load_conversation", load_conversation) + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-latest" + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("save_conversation", save_conversation) + llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) + llm.register_function("load_conversation", load_conversation) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), - llm, # LLM - tts, - transport.output(), # Transport bot output - context_aggregator.assistant(), - ] - ) + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - # report_only_initial_ttfb=True, - ), - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), + llm, # LLM + tts, + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + # report_only_initial_ttfb=True, + ), + ) - runner = PipelineRunner() + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - await runner.run(task) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index a00eb2c44..a97c0ac7e 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -4,17 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import glob import json import os -import sys from datetime import datetime -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -25,19 +21,21 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") video_participant_id = None BASE_FILENAME = "/tmp/pipecat_conversation_" tts = None +webrtc_peer_id = None async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): @@ -54,8 +52,11 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): question = arguments["question"] + logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + + # Request the image frame await llm.request_image_frame( - user_id=video_participant_id, + user_id=webrtc_peer_id, function_name=function_name, tool_call_id=tool_call_id, text_content=question, @@ -220,75 +221,87 @@ tools = [ ] -async def main(): - global tts - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + global tts, webrtc_peer_id + webrtc_peer_id = webrtc_connection.pc_id - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) + logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) - llm = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - # you can either register a single function for all function calls, or specific functions - # llm.register_function(None, fetch_weather_from_api) - llm.register_function("get_current_weather", fetch_weather_from_api) - llm.register_function("save_conversation", save_conversation) - llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) - llm.register_function("load_conversation", load_conversation) - llm.register_function("get_image", get_image) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + llm = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), - llm, # LLM - tts, - transport.output(), # Transport bot output - context_aggregator.assistant(), - ] - ) + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("save_conversation", save_conversation) + llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) + llm.register_function("load_conversation", load_conversation) + llm.register_function("get_image", get_image) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - # report_only_initial_ttfb=True, - ), - ) + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - global video_participant_id - video_participant_id = participant["id"] - await transport.capture_participant_transcription(participant["id"]) - await transport.capture_participant_video(video_participant_id, framerate=0) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), + llm, # LLM + tts, + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) - runner = PipelineRunner() + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + # report_only_initial_ttfb=True, + ), + ) - await runner.run(task) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 1640f8fdf..e2047c7aa 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TextFrame @@ -28,142 +24,146 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + # This is the LLM that will be used to detect if the user has finished a + # statement. This doesn't really need to be an LLM, we could use NLP + # libraries for that, but it was easier as an example because we + # leverage the context aggregators. + statement_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + statement_messages = [ + { + "role": "system", + "content": "Determine if the user's statement is a complete sentence or question, ending in a natural pause or punctuation. Return 'YES' if it is complete and 'NO' if it seems to leave a thought unfinished.", + }, + ] + + statement_context = OpenAILLMContext(statement_messages) + statement_context_aggregator = statement_llm.create_context_aggregator(statement_context) + + # This is the regular LLM. + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # We have instructed the LLM to return 'YES' if it thinks the user + # completed a sentence. So, if it's 'YES' we will return true in this + # predicate which will wake up the notifier. + async def wake_check_filter(frame): + return frame.text == "YES" + + # This is a notifier that we use to synchronize the two LLMs. + notifier = EventNotifier() + + # This a filter that will wake up the notifier if the given predicate + # (wake_check_filter) returns true. + completness_check = WakeNotifierFilter(notifier, types=(TextFrame,), filter=wake_check_filter) + + # This processor keeps the last context and will let it through once the + # notifier is woken up. We start with the gate open because we send an + # initial context frame to start the conversation. + gated_context_aggregator = GatedOpenAILLMContextAggregator(notifier=notifier, start_open=True) + + # Notify if the user hasn't said anything. + async def user_idle_notifier(frame): + await notifier.notify() + + # Sometimes the LLM will fail detecting if a user has completed a + # sentence, this will wake up the notifier if that happens. + user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=3.0) + + # The ParallePipeline input are the user transcripts. We have two + # contexts. The first one will be used to determine if the user finished + # a statement and if so the notifier will be woken up. The second + # context is simply the regular context but it's gated waiting for the + # notifier to be woken up. + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + ParallelPipeline( + [ + statement_context_aggregator.user(), + statement_llm, + completness_check, + NullFilter(), + ], + [context_aggregator.user(), gated_context_aggregator, llm], ), - ) - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - # This is the LLM that will be used to detect if the user has finished a - # statement. This doesn't really need to be an LLM, we could use NLP - # libraries for that, but it was easier as an example because we - # leverage the context aggregators. - statement_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - - statement_messages = [ - { - "role": "system", - "content": "Determine if the user's statement is a complete sentence or question, ending in a natural pause or punctuation. Return 'YES' if it is complete and 'NO' if it seems to leave a thought unfinished.", - }, + user_idle, + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - statement_context = OpenAILLMContext(statement_messages) - statement_context_aggregator = statement_llm.create_context_aggregator(statement_context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - # This is the regular LLM. - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - # We have instructed the LLM to return 'YES' if it thinks the user - # completed a sentence. So, if it's 'YES' we will return true in this - # predicate which will wake up the notifier. - async def wake_check_filter(frame): - return frame.text == "YES" + runner = PipelineRunner(handle_sigint=False) - # This is a notifier that we use to synchronize the two LLMs. - notifier = EventNotifier() - - # This a filter that will wake up the notifier if the given predicate - # (wake_check_filter) returns true. - completness_check = WakeNotifierFilter( - notifier, types=(TextFrame,), filter=wake_check_filter - ) - - # This processor keeps the last context and will let it through once the - # notifier is woken up. We start with the gate open because we send an - # initial context frame to start the conversation. - gated_context_aggregator = GatedOpenAILLMContextAggregator( - notifier=notifier, start_open=True - ) - - # Notify if the user hasn't said anything. - async def user_idle_notifier(frame): - await notifier.notify() - - # Sometimes the LLM will fail detecting if a user has completed a - # sentence, this will wake up the notifier if that happens. - user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=3.0) - - # The ParallePipeline input are the user transcripts. We have two - # contexts. The first one will be used to determine if the user finished - # a statement and if so the notifier will be woken up. The second - # context is simply the regular context but it's gated waiting for the - # notifier to be woken up. - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - ParallelPipeline( - [ - statement_context_aggregator.user(), - statement_llm, - completness_check, - NullFilter(), - ], - [context_aggregator.user(), gated_context_aggregator, llm], - ), - user_idle, - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 21ce29279..05538bec0 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -6,14 +6,11 @@ import asyncio import os -import sys import time -import aiohttp from dotenv import load_dotenv from loguru import logger from openai.types.chat import ChatCompletionToolParam -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( @@ -49,13 +46,12 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - classifier_statement = "Determine if the user's statement ends with a complete thought and you should respond. The user text is transcribed speech. It may contain multiple fragments concatentated together. You are trying to determine only the completeness of the last user statement. The previous assistant statement is provided only for context. Categorize the text as either complete with the user now expecting a response, or incomplete. Return 'YES' if text is likely complete and the user is expecting a response. Return 'NO' if the text seems to be a partial expression or unfinished thought." @@ -204,186 +200,194 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # This is the LLM that will be used to detect if the user has finished a - # statement. This doesn't really need to be an LLM, we could use NLP - # libraries for that, but we have the machinery to use an LLM, so we might as well! - statement_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # This is the LLM that will be used to detect if the user has finished a + # statement. This doesn't really need to be an LLM, we could use NLP + # libraries for that, but we have the machinery to use an LLM, so we might as well! + statement_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - # This is the regular LLM. - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - # You can also register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + # This is the regular LLM. + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", }, - "required": ["location", "format"], }, + "required": ["location", "format"], }, - ) - ] - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", }, + ) + ] + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + # We have instructed the LLM to return 'YES' if it thinks the user + # completed a sentence. So, if it's 'YES' we will return true in this + # predicate which will wake up the notifier. + async def wake_check_filter(frame): + logger.debug(f"Completeness check frame: {frame}") + return frame.text == "YES" + + # This is a notifier that we use to synchronize the two LLMs. + notifier = EventNotifier() + + # This turns the LLM context into an inference request to classify the user's speech + # as complete or incomplete. + statement_judge_context_filter = StatementJudgeContextFilter(notifier=notifier) + + # This sends a UserStoppedSpeakingFrame and triggers the notifier event + completeness_check = CompletenessCheck(notifier=notifier) + + # # Notify if the user hasn't said anything. + async def user_idle_notifier(frame): + await notifier.notify() + + # Sometimes the LLM will fail detecting if a user has completed a + # sentence, this will wake up the notifier if that happens. + user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=5.0) + + # We start with the gate open because we send an initial context frame + # to start the conversation. + bot_output_gate = OutputGate(notifier=notifier, start_open=True) + + async def block_user_stopped_speaking(frame): + return not isinstance(frame, UserStoppedSpeakingFrame) + + async def pass_only_llm_trigger_frames(frame): + return ( + isinstance(frame, OpenAILLMContextFrame) + or isinstance(frame, LLMMessagesFrame) + or isinstance(frame, StartInterruptionFrame) + or isinstance(frame, StopInterruptionFrame) + or isinstance(frame, FunctionCallInProgressFrame) + or isinstance(frame, FunctionCallResultFrame) + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + ParallelPipeline( + [ + # Pass everything except UserStoppedSpeaking to the elements after + # this ParallelPipeline + FunctionFilter(filter=block_user_stopped_speaking), + ], + [ + # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed + # LLMMessagesFrame to the statement classifier LLM. The only frame this + # sub-pipeline will output is a UserStoppedSpeakingFrame. + statement_judge_context_filter, + statement_llm, + completeness_check, + ], + [ + # Block everything except OpenAILLMContextFrame and LLMMessagesFrame + FunctionFilter(filter=pass_only_llm_trigger_frames), + llm, + bot_output_gate, # Buffer all llm/tts output until notified. + ], + ), + tts, + user_idle, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - # We have instructed the LLM to return 'YES' if it thinks the user - # completed a sentence. So, if it's 'YES' we will return true in this - # predicate which will wake up the notifier. - async def wake_check_filter(frame): - logger.debug(f"Completeness check frame: {frame}") - return frame.text == "YES" + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - # This is a notifier that we use to synchronize the two LLMs. - notifier = EventNotifier() + @transport.event_handler("on_app_message") + async def on_app_message(transport, message): + logger.debug(f"Received app message: {message}") + if "message" not in message: + return - # This turns the LLM context into an inference request to classify the user's speech - # as complete or incomplete. - statement_judge_context_filter = StatementJudgeContextFilter(notifier=notifier) - - # This sends a UserStoppedSpeakingFrame and triggers the notifier event - completeness_check = CompletenessCheck(notifier=notifier) - - # # Notify if the user hasn't said anything. - async def user_idle_notifier(frame): - await notifier.notify() - - # Sometimes the LLM will fail detecting if a user has completed a - # sentence, this will wake up the notifier if that happens. - user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=5.0) - - # We start with the gate open because we send an initial context frame - # to start the conversation. - bot_output_gate = OutputGate(notifier=notifier, start_open=True) - - async def block_user_stopped_speaking(frame): - return not isinstance(frame, UserStoppedSpeakingFrame) - - async def pass_only_llm_trigger_frames(frame): - return ( - isinstance(frame, OpenAILLMContextFrame) - or isinstance(frame, LLMMessagesFrame) - or isinstance(frame, StartInterruptionFrame) - or isinstance(frame, StopInterruptionFrame) - or isinstance(frame, FunctionCallInProgressFrame) - or isinstance(frame, FunctionCallResultFrame) - ) - - pipeline = Pipeline( + await task.queue_frames( [ - transport.input(), - stt, - context_aggregator.user(), - ParallelPipeline( - [ - # Pass everything except UserStoppedSpeaking to the elements after - # this ParallelPipeline - FunctionFilter(filter=block_user_stopped_speaking), - ], - [ - # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed - # LLMMessagesFrame to the statement classifier LLM. The only frame this - # sub-pipeline will output is a UserStoppedSpeakingFrame. - statement_judge_context_filter, - statement_llm, - completeness_check, - ], - [ - # Block everything except OpenAILLMContextFrame and LLMMessagesFrame - FunctionFilter(filter=pass_only_llm_trigger_frames), - llm, - bot_output_gate, # Buffer all llm/tts output until notified. - ], - ), - tts, - user_idle, - transport.output(), - context_aggregator.assistant(), + UserStartedSpeakingFrame(), + TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]), + UserStoppedSpeakingFrame(), ] ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_app_message") - async def on_app_message(transport, message, sender): - logger.debug(f"Received app message: {message} - {sender}") - if "message" not in message: - return + runner = PipelineRunner(handle_sigint=False) - await task.queue_frames( - [ - UserStartedSpeakingFrame(), - TranscriptionFrame( - user_id=sender, timestamp=time.time(), text=message["message"] - ), - UserStoppedSpeakingFrame(), - ] - ) - - runner = PipelineRunner() - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 71b4bb078..7d0d0a16b 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -6,14 +6,11 @@ import asyncio import os -import sys import time -import aiohttp from dotenv import load_dotenv from loguru import logger from openai.types.chat import ChatCompletionToolParam -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( @@ -50,13 +47,12 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - classifier_statement = """CRITICAL INSTRUCTION: You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO". @@ -408,195 +404,203 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # This is the LLM that will be used to detect if the user has finished a - # statement. This doesn't really need to be an LLM, we could use NLP - # libraries for that, but we have the machinery to use an LLM, so we might as well! - statement_llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - model="claude-3-5-sonnet-20241022", - ) + # This is the LLM that will be used to detect if the user has finished a + # statement. This doesn't really need to be an LLM, we could use NLP + # libraries for that, but we have the machinery to use an LLM, so we might as well! + statement_llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), + model="claude-3-5-sonnet-20241022", + ) - # This is the regular LLM. - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - model="gpt-4o", - ) - # Register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("get_current_weather", fetch_weather_from_api) + # This is the regular LLM. + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o", + ) + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", }, - "required": ["location", "format"], }, + "required": ["location", "format"], }, - ) - ] - - messages = [ - { - "role": "system", - "content": conversational_system_message, }, + ) + ] + + messages = [ + { + "role": "system", + "content": conversational_system_message, + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + # We have instructed the LLM to return 'YES' if it thinks the user + # completed a sentence. So, if it's 'YES' we will return true in this + # predicate which will wake up the notifier. + async def wake_check_filter(frame): + return frame.text == "YES" + + # This is a notifier that we use to synchronize the two LLMs. + notifier = EventNotifier() + + # This turns the LLM context into an inference request to classify the user's speech + # as complete or incomplete. + statement_judge_context_filter = StatementJudgeContextFilter(notifier=notifier) + + # This sends a UserStoppedSpeakingFrame and triggers the notifier event + completeness_check = CompletenessCheck(notifier=notifier) + + # # Notify if the user hasn't said anything. + async def user_idle_notifier(frame): + await notifier.notify() + + # Sometimes the LLM will fail detecting if a user has completed a + # sentence, this will wake up the notifier if that happens. + user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=5.0) + + # We start with the gate open because we send an initial context frame + # to start the conversation. + bot_output_gate = OutputGate(notifier=notifier, start_open=True) + + async def block_user_stopped_speaking(frame): + return not isinstance(frame, UserStoppedSpeakingFrame) + + async def pass_only_llm_trigger_frames(frame): + return ( + isinstance(frame, OpenAILLMContextFrame) + or isinstance(frame, LLMMessagesFrame) + or isinstance(frame, StartInterruptionFrame) + or isinstance(frame, StopInterruptionFrame) + or isinstance(frame, FunctionCallInProgressFrame) + or isinstance(frame, FunctionCallResultFrame) + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + ParallelPipeline( + [ + # Pass everything except UserStoppedSpeaking to the elements after + # this ParallelPipeline + FunctionFilter(filter=block_user_stopped_speaking), + ], + [ + # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed + # LLMMessagesFrame to the statement classifier LLM. The only frame this + # sub-pipeline will output is a UserStoppedSpeakingFrame. + statement_judge_context_filter, + statement_llm, + completeness_check, + ], + [ + # Block everything except OpenAILLMContextFrame and LLMMessagesFrame + FunctionFilter(filter=pass_only_llm_trigger_frames), + llm, + bot_output_gate, # Buffer all llm/tts output until notified. + ], + ), + tts, + user_idle, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - # We have instructed the LLM to return 'YES' if it thinks the user - # completed a sentence. So, if it's 'YES' we will return true in this - # predicate which will wake up the notifier. - async def wake_check_filter(frame): - return frame.text == "YES" + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append( + { + "role": "user", + "content": "Start by just saying \"Hello I'm ready.\" Don't say anything else.", + } + ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - # This is a notifier that we use to synchronize the two LLMs. - notifier = EventNotifier() + @transport.event_handler("on_app_message") + async def on_app_message(transport, message): + logger.debug(f"Received app message: {message}") + if "message" not in message: + return - # This turns the LLM context into an inference request to classify the user's speech - # as complete or incomplete. - statement_judge_context_filter = StatementJudgeContextFilter(notifier=notifier) - - # This sends a UserStoppedSpeakingFrame and triggers the notifier event - completeness_check = CompletenessCheck(notifier=notifier) - - # # Notify if the user hasn't said anything. - async def user_idle_notifier(frame): - await notifier.notify() - - # Sometimes the LLM will fail detecting if a user has completed a - # sentence, this will wake up the notifier if that happens. - user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=5.0) - - # We start with the gate open because we send an initial context frame - # to start the conversation. - bot_output_gate = OutputGate(notifier=notifier, start_open=True) - - async def block_user_stopped_speaking(frame): - return not isinstance(frame, UserStoppedSpeakingFrame) - - async def pass_only_llm_trigger_frames(frame): - return ( - isinstance(frame, OpenAILLMContextFrame) - or isinstance(frame, LLMMessagesFrame) - or isinstance(frame, StartInterruptionFrame) - or isinstance(frame, StopInterruptionFrame) - or isinstance(frame, FunctionCallInProgressFrame) - or isinstance(frame, FunctionCallResultFrame) - ) - - pipeline = Pipeline( + await task.queue_frames( [ - transport.input(), - stt, - context_aggregator.user(), - ParallelPipeline( - [ - # Pass everything except UserStoppedSpeaking to the elements after - # this ParallelPipeline - FunctionFilter(filter=block_user_stopped_speaking), - ], - [ - # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed - # LLMMessagesFrame to the statement classifier LLM. The only frame this - # sub-pipeline will output is a UserStoppedSpeakingFrame. - statement_judge_context_filter, - statement_llm, - completeness_check, - ], - [ - # Block everything except OpenAILLMContextFrame and LLMMessagesFrame - FunctionFilter(filter=pass_only_llm_trigger_frames), - llm, - bot_output_gate, # Buffer all llm/tts output until notified. - ], - ), - tts, - user_idle, - transport.output(), - context_aggregator.assistant(), + UserStartedSpeakingFrame(), + TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]), + UserStoppedSpeakingFrame(), ] ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append( - { - "role": "user", - "content": "Start by just saying \"Hello I'm ready.\" Don't say anything else.", - } - ) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_app_message") - async def on_app_message(transport, message, sender): - logger.debug(f"Received app message: {message} - {sender}") - if "message" not in message: - return + runner = PipelineRunner(handle_sigint=False) - await task.queue_frames( - [ - UserStartedSpeakingFrame(), - TranscriptionFrame( - user_id=sender, timestamp=time.time(), text=message["message"] - ), - UserStoppedSpeakingFrame(), - ] - ) - - runner = PipelineRunner() - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 71bc58782..a0035e11c 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -6,14 +6,11 @@ import asyncio import os -import sys import time -import aiohttp import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( @@ -47,12 +44,12 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") TRANSCRIBER_MODEL = "gemini-2.0-flash-001" CLASSIFIER_MODEL = "gemini-2.0-flash-001" @@ -626,149 +623,155 @@ class OutputGate(FrameProcessor): break -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # This is the LLM that will transcribe user speech. - tx_llm = GoogleLLMService( - name="Transcriber", - model=TRANSCRIBER_MODEL, - api_key=os.getenv("GOOGLE_API_KEY"), - temperature=0.0, - system_instruction=transcriber_system_instruction, - ) + # This is the LLM that will transcribe user speech. + tx_llm = GoogleLLMService( + name="Transcriber", + model=TRANSCRIBER_MODEL, + api_key=os.getenv("GOOGLE_API_KEY"), + temperature=0.0, + system_instruction=transcriber_system_instruction, + ) - # This is the LLM that will classify user speech as complete or incomplete. - classifier_llm = GoogleLLMService( - name="Classifier", - model=CLASSIFIER_MODEL, - api_key=os.getenv("GOOGLE_API_KEY"), - temperature=0.0, - system_instruction=classifier_system_instruction, - ) + # This is the LLM that will classify user speech as complete or incomplete. + classifier_llm = GoogleLLMService( + name="Classifier", + model=CLASSIFIER_MODEL, + api_key=os.getenv("GOOGLE_API_KEY"), + temperature=0.0, + system_instruction=classifier_system_instruction, + ) - # This is the regular LLM that responds conversationally. - conversation_llm = GoogleLLMService( - name="Conversation", - model=CONVERSATION_MODEL, - api_key=os.getenv("GOOGLE_API_KEY"), - system_instruction=conversation_system_instruction, - ) + # This is the regular LLM that responds conversationally. + conversation_llm = GoogleLLMService( + name="Conversation", + model=CONVERSATION_MODEL, + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=conversation_system_instruction, + ) - context = OpenAILLMContext() - context_aggregator = conversation_llm.create_context_aggregator(context) + context = OpenAILLMContext() + context_aggregator = conversation_llm.create_context_aggregator(context) - # This is a notifier that we use to synchronize the two LLMs. - notifier = EventNotifier() + # This is a notifier that we use to synchronize the two LLMs. + notifier = EventNotifier() - # This turns the LLM context into an inference request to classify the user's speech - # as complete or incomplete. - # statement_judge_context_filter = StatementJudgeAudioContextAccumulator(notifier=notifier) + # This turns the LLM context into an inference request to classify the user's speech + # as complete or incomplete. + # statement_judge_context_filter = StatementJudgeAudioContextAccumulator(notifier=notifier) - audio_accumulater = AudioAccumulator() - # This sends a UserStoppedSpeakingFrame and triggers the notifier event - completeness_check = CompletenessCheck( - notifier=notifier, audio_accumulator=audio_accumulater - ) + audio_accumulater = AudioAccumulator() + # This sends a UserStoppedSpeakingFrame and triggers the notifier event + completeness_check = CompletenessCheck(notifier=notifier, audio_accumulator=audio_accumulater) - async def block_user_stopped_speaking(frame): - return not isinstance(frame, UserStoppedSpeakingFrame) + async def block_user_stopped_speaking(frame): + return not isinstance(frame, UserStoppedSpeakingFrame) - conversation_audio_context_assembler = ConversationAudioContextAssembler(context=context) + conversation_audio_context_assembler = ConversationAudioContextAssembler(context=context) - llm_aggregator_buffer = LLMAggregatorBuffer() + llm_aggregator_buffer = LLMAggregatorBuffer() - bot_output_gate = OutputGate( - notifier=notifier, context=context, llm_transcription_buffer=llm_aggregator_buffer - ) + bot_output_gate = OutputGate( + notifier=notifier, context=context, llm_transcription_buffer=llm_aggregator_buffer + ) - pipeline = Pipeline( - [ - transport.input(), - audio_accumulater, - ParallelPipeline( - [ - # Pass everything except UserStoppedSpeaking to the elements after - # this ParallelPipeline - FunctionFilter(filter=block_user_stopped_speaking), - ], - [ - ParallelPipeline( - [ - classifier_llm, - completeness_check, - ], - [ - tx_llm, - llm_aggregator_buffer, - ], - ) - ], - [ - conversation_audio_context_assembler, - conversation_llm, - bot_output_gate, # buffer output until notified, then flush frames and update context - # TempPrinter(), - ], - ), - tts, - transport.output(), - context_aggregator.assistant(), - ], - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_app_message") - async def on_app_message(transport, message, sender): - logger.debug(f"Received app message: {message} - {sender}") - if "message" not in message: - return - - await task.queue_frames( + pipeline = Pipeline( + [ + transport.input(), + audio_accumulater, + ParallelPipeline( [ - UserStartedSpeakingFrame(), - TranscriptionFrame( - user_id=sender, timestamp=time.time(), text=message["message"] - ), - UserStoppedSpeakingFrame(), - ] - ) + # Pass everything except UserStoppedSpeaking to the elements after + # this ParallelPipeline + FunctionFilter(filter=block_user_stopped_speaking), + ], + [ + ParallelPipeline( + [ + classifier_llm, + completeness_check, + ], + [ + tx_llm, + llm_aggregator_buffer, + ], + ) + ], + [ + conversation_audio_context_assembler, + conversation_llm, + bot_output_gate, # buffer output until notified, then flush frames and update context + # TempPrinter(), + ], + ), + tts, + transport.output(), + context_aggregator.assistant(), + ], + ) - runner = PipelineRunner() - await runner.run(task) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_app_message") + async def on_app_message(transport, message): + logger.debug(f"Received app message: {message}") + if "message" not in message: + return + + await task.queue_frames( + [ + UserStartedSpeakingFrame(), + TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]), + UserStoppedSpeakingFrame(), + ] + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound-daily.py similarity index 98% rename from examples/foundational/23-bot-background-sound.py rename to examples/foundational/23-bot-background-sound-daily.py index c58fe5f13..0379b0a11 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound-daily.py @@ -10,9 +10,9 @@ import os import sys import aiohttp +from daily_runner import configure_with_args from dotenv import load_dotenv from loguru import logger -from runner import configure_with_args from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer from pipecat.audio.vad.silero import SileroVADAnalyzer diff --git a/examples/foundational/23-bot-background-sound-p2p.py b/examples/foundational/23-bot-background-sound-p2p.py new file mode 100644 index 000000000..d854770c7 --- /dev/null +++ b/examples/foundational/23-bot-background-sound-p2p.py @@ -0,0 +1,140 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +""" +Usage +----- +Set the path to your background audio file using the `INPUT_AUDIO_PATH` environment variable, then run the bot using: + + INPUT_AUDIO_PATH=path/to/your_audio.mp3 python 23-bot-background-sound.py + +Example: + + INPUT_AUDIO_PATH=my_audio.mp3 python 23-bot-background-sound.py +""" + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import MixerEnableFrame, MixerUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + +audio_path = os.getenv("INPUT_AUDIO_PATH") +if not audio_path: + raise ValueError("No INPUT_AUDIO_PATH specified in environment variables") + + +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + + soundfile_mixer = SoundfileMixer( + sound_files={"office": audio_path}, + default_sound="office", + volume=2.0, + ) + + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_mixer=soundfile_mixer, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT service + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Show how to use mixer control frames. + await asyncio.sleep(10.0) + await task.queue_frame(MixerUpdateSettingsFrame({"volume": 0.5})) + await asyncio.sleep(5.0) + await task.queue_frame(MixerEnableFrame(False)) + await asyncio.sleep(5.0) + await task.queue_frame(MixerEnableFrame(True)) + await asyncio.sleep(5.0) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index 7a0c9777c..fb00c3114 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -6,14 +6,12 @@ import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam -from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -23,13 +21,12 @@ from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFil from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): # Add a delay to test interruption during function calls @@ -39,103 +36,107 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context await result_callback({"conditions": "nice", "temperature": "75"}) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, _) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - # Configure the mute processor with both strategies - stt_mute_processor = STTMuteFilter( - config=STTMuteConfig( - strategies={ - STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE, - STTMuteStrategy.FUNCTION_CALL, - } - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") + # Configure the mute processor with both strategies + stt_mute_processor = STTMuteFilter( + config=STTMuteConfig( + strategies={ + STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE, + STTMuteStrategy.FUNCTION_CALL, + } + ), + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm.register_function("get_current_weather", fetch_weather_from_api) + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, - }, - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm.register_function("get_current_weather", fetch_weather_from_api) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant who can check the weather. Always check the weather when a location is mentioned. Respond concisely and naturally. Your output will be converted to audio so use only simple words and punctuation.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt_mute_processor, # Add the mute processor before STT + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - messages = [ + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation with a weather-related prompt + messages.append( { "role": "system", - "content": "You are a helpful assistant who can check the weather. Always check the weather when a location is mentioned. Respond concisely and naturally. Your output will be converted to audio so use only simple words and punctuation.", - }, - ] - - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt_mute_processor, # Add the mute processor before STT - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] + "content": "Ask the user what city they'd like to know the weather for.", + } ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Kick off the conversation with a weather-related prompt - messages.append( - { - "role": "system", - "content": "Ask the user what city they'd like to know the weather for.", - } - ) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index 80c0545ee..b133086cf 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -4,16 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from dataclasses import dataclass -import aiohttp import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( @@ -37,13 +33,12 @@ from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.frame_processor import FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - # # The system prompt for the main conversation. # @@ -273,102 +268,110 @@ class TranscriptionContextFixup(FrameProcessor): await self.push_frame(frame, direction) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - # No transcription at all. just audio input to Gemini! - # transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + conversation_llm = GoogleLLMService( + name="Conversation", + model="gemini-2.0-flash-001", + # model="gemini-exp-1121", + api_key=os.getenv("GOOGLE_API_KEY"), + # we can give the GoogleLLMService a system instruction to use directly + # in the GenerativeModel constructor. Let's do that rather than put + # our system message in the messages list. + system_instruction=conversation_system_message, + ) + + input_transcription_llm = GoogleLLMService( + name="Transcription", + model="gemini-2.0-flash-001", + # model="gemini-exp-1121", + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=transcriber_system_message, + ) + + messages = [ + { + "role": "user", + "content": "Start by saying hello.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = conversation_llm.create_context_aggregator(context) + audio_collector = UserAudioCollector(context, context_aggregator.user()) + input_transcription_context_filter = InputTranscriptionContextFilter() + transcription_frames_emitter = InputTranscriptionFrameEmitter() + fixup_context_messages = TranscriptionContextFixup(context) + + pipeline = Pipeline( + [ + transport.input(), + audio_collector, + context_aggregator.user(), + ParallelPipeline( + [ # transcribe + input_transcription_context_filter, + input_transcription_llm, + transcription_frames_emitter, + ], + [ # conversation inference + conversation_llm, + ], ), - ) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - conversation_llm = GoogleLLMService( - name="Conversation", - model="gemini-2.0-flash-001", - # model="gemini-exp-1121", - api_key=os.getenv("GOOGLE_API_KEY"), - # we can give the GoogleLLMService a system instruction to use directly - # in the GenerativeModel constructor. Let's do that rather than put - # our system message in the messages list. - system_instruction=conversation_system_message, - ) - - input_transcription_llm = GoogleLLMService( - name="Transcription", - model="gemini-2.0-flash-001", - # model="gemini-exp-1121", - api_key=os.getenv("GOOGLE_API_KEY"), - system_instruction=transcriber_system_message, - ) - - messages = [ - { - "role": "user", - "content": "Start by saying hello.", - }, + tts, + transport.output(), + context_aggregator.assistant(), + fixup_context_messages, ] + ) - context = OpenAILLMContext(messages) - context_aggregator = conversation_llm.create_context_aggregator(context) - audio_collector = UserAudioCollector(context, context_aggregator.user()) - input_transcription_context_filter = InputTranscriptionContextFilter() - transcription_frames_emitter = InputTranscriptionFrameEmitter() - fixup_context_messages = TranscriptionContextFixup(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - audio_collector, - context_aggregator.user(), - ParallelPipeline( - [ # transcribe - input_transcription_context_filter, - input_transcription_llm, - transcription_frames_emitter, - ], - [ # conversation inference - conversation_llm, - ], - ), - tts, - transport.output(), - context_aggregator.assistant(), - fixup_context_messages, - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() + runner = PipelineRunner(handle_sigint=False) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index c26d6ad2f..bdbd8a7e2 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -20,75 +16,100 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +# Load environment variables load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + # Initialize the SmallWebRTCTransport with the connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=False, + vad_enabled=True, + vad_audio_passthrough=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_audio_passthrough=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + # Create the Gemini Multimodal Live LLM service + system_instruction = f""" + You are a helpful AI assistant. + Your goal is to demonstrate your capabilities in a helpful and engaging way. + Your output will be converted to audio so don't include special characters in your answers. + Respond to what the user said in a creative and helpful way. + """ - llm = GeminiMultimodalLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - # system_instruction="Talk like a pirate." - ) + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=system_instruction, + voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck + transcribe_user_audio=True, + ) - pipeline = Pipeline( + # Build the pipeline + pipeline = Pipeline( + [ + transport.input(), + llm, + transport.output(), + ] + ) + + # Configure the pipeline task + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + # Handle client connection event + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames( [ - transport.input(), - llm, - transport.output(), + LLMMessagesAppendFrame( + messages=[ + { + "role": "user", + "content": f"Greet the user and introduce yourself.", + } + ] + ) ] ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + # Handle client disconnection events + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await task.queue_frames( - [ - LLMMessagesAppendFrame( - messages=[ - { - "role": "user", - "content": "Greet the user.", - } - ] - ) - ] - ) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() - - await runner.run(task) + # Run the pipeline + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py index aa07ace93..311052967 100644 --- a/examples/foundational/26a-gemini-multimodal-live-transcription.py +++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -20,90 +16,100 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) + # Initialize the SmallWebRTCTransport with the connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_audio_passthrough=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_audio_passthrough=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede + # system_instruction="Talk like a pirate." + transcribe_user_audio=True, + # inference_on_context_initialization=False, + ) - llm = GeminiMultimodalLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede - # system_instruction="Talk like a pirate." - transcribe_user_audio=True, - transcribe_model_audio=True, - # inference_on_context_initialization=False, - ) + context = OpenAILLMContext( + [ + { + "role": "user", + "content": "Say hello. Then ask if I want to hear a joke.", + }, + # {"role": "assistant", "content": "Hello! Why don't scientists trust atoms?"}, + # { + # "role": "user", + # "content": [ + # { + # "type": "text", + # "text": "Oh, I know this one: because they make up everything.", + # } + # ], + # }, + ], + ) + context_aggregator = llm.create_context_aggregator(context) - context = OpenAILLMContext( - [ - { - "role": "user", - "content": "Say hello. Then ask if I want to hear a joke.", - }, - # {"role": "assistant", "content": "Hello! Why don't scientists trust atoms?"}, - # { - # "role": "user", - # "content": [ - # { - # "type": "text", - # "text": "Oh, I know this one: because they make up everything.", - # } - # ], - # }, - ], - ) - context_aggregator = llm.create_context_aggregator(context) + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + transport.output(), + context_aggregator.assistant(), + ] + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - transport.output(), - context_aggregator.assistant(), - ] - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - runner = PipelineRunner() + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py index 240261d50..612f4313e 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from datetime import datetime -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema @@ -23,13 +19,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): temperature = 75 if args["format"] == "fahrenheit" else 24 @@ -51,87 +46,99 @@ for the weather, call this function. """ -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_audio_passthrough=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + # Initialize the SmallWebRTCTransport with the connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_audio_passthrough=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + ) - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - required=["location", "format"], - ) - search_tool = {"google_search": {}} - tools = ToolsSchema( - standard_tools=[weather_function], custom_tools={AdapterType.GEMINI: [search_tool]} - ) + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + search_tool = {"google_search": {}} + tools = ToolsSchema( + standard_tools=[weather_function], custom_tools={AdapterType.GEMINI: [search_tool]} + ) - llm = GeminiMultimodalLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - system_instruction=system_instruction, - tools=tools, - ) + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=system_instruction, + tools=tools, + ) - llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) - context = OpenAILLMContext( - [{"role": "user", "content": "Say hello."}], - ) - context_aggregator = llm.create_context_aggregator(context) + context = OpenAILLMContext( + [{"role": "user", "content": "Say hello."}], + ) + context_aggregator = llm.create_context_aggregator(context) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - transport.output(), - context_aggregator.assistant(), - ] - ) + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + transport.output(), + context_aggregator.assistant(), + ] + ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - await runner.run(task) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py index 4ec8663c4..70aa3ee95 100644 --- a/examples/foundational/26c-gemini-multimodal-live-video.py +++ b/examples/foundational/26c-gemini-multimodal-live-video.py @@ -9,9 +9,9 @@ import os import sys import aiohttp +from daily_runner import configure from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -53,7 +53,6 @@ async def main(): voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede # system_instruction="Talk like a pirate." transcribe_user_audio=True, - transcribe_model_audio=True, # inference_on_context_initialization=False, ) diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index 150bb1613..f5ee1d9aa 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -25,12 +21,12 @@ from pipecat.services.gemini_multimodal_live.gemini import ( GeminiMultimodalModalities, InputParams, ) -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") SYSTEM_INSTRUCTION = f""" "You are Gemini Chatbot, a friendly, helpful robot. @@ -43,90 +39,95 @@ Respond to what the user said in a creative and helpful way. Keep your responses """ -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_audio_passthrough=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + # Initialize the SmallWebRTCTransport with the connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_audio_passthrough=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + ) - llm = GeminiMultimodalLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - transcribe_user_audio=True, - transcribe_model_audio=True, - system_instruction=SYSTEM_INSTRUCTION, - tools=[{"google_search": {}}, {"code_execution": {}}], - params=InputParams(modalities=GeminiMultimodalModalities.TEXT), - ) + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + transcribe_user_audio=True, + system_instruction=SYSTEM_INSTRUCTION, + tools=[{"google_search": {}}, {"code_execution": {}}], + params=InputParams(modalities=GeminiMultimodalModalities.TEXT), + ) - # Optionally, you can set the response modalities via a function - # llm.set_model_modalities( - # GeminiMultimodalModalities.TEXT - # ) + # Optionally, you can set the response modalities via a function + # llm.set_model_modalities( + # GeminiMultimodalModalities.TEXT + # ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121" - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121" + ) - messages = [ - { - "role": "user", - "content": 'Start by saying "Hello, I\'m Gemini".', - }, + messages = [ + { + "role": "user", + "content": 'Start by saying "Hello, I\'m Gemini".', + }, + ] + + # Set up conversation context and management + # The context_aggregator will automatically collect conversation context + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - # Set up conversation context and management - # The context_aggregator will automatically collect conversation context - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - print(f"Participant left: {participant}") - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-multimodal-google-search.py index 7bfd017d1..432c665f2 100644 --- a/examples/foundational/26e-gemini-multimodal-google-search.py +++ b/examples/foundational/26e-gemini-multimodal-google-search.py @@ -4,12 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -from pathlib import Path -import aiohttp from dotenv import load_dotenv from loguru import logger @@ -19,15 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -sys.path.append(str(Path(__file__).parent.parent)) -from runner import configure +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") # Function handlers for the LLM search_tool = {"google_search": {}} @@ -47,62 +40,73 @@ Start each interaction by asking the user about which place they would like to k """ -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Latest news!", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + # Initialize the SmallWebRTCTransport with the connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_audio_passthrough=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) - # Initialize the Gemini Multimodal Live model - llm = GeminiMultimodalLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck - transcribe_user_audio=True, - transcribe_model_audio=True, - system_instruction=system_instruction, - tools=tools, - ) + # Initialize the Gemini Multimodal Live model + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck + transcribe_user_audio=True, + system_instruction=system_instruction, + tools=tools, + ) - context = OpenAILLMContext( - [ - { - "role": "user", - "content": "Start by greeting the user warmly, introducing yourself, and mentioning the current day. Be friendly and engaging to set a positive tone for the interaction.", - } - ], - ) - context_aggregator = llm.create_context_aggregator(context) + context = OpenAILLMContext( + [ + { + "role": "user", + "content": "Start by greeting the user warmly, introducing yourself, and mentioning the current day. Be friendly and engaging to set a positive tone for the interaction.", + } + ], + ) + context_aggregator = llm.create_context_aggregator(context) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) - runner = PipelineRunner() - await runner.run(task) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index d13ea705a..756ac4d9f 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from simli import SimliConfig from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -20,83 +16,97 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.simli.video import SimliVideoService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") -async def main(): - async with aiohttp.ClientSession() as session: - room, token = await configure(session) - transport = DailyTransport( - room, - token, - "Simli", - DailyParams( - audio_out_enabled=True, - camera_out_enabled=True, - camera_out_width=512, - camera_out_height=512, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - transcription_enabled=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_out_enabled=True, + camera_out_width=512, + camera_out_height=512, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="a167e0f3-df7e-4d52-a9c3-f949145efdab", - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - simli_ai = SimliVideoService( - SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID")) - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="a167e0f3-df7e-4d52-a9c3-f949145efdab", + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") + simli_ai = SimliVideoService( + SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID")) + ) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + simli_ai, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - simli_ai, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/28-transcription-processor.py b/examples/foundational/28-transcription-processor.py index bdb0da104..069235a4e 100644 --- a/examples/foundational/28-transcription-processor.py +++ b/examples/foundational/28-transcription-processor.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys from typing import List, Optional -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame @@ -24,13 +20,12 @@ from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class TranscriptHandler: """Handles real-time transcript processing and output. @@ -93,85 +88,88 @@ class TranscriptHandler: await self.save_message(msg) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - model="gpt-4o", - ) + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o", + ) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript = TranscriptProcessor() + transcript_handler = TranscriptHandler() # Output to log only + # transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - # Create transcript processor and handler - transcript = TranscriptProcessor() - transcript_handler = TranscriptHandler() # Output to log only - # transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - transcript.user(), # User transcripts - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - # Register event handler for transcript updates - @transcript.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - await transcript_handler.on_transcript_update(processor, frame) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - # Stop the pipeline immediately when the participant leaves - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index 82dcff0ef..b53a0a026 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -4,14 +4,10 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( @@ -28,14 +24,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - class DebugObserver(BaseObserver): """Observer to log interruptions and bot speaking events to the console. @@ -71,76 +67,84 @@ class DebugObserver(BaseObserver): logger.info(f"🤖 BOT STOP SPEAKING: {src} {arrow} {dst} at {time_sec:.2f}s") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + observers=[DebugObserver(), LLMLogObserver()], + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - observers=[DebugObserver(), LLMLogObserver()], - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + runner = PipelineRunner(handle_sigint=False) - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/31-heartbeats.py b/examples/foundational/31-heartbeats.py index 2487c13e7..70f6829da 100644 --- a/examples/foundational/31-heartbeats.py +++ b/examples/foundational/31-heartbeats.py @@ -25,11 +25,11 @@ class NullProcessor(FrameProcessor): async def main(): - """This test shows heartbeat monitoring by displaying a warning when - heartbeats are not received. + """This test shows heartbeat monitoring. + A warning is dispalyed when heartbeats are not received within the + default (5 seconds) timeout. """ - pipeline = Pipeline([NullProcessor()]) task = PipelineTask(pipeline, params=PipelineParams(enable_heartbeats=True)) diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 1f7fb1ab3..b517cdd03 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -4,14 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os import sys from pathlib import Path -import aiohttp from dotenv import load_dotenv from loguru import logger +from openai import audio from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame @@ -23,15 +22,14 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService, LLMSearchResponseFrame -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection sys.path.append(str(Path(__file__).parent.parent)) -from runner import configure load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") # Function handlers for the LLM search_tool = {"google_search_retrieval": {}} @@ -61,71 +59,82 @@ class LLMSearchLoggerProcessor(FrameProcessor): await self.push_frame(frame) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Latest news!", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - # Initialize the Gemini Multimodal Live model - llm = GoogleLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - system_instruction=system_instruction, - tools=tools, - model="gemini-1.5-flash-002", - ) + # Initialize the Gemini Multimodal Live model + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=system_instruction, + tools=tools, + model="gemini-1.5-flash-002", + ) - context = OpenAILLMContext( - [ - { - "role": "user", - "content": "Start by greeting the user warmly, introducing yourself, and mentioning the current day. Be friendly and engaging to set a positive tone for the interaction.", - } - ], - ) - context_aggregator = llm.create_context_aggregator(context) + context = OpenAILLMContext( + [ + { + "role": "user", + "content": "Start by greeting the user warmly, introducing yourself, and mentioning the current day. Be friendly and engaging to set a positive tone for the interaction.", + } + ], + ) + context_aggregator = llm.create_context_aggregator(context) - llm_search_logger = LLMSearchLoggerProcessor() + llm_search_logger = LLMSearchLoggerProcessor() - pipeline = Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - llm_search_logger, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + llm_search_logger, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) - runner = PipelineRunner() - await runner.run(task) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/33-gemini-rag.py b/examples/foundational/33-gemini-rag.py index 74905084b..9087ab287 100644 --- a/examples/foundational/33-gemini-rag.py +++ b/examples/foundational/33-gemini-rag.py @@ -47,17 +47,13 @@ Customization options: - change the function calling logic """ -import asyncio import json import os -import sys import time -import aiohttp import google.generativeai as genai from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -65,16 +61,14 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="INFO") - -video_participant_id = None - def get_rag_content(): """Get the RAG content from the file.""" @@ -158,97 +152,106 @@ async def query_knowledge_base( await result_callback(response.text) -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Gemini RAG Bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="f9836c6e-a0bd-460e-9d3c-f7299fa60f94", # Southern Lady - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - llm = GoogleLLMService( - model=VOICE_MODEL, - api_key=os.getenv("GOOGLE_API_KEY"), - ) - llm.register_function("query_knowledge_base", query_knowledge_base) - tools = [ - { - "function_declarations": [ - { - "name": "query_knowledge_base", - "description": "Query the knowledge base for the answer to the question.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question to query the knowledge base with.", - }, + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="f9836c6e-a0bd-460e-9d3c-f7299fa60f94", # Southern Lady + ) + + llm = GoogleLLMService( + model=VOICE_MODEL, + api_key=os.getenv("GOOGLE_API_KEY"), + ) + llm.register_function("query_knowledge_base", query_knowledge_base) + tools = [ + { + "function_declarations": [ + { + "name": "query_knowledge_base", + "description": "Query the knowledge base for the answer to the question.", + "parameters": { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": "The question to query the knowledge base with.", }, }, }, - ], - }, - ] - system_prompt = """\ + }, + ], + }, + ] + system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. You have access to the tool, query_knowledge_base, that allows you to query the knowledge base for the answer to the user's question. Your response will be turned into speech so use only simple words and punctuation. """ - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "Greet the user."}, + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "Greet the user."}, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - global video_participant_id - video_participant_id = participant["id"] - await transport.capture_participant_transcription(participant["id"]) - await transport.capture_participant_video(video_participant_id, framerate=0) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index 09b9e3827..72e4beb9d 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -46,18 +46,14 @@ Note: handling merged and separate audio tracks respectively. """ -import asyncio import datetime import io import os -import sys import wave import aiofiles -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -68,11 +64,11 @@ from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") async def save_audio_file(audio: bytes, filename: str, sample_rate: int, num_channels: int): @@ -89,102 +85,101 @@ async def save_audio_file(audio: bytes, filename: str, sample_rate: int, num_cha logger.info(f"Audio saved to {filename}") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Recording bot", - DailyParams( - # audio_in_enabled=True, - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4") + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4") - # Create audio buffer processor - audiobuffer = AudioBufferProcessor() + # Create audio buffer processor + audiobuffer = AudioBufferProcessor() - messages = [ - { - "role": "system", - "content": "You are a helpful assistant demonstrating audio recording capabilities. Keep your responses brief and clear.", - }, + messages = [ + { + "role": "system", + "content": "You are a helpful assistant demonstrating audio recording capabilities. Keep your responses brief and clear.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + audiobuffer, # Add audio buffer to pipeline + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - pipeline = Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - audiobuffer, # Add audio buffer to pipeline - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start recording audio + await audiobuffer.start_recording() + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await audiobuffer.start_recording() - messages.append( - { - "role": "system", - "content": "Greet the user and explain that this conversation will be recorded.", - } - ) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await audiobuffer.stop_recording() - await task.cancel() + # Handler for merged audio + @audiobuffer.event_handler("on_audio_data") + async def on_audio_data(buffer, audio, sample_rate, num_channels): + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"recordings/merged_{timestamp}.wav" + os.makedirs("recordings", exist_ok=True) + await save_audio_file(audio, filename, sample_rate, num_channels) - # Handler for merged audio - @audiobuffer.event_handler("on_audio_data") - async def on_audio_data(buffer, audio, sample_rate, num_channels): - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - filename = f"recordings/merged_{timestamp}.wav" - os.makedirs("recordings", exist_ok=True) - await save_audio_file(audio, filename, sample_rate, num_channels) + # Handler for separate tracks + @audiobuffer.event_handler("on_track_audio_data") + async def on_track_audio_data(buffer, user_audio, bot_audio, sample_rate, num_channels): + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + os.makedirs("recordings", exist_ok=True) - # Handler for separate tracks - @audiobuffer.event_handler("on_track_audio_data") - async def on_track_audio_data(buffer, user_audio, bot_audio, sample_rate, num_channels): - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - os.makedirs("recordings", exist_ok=True) + # Save user audio + user_filename = f"recordings/user_{timestamp}.wav" + await save_audio_file(user_audio, user_filename, sample_rate, 1) - # Save user audio - user_filename = f"recordings/user_{timestamp}.wav" - await save_audio_file(user_audio, user_filename, sample_rate, 1) + # Save bot audio + bot_filename = f"recordings/bot_{timestamp}.wav" + await save_audio_file(bot_audio, bot_filename, sample_rate, 1) - # Save bot audio - bot_filename = f"recordings/bot_{timestamp}.wav" - await save_audio_file(bot_audio, bot_filename, sample_rate, 1) - - runner = PipelineRunner() - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 29218b2c3..c12dfea72 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -44,14 +44,10 @@ Note: such as formatting instructions, command recognition, or structured data extraction. """ -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -59,14 +55,15 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") # Define voice IDs VOICE_IDS = { @@ -76,57 +73,57 @@ VOICE_IDS = { } -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Multi-voice storyteller", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - # Create pattern pair aggregator for voice switching - pattern_aggregator = PatternPairAggregator() + # Create pattern pair aggregator for voice switching + pattern_aggregator = PatternPairAggregator() - # Add pattern for voice switching - pattern_aggregator.add_pattern_pair( - pattern_id="voice_tag", - start_pattern="", - end_pattern="", - remove_match=True, - ) + # Add pattern for voice switching + pattern_aggregator.add_pattern_pair( + pattern_id="voice_tag", + start_pattern="", + end_pattern="", + remove_match=True, + ) - # Register handler for voice switching - def on_voice_tag(match: PatternMatch): - voice_name = match.content.strip().lower() - if voice_name in VOICE_IDS: - voice_id = VOICE_IDS[voice_name] - tts.set_voice(voice_id) - logger.info(f"Switched to {voice_name} voice") - else: - logger.warning(f"Unknown voice: {voice_name}") + # Register handler for voice switching + def on_voice_tag(match: PatternMatch): + voice_name = match.content.strip().lower() + if voice_name in VOICE_IDS: + voice_id = VOICE_IDS[voice_name] + tts.set_voice(voice_id) + logger.info(f"Switched to {voice_name} voice") + else: + logger.warning(f"Unknown voice: {voice_name}") - pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) + pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) - # Initialize TTS with narrator voice as default - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id=VOICE_IDS["narrator"], - text_aggregator=pattern_aggregator, - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - # Initialize LLM - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # Initialize TTS with narrator voice as default + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id=VOICE_IDS["narrator"], + text_aggregator=pattern_aggregator, + ) - # System prompt for storytelling with voice switching - system_prompt = """You are an engaging storyteller that uses different voices to bring stories to life. + # Initialize LLM + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + # System prompt for storytelling with voice switching + system_prompt = """You are an engaging storyteller that uses different voices to bring stories to life. You have three voices to use, but each has a specific purpose: @@ -173,58 +170,60 @@ FOLLOW THESE RULES: Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue.""" - # Set up LLM context - messages = [ - { - "role": "system", - "content": system_prompt, - }, + # Set up LLM context + messages = [ + { + "role": "system", + "content": system_prompt, + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create pipeline + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, # TTS with pattern aggregator + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - # Create pipeline - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, # TTS with pattern aggregator - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - logger.info(f"First participant joined: {participant['id']}") - await transport.capture_participant_transcription(participant["id"]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - # Start conversation - empty prompt to let LLM follow system instructions - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - logger.info(f"Participant left: {participant['id']}") - await task.cancel() - - logger.info(f"Starting storytelling bot at: {room_url}") - logger.info("Join the room to interact with the bot!") - - runner = PipelineRunner() - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py index 54a5cb480..8cbb826c8 100644 --- a/examples/foundational/36-user-email-gathering.py +++ b/examples/foundational/36-user-email-gathering.py @@ -4,15 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger from openai.types.chat import ChatCompletionToolParam -from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -20,123 +16,133 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.rime.tts import RimeHttpTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - async def store_user_emails(function_name, tool_call_id, args, llm, context, result_callback): print(f"User emails: {args}") -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - # Cartesia offers a `` tags that we can use to ask the user - # to confirm the emails. - # (see https://docs.cartesia.ai/build-with-sonic/formatting-text-for-sonic/spelling-out-input-text) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - aiohttp_session=session, - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - # Rime offers a function `spell()` that we can use to ask the user - # to confirm the emails. - # (see https://docs.rime.ai/api-reference/spell) - # tts = RimeHttpTTSService( - # api_key=os.getenv("RIME_API_KEY", ""), - # voice_id="eva", - # aiohttp_session=session, - # ) + # Cartesia offers a `` tags that we can use to ask the user + # to confirm the emails. + # (see https://docs.cartesia.ai/build-with-sonic/formatting-text-for-sonic/spelling-out-input-text) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - # You can aslo register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - llm.register_function("store_user_emails", store_user_emails) + # Rime offers a function `spell()` that we can use to ask the user + # to confirm the emails. + # (see https://docs.rime.ai/api-reference/spell) + # tts = RimeHttpTTSService( + # api_key=os.getenv("RIME_API_KEY", ""), + # voice_id="eva", + # aiohttp_session=session, + # ) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "store_user_emails", - "description": "Store user emails when confirmed", - "parameters": { - "type": "object", - "properties": { - "emails": { - "type": "array", - "description": "The list of user emails", - "items": {"type": "string"}, - }, + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # You can aslo register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("store_user_emails", store_user_emails) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "store_user_emails", + "description": "Store user emails when confirmed", + "parameters": { + "type": "object", + "properties": { + "emails": { + "type": "array", + "description": "The list of user emails", + "items": {"type": "string"}, }, - "required": ["emails"], }, + "required": ["emails"], }, - ) - ] - messages = [ - { - "role": "system", - # Cartesia - "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", - # Rime spell() - # "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", }, + ) + ] + messages = [ + { + "role": "system", + # Cartesia + "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", + # Rime spell() + # "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - context = OpenAILLMContext(messages, tools) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - runner = PipelineRunner() - - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/37-mem0.py b/examples/foundational/37-mem0.py index 05e576495..8fb8e41f0 100644 --- a/examples/foundational/37-mem0.py +++ b/examples/foundational/37-mem0.py @@ -36,14 +36,11 @@ Requirements: The bot runs as part of a pipeline that processes audio frames and manages the conversation flow. """ -import asyncio import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger -from runner import configure +from openai import audio from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -51,13 +48,13 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor +from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.mem0.memory import Mem0MemoryService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) @@ -115,7 +112,7 @@ async def get_initial_greeting( return "Hello! How can I help you today?" -async def main(): +async def run_bot(webrtc_connection: SmallWebRTCConnection): """Main bot execution function. Sets up and runs the bot pipeline including: @@ -127,116 +124,121 @@ async def main(): """ # Note: You can pass the user_id as a parameter in API call USER_ID = "pipecat-demo-user" - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - # Set up Daily transport with video/audio parameters - transport = DailyTransport( - room_url, - token, - "Chatbot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - transcription_enabled=True, - ), - ) + logger.info(f"Starting bot") - # Initialize text-to-speech service - tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id="pNInz6obpgDQGcFmaJgB", - ) + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) - # Initialize Mem0 memory service - memory = Mem0MemoryService( - api_key=os.getenv("MEM0_API_KEY"), - user_id=USER_ID, # Unique identifier for the user - # agent_id="agent1", # Optional identifier for the agent - # run_id="session1", # Optional identifier for the run - params=Mem0MemoryService.InputParams( - search_limit=10, - search_threshold=0.3, - api_version="v2", - system_prompt="Based on previous conversations, I recall: \n\n", - add_as_system_message=True, - position=1, - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - # Initialize LLM service - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") + # Initialize text-to-speech service + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="pNInz6obpgDQGcFmaJgB", + ) - messages = [ - { - "role": "system", - "content": """You are a personal assistant. You can remember things about the person you are talking to. - Some Guidelines: - - Make sure your responses are friendly yet short and concise. - - If the user asks you to remember something, make sure to remember it. - - Greet the user by their name if you know about it. - """, - }, + # Initialize Mem0 memory service + memory = Mem0MemoryService( + api_key=os.getenv("MEM0_API_KEY"), + user_id=USER_ID, # Unique identifier for the user + # agent_id="agent1", # Optional identifier for the agent + # run_id="session1", # Optional identifier for the run + params=Mem0MemoryService.InputParams( + search_limit=10, + search_threshold=0.3, + api_version="v2", + system_prompt="Based on previous conversations, I recall: \n\n", + add_as_system_message=True, + position=1, + ), + ) + + # Initialize LLM service + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") + + messages = [ + { + "role": "system", + "content": """You are a personal assistant. You can remember things about the person you are talking to. + Some Guidelines: + - Make sure your responses are friendly yet short and concise. + - If the user asks you to remember something, make sure to remember it. + - Greet the user by their name if you know about it. + """, + }, + ] + + # Set up conversation context and management + # The context_aggregator will automatically collect conversation context + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + + pipeline = Pipeline( + [ + transport.input(), + rtvi, + stt, + context_aggregator.user(), + memory, + llm, + tts, + transport.output(), + context_aggregator.assistant(), ] + ) - # Set up conversation context and management - # The context_aggregator will automatically collect conversation context - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + observers=[RTVIObserver(rtvi)], + ) - pipeline = Pipeline( - [ - transport.input(), - rtvi, - context_aggregator.user(), - memory, - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + await rtvi.set_bot_ready() + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Get personalized greeting based on user memories. Can pass agent_id and run_id as per requirement of the application to manage short term memory or agent specific memory. + greeting = await get_initial_greeting( + memory_client=memory.memory_client, user_id=USER_ID, agent_id=None, run_id=None ) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - observers=[RTVIObserver(rtvi)], - ) + # Add the greeting as an assistant message to start the conversation + context.add_message({"role": "assistant", "content": greeting}) - @rtvi.event_handler("on_client_ready") - async def on_client_ready(rtvi): - await rtvi.set_bot_ready() + # Queue the context frame to start the conversation + await task.queue_frames([context_aggregator.user().get_context_frame()]) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") - # Get personalized greeting based on user memories. Can pass agent_id and run_id as per requirement of the application to manage short term memory or agent specific memory. - greeting = await get_initial_greeting( - memory_client=memory.memory_client, user_id=USER_ID, agent_id=None, run_id=None - ) + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() - # Add the greeting as an assistant message to start the conversation - context.add_message({"role": "assistant", "content": greeting}) - - # Queue the context frame to start the conversation - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - print(f"Participant left: {participant}") - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from run import main + + main() diff --git a/examples/foundational/README.md b/examples/foundational/README.md new file mode 100644 index 000000000..622c06875 --- /dev/null +++ b/examples/foundational/README.md @@ -0,0 +1,61 @@ +# Pipecat Foundational Examples + +This directory contains foundational examples showing how to use Pipecat to build voice and multimodal agents. Each example demonstrates specific features of the framework, building from basic to more complex concepts. + +## Prerequisites + +1. If you haven't already, set up a virtual environment: + + ```bash + python -m venv venv + source venv/bin/activate + ``` + +2. Install Pipecat with the required dependencies: + + ```bash + pip install -r requirements.txt + ``` + +3. Set up an `.env` file with the API keys of services you'll run. + +## Running the examples + +Each example is a self-contained bot that runs using our built-in `run.py` FastAPI server. The server automatically starts a web interface ([Pipecat SmallWebRTC Prebuilt](https://pypi.org/project/pipecat-ai-small-webrtc-prebuilt/)) that allows you to interact with the bot via WebRTC in your browser. + +1. **Run a specific example**: + + ```bash + python + ``` + + For example: + + ```bash + python 07-interruptible.py + ``` + +2. **Open the web app** at the URL displayed in the console: + + ``` + Open your browser to: http://localhost:7860 + ``` + +3. **Start the example**: + + Click the "Connect" button in the web interface, grant camera/microphone permissions when prompted, and start interacting with the bot. + +## Troubleshooting + +- **No audio or video**: Make sure your browser permissions for microphone and camera are granted +- **Connection errors**: Check that your API keys are correctly set in the `.env` file +- **Missing dependencies**: Ensure you've installed all required dependencies with `pip install -r requirements.txt` +- **Port already in use**: Change the port with `--port ` if the default port is unavailable + +### Customizing the network interface + +If you have conflict on a host or port, you can customize using: + +```bash +python --host 0.0.0.0 --port 8080 +``` diff --git a/examples/foundational/runner.py b/examples/foundational/daily_runner.py similarity index 100% rename from examples/foundational/runner.py rename to examples/foundational/daily_runner.py diff --git a/examples/foundational/requirements.txt b/examples/foundational/requirements.txt new file mode 100644 index 000000000..9f352efbc --- /dev/null +++ b/examples/foundational/requirements.txt @@ -0,0 +1,5 @@ +fastapi +uvicorn +python-dotenv +pipecat-ai[webrtc] +pipecat-ai-small-webrtc-prebuilt \ No newline at end of file diff --git a/examples/foundational/run.py b/examples/foundational/run.py new file mode 100644 index 000000000..213eb14ce --- /dev/null +++ b/examples/foundational/run.py @@ -0,0 +1,202 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import asyncio +import importlib.util +import logging +import os +import sys +from contextlib import asynccontextmanager +from inspect import iscoroutinefunction, signature +from typing import Any, Callable, Dict, Optional, Tuple + +import uvicorn +from dotenv import load_dotenv +from fastapi import BackgroundTasks, FastAPI +from fastapi.responses import RedirectResponse +from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI + +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +# Load environment variables +load_dotenv(override=True) + +# Configure logger +logging.basicConfig( + level=logging.INFO, + format="%(message)s", + handlers=[logging.StreamHandler()], +) +logger = logging.getLogger("pipecat-server") + +app = FastAPI() + +# Store connections by pc_id +pcs_map: Dict[str, SmallWebRTCConnection] = {} + +ice_servers = ["stun:stun.l.google.com:19302"] + +# Mount the frontend at / +app.mount("/client", SmallWebRTCPrebuiltUI) + +# Store the bot module and function info +bot_module: Any = None +run_bot_func: Optional[Callable] = None +is_webrtc_bot: bool = True + + +def import_bot_file(file_path: str) -> Tuple[Any, Callable, bool]: + """Dynamically import the bot file and determine how to run it. + + Returns: + tuple: (module, run_function, is_webrtc_bot) + - module: The imported module + - run_function: Either run_bot or main function + - is_webrtc_bot: True if run_bot function exists and accepts a WebRTC connection + """ + if not os.path.exists(file_path): + raise FileNotFoundError(f"Bot file not found: {file_path}") + + # Extract module name without extension + module_name = os.path.splitext(os.path.basename(file_path))[0] + + # Load the module + spec = importlib.util.spec_from_file_location(module_name, file_path) + if not spec or not spec.loader: + raise ImportError(f"Could not load spec for {file_path}") + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + + # Check for run_bot function first + if hasattr(module, "run_bot"): + run_func = module.run_bot + # Check if the function accepts a WebRTC connection + sig = signature(run_func) + is_webrtc = len(sig.parameters) > 0 + return module, run_func, is_webrtc + + # Fall back to main function + if hasattr(module, "main") and iscoroutinefunction(module.main): + return module, module.main, False + + raise AttributeError(f"No run_bot or async main function found in {file_path}") + + +@app.get("/", include_in_schema=False) +async def root_redirect(): + return RedirectResponse(url="/client/") + + +@app.post("/api/offer") +async def offer(request: dict, background_tasks: BackgroundTasks): + global run_bot_func, is_webrtc_bot + + if not run_bot_func: + raise RuntimeError("No bot file has been loaded") + + if not is_webrtc_bot: + return { + "error": "This bot doesn't support WebRTC connections, it's running in standalone mode" + } + + pc_id = request.get("pc_id") + + if pc_id and pc_id in pcs_map: + pipecat_connection = pcs_map[pc_id] + logger.info(f"Reusing existing connection for pc_id: {pc_id}") + await pipecat_connection.renegotiate( + sdp=request["sdp"], type=request["type"], restart_pc=request.get("restart_pc", False) + ) + else: + pipecat_connection = SmallWebRTCConnection(ice_servers) + await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"]) + + @pipecat_connection.event_handler("closed") + async def handle_disconnected(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}") + pcs_map.pop(webrtc_connection.pc_id, None) + + # We've already checked that run_bot_func exists + assert run_bot_func is not None + background_tasks.add_task(run_bot_func, pipecat_connection) + + answer = pipecat_connection.get_answer() + # Updating the peer connection inside the map + pcs_map[answer["pc_id"]] = pipecat_connection + + return answer + + +@asynccontextmanager +async def lifespan(app: FastAPI): + yield # Run app + coros = [pc.close() for pc in pcs_map.values()] + await asyncio.gather(*coros) + pcs_map.clear() + + +async def run_standalone_bot() -> None: + """Run a standalone bot that doesn't require WebRTC""" + global run_bot_func + if run_bot_func is not None: + await run_bot_func() + else: + raise RuntimeError("No bot function available to run") + + +def main(): + parser = argparse.ArgumentParser(description="Pipecat Bot Runner") + parser.add_argument("bot_file", nargs="?", help="Path to the bot file", default=None) + parser.add_argument( + "--host", default="localhost", help="Host for HTTP server (default: localhost)" + ) + parser.add_argument( + "--port", type=int, default=7860, help="Port for HTTP server (default: 7860)" + ) + parser.add_argument("--verbose", "-v", action="count", default=0) + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + # Infer the bot file from the caller if not provided explicitly + bot_file = args.bot_file + if bot_file is None: + # Get the __file__ of the script that called main() + import inspect + + caller_frame = inspect.stack()[1] + caller_globals = caller_frame.frame.f_globals + bot_file = caller_globals.get("__file__") + + if not bot_file: + print("❌ Could not determine the bot file. Pass it explicitly to main().") + sys.exit(1) + + # Import the bot file + try: + global run_bot_func, bot_module, is_webrtc_bot + bot_module, run_bot_func, is_webrtc_bot = import_bot_file(bot_file) + logger.info(f"Successfully loaded bot from {bot_file}") + + if is_webrtc_bot: + logger.info("Detected WebRTC-compatible bot, starting web server...") + uvicorn.run(app, host=args.host, port=args.port) + else: + logger.info("Detected standalone bot, running directly...") + asyncio.run(run_standalone_bot()) + except Exception as e: + logger.error(f"Error loading bot file: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/instant-voice/server/src/single_bot.py b/examples/instant-voice/server/src/single_bot.py index f1a44fd77..fbd3a5fc9 100644 --- a/examples/instant-voice/server/src/single_bot.py +++ b/examples/instant-voice/server/src/single_bot.py @@ -69,7 +69,6 @@ async def main(): api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck transcribe_user_audio=True, - transcribe_model_audio=True, system_instruction=SYSTEM_INSTRUCTION, ) diff --git a/examples/p2p-webrtc/video-transform/server/bot.py b/examples/p2p-webrtc/video-transform/server/bot.py index d65bb0a64..7d4a801b4 100644 --- a/examples/p2p-webrtc/video-transform/server/bot.py +++ b/examples/p2p-webrtc/video-transform/server/bot.py @@ -93,7 +93,6 @@ async def run_bot(webrtc_connection): api_key=os.getenv("GOOGLE_API_KEY"), voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck transcribe_user_audio=True, - transcribe_model_audio=True, system_instruction=SYSTEM_INSTRUCTION, ) diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index 753cb2af7..0bab332d6 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -136,7 +136,6 @@ async def main(): api_key=os.getenv("GEMINI_API_KEY"), voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck transcribe_user_audio=True, - transcribe_model_audio=True, ) messages = [ diff --git a/src/pipecat/transports/network/small_webrtc.py b/src/pipecat/transports/network/small_webrtc.py index 2b4583288..343469a71 100644 --- a/src/pipecat/transports/network/small_webrtc.py +++ b/src/pipecat/transports/network/small_webrtc.py @@ -20,7 +20,9 @@ from pipecat.frames.frames import ( Frame, InputAudioRawFrame, InputImageRawFrame, + OutputAudioRawFrame, OutputImageRawFrame, + SpriteFrame, StartFrame, TransportMessageFrame, TransportMessageUrgentFrame, @@ -524,10 +526,8 @@ class SmallWebRTCTransport(BaseTransport): self._client = SmallWebRTCClient(webrtc_connection, self._callbacks) - self._input = SmallWebRTCInputTransport(self._client, self._params, name=self._input_name) - self._output = SmallWebRTCOutputTransport( - self._client, self._params, name=self._output_name - ) + self._input: Optional[SmallWebRTCInputTransport] = None + self._output: Optional[SmallWebRTCOutputTransport] = None # Register supported handlers. The user will only be able to register # these handlers. @@ -550,6 +550,14 @@ class SmallWebRTCTransport(BaseTransport): ) return self._output + async def send_image(self, frame: OutputImageRawFrame | SpriteFrame): + if self._output: + await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM) + + async def send_audio(self, frame: OutputAudioRawFrame): + if self._output: + await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM) + async def _on_app_message(self, message: Any): if self._input: await self._input.push_app_message(message)