Updating foundation examples to use SmallWebRTCTransport and pipecat-ai-small-webrtc-prebuilt (#1534)

Co-authored-by: Filipi Fuchter <filipi@daily.co>
This commit is contained in:
Mark Backman
2025-04-11 19:44:16 -04:00
committed by GitHub
parent 8186219879
commit f6accbd510
120 changed files with 7989 additions and 7179 deletions

View File

@@ -4,54 +4,54 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.frames.frames import EndFrame, TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.services.piper.tts import PiperTTSService from pipecat.services.piper.tts import PiperTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Create a transport using the WebRTC connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, _) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( audio_out_enabled=True,
room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True) ),
) )
# Create an HTTP session
async with aiohttp.ClientSession() as session:
tts = PiperTTSService( tts = PiperTTSService(
base_url=os.getenv("PIPER_BASE_URL"), aiohttp_session=session, sample_rate=24000 base_url=os.getenv("PIPER_BASE_URL"), aiohttp_session=session, sample_rate=24000
) )
runner = PipelineRunner()
task = PipelineTask(Pipeline([tts, transport.output()])) task = PipelineTask(Pipeline([tts, transport.output()]))
# Register an event handler so we can play the audio when the # Register an event handler so we can play the audio when the client joins
# participant joins. @transport.event_handler("on_client_connected")
@transport.event_handler("on_first_participant_joined") async def on_client_connected(transport, client):
async def on_first_participant_joined(transport, participant): await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
await task.queue_frames(
[TTSSpeakFrame(f"Hello there, how are you today ?"), EndFrame()] runner = PipelineRunner(handle_sigint=False)
)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -0,0 +1,59 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import os
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.services.rime.tts import RimeHttpTTSService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True)
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
# Create a transport using the WebRTC connection
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_out_enabled=True,
),
)
# Create an HTTP session
async with aiohttp.ClientSession() as session:
tts = RimeHttpTTSService(
api_key=os.getenv("RIME_API_KEY", ""),
voice_id="rex",
aiohttp_session=session,
)
task = PipelineTask(Pipeline([tts, transport.output()]))
# Register an event handler so we can play the audio when the client joins
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
if __name__ == "__main__":
from run import main
main()

View File

@@ -4,34 +4,32 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.frames.frames import EndFrame, TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Create a transport using the WebRTC connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, _) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( audio_out_enabled=True,
room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True) ),
) )
tts = CartesiaTTSService( tts = CartesiaTTSService(
@@ -39,21 +37,19 @@ async def main():
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
) )
runner = PipelineRunner()
task = PipelineTask(Pipeline([tts, transport.output()])) task = PipelineTask(Pipeline([tts, transport.output()]))
# Register an event handler so we can play the audio when the # Register an event handler so we can play the audio when the client joins
# participant joins. @transport.event_handler("on_client_connected")
@transport.event_handler("on_first_participant_joined") async def on_client_connected(transport, client):
async def on_first_participant_joined(transport, participant): await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
participant_name = participant.get("info", {}).get("userName", "")
await task.queue_frames( runner = PipelineRunner(handle_sigint=False)
[TTSSpeakFrame(f"Hello there, {participant_name}!"), EndFrame()]
)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,51 +4,49 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.frames.frames import EndFrame, TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.services.riva.tts import FastPitchTTSService from pipecat.services.riva.tts import FastPitchTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Create a transport using the WebRTC connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, _) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( audio_out_enabled=True,
room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True) ),
) )
tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
runner = PipelineRunner()
task = PipelineTask(Pipeline([tts, transport.output()])) task = PipelineTask(Pipeline([tts, transport.output()]))
# Register an event handler so we can play the audio when the # Register an event handler so we can play the audio when the client joins
# participant joins. @transport.event_handler("on_client_connected")
@transport.event_handler("on_first_participant_joined") async def on_client_connected(transport, client):
async def on_first_participant_joined(transport, participant): await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
participant_name = participant.get("info", {}).get("userName", "")
await task.queue_frames([TTSSpeakFrame(f"Aloha, {participant_name}!"), EndFrame()]) runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.frames.frames import EndFrame, LLMMessagesFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -19,20 +15,22 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Create a transport using the WebRTC connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, _) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( audio_out_enabled=True,
room_url, None, "Say One Thing From an LLM", DailyParams(audio_out_enabled=True) ),
) )
tts = CartesiaTTSService( tts = CartesiaTTSService(
@@ -49,16 +47,19 @@ async def main():
} }
] ]
runner = PipelineRunner()
task = PipelineTask(Pipeline([llm, tts, transport.output()])) task = PipelineTask(Pipeline([llm, tts, transport.output()]))
@transport.event_handler("on_first_participant_joined") # Register an event handler so we can play the audio when the client joins
async def on_first_participant_joined(transport, participant): @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
await task.queue_frames([LLMMessagesFrame(messages), EndFrame()]) await task.queue_frames([LLMMessagesFrame(messages), EndFrame()])
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,59 +4,67 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import TextFrame from pipecat.frames.frames import TextFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.services.fal.image import FalImageGenService from pipecat.services.fal.image import FalImageGenService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Create a transport using the WebRTC connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, _) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( camera_out_enabled=True,
room_url, camera_out_width=1024,
None, camera_out_height=1024,
"Show a still frame image", ),
DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024),
) )
# Create an HTTP session
async with aiohttp.ClientSession() as session:
imagegen = FalImageGenService( imagegen = FalImageGenService(
params=FalImageGenService.InputParams(image_size="square_hd"), params=FalImageGenService.InputParams(image_size="square_hd"),
aiohttp_session=session, aiohttp_session=session,
key=os.getenv("FAL_KEY"), key=os.getenv("FAL_KEY"),
) )
runner = PipelineRunner()
task = PipelineTask(Pipeline([imagegen, transport.output()])) task = PipelineTask(Pipeline([imagegen, transport.output()]))
@transport.event_handler("on_first_participant_joined") # Register an event handler so we can play the audio when the client joins
async def on_first_participant_joined(transport, participant): @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
await task.queue_frame(TextFrame("a cat in the style of picasso")) await task.queue_frame(TextFrame("a cat in the style of picasso"))
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,62 +4,67 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import EndFrame, TextFrame from pipecat.frames.frames import TextFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.services.google.image import GoogleImageGenService from pipecat.services.google.image import GoogleImageGenService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Create a transport using the WebRTC connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, _) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( camera_out_enabled=True,
room_url, camera_out_width=1024,
None, camera_out_height=1024,
"Show a still frame image", ),
DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024),
) )
imagegen = GoogleImageGenService( imagegen = GoogleImageGenService(
api_key=os.getenv("GOOGLE_API_KEY"), api_key=os.getenv("GOOGLE_API_KEY"),
) )
runner = PipelineRunner()
task = PipelineTask( task = PipelineTask(
Pipeline([imagegen, transport.output()]), Pipeline([imagegen, transport.output()]),
params=PipelineParams(enable_metrics=True), params=PipelineParams(enable_metrics=True),
) )
@transport.event_handler("on_first_participant_joined") # Register an event handler so we can play the audio when the client joins
async def on_first_participant_joined(transport, participant): @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
await task.queue_frame(TextFrame("a cat in the style of picasso")) await task.queue_frame(TextFrame("a cat in the style of picasso"))
await task.queue_frame(TextFrame("a dog in the style of picasso")) await task.queue_frame(TextFrame("a dog in the style of picasso"))
await task.queue_frame(TextFrame("a fish in the style of picasso")) await task.queue_frame(TextFrame("a fish in the style of picasso"))
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
await task.queue_frame(EndFrame()) logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -13,9 +13,9 @@ import os
import sys import sys
import aiohttp import aiohttp
from daily_runner import configure
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
from pipecat.pipeline.merge_pipeline import SequentialMergePipeline from pipecat.pipeline.merge_pipeline import SequentialMergePipeline

View File

@@ -4,15 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from dataclasses import dataclass from dataclasses import dataclass
import aiohttp import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import ( from pipecat.frames.frames import (
DataFrame, DataFrame,
@@ -30,13 +27,12 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.cartesia.tts import CartesiaHttpTTSService from pipecat.services.cartesia.tts import CartesiaHttpTTSService
from pipecat.services.fal.image import FalImageGenService from pipecat.services.fal.image import FalImageGenService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
@dataclass @dataclass
class MonthFrame(DataFrame): class MonthFrame(DataFrame):
@@ -67,15 +63,19 @@ class MonthPrepender(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: """Run the Calendar Month Narration bot using WebRTC transport.
(room_url, _) = await configure(session)
transport = DailyTransport( Args:
room_url, webrtc_connection: The WebRTC connection to use
None, room_name: Optional room name for display purposes
"Month Narration Bot", """
DailyParams( logger.info(f"Starting bot")
# Create a transport using the WebRTC connection
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_out_enabled=True, audio_out_enabled=True,
camera_out_enabled=True, camera_out_enabled=True,
camera_out_width=1024, camera_out_width=1024,
@@ -83,6 +83,8 @@ async def main():
), ),
) )
# Create an HTTP session for API calls
async with aiohttp.ClientSession() as session:
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
tts = CartesiaHttpTTSService( tts = CartesiaHttpTTSService(
@@ -144,14 +146,30 @@ async def main():
frames.append(MonthFrame(month=month)) frames.append(MonthFrame(month=month))
frames.append(LLMMessagesFrame(messages)) frames.append(LLMMessagesFrame(messages))
runner = PipelineRunner()
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
# Set up transport event handlers
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Start the month narration once connected
await task.queue_frames(frames) await task.queue_frames(frames)
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
# Run the pipeline
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import Frame, MetricsFrame from pipecat.frames.frames import Frame, MetricsFrame
@@ -27,14 +23,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class MetricsLogger(FrameProcessor): class MetricsLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -56,22 +52,22 @@ class MetricsLogger(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -94,6 +90,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -111,21 +108,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from PIL import Image from PIL import Image
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import ( from pipecat.frames.frames import (
@@ -20,7 +16,6 @@ from pipecat.frames.frames import (
BotStoppedSpeakingFrame, BotStoppedSpeakingFrame,
Frame, Frame,
OutputImageRawFrame, OutputImageRawFrame,
TextFrame,
) )
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -28,14 +23,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class ImageSyncAggregator(FrameProcessor): class ImageSyncAggregator(FrameProcessor):
def __init__(self, speaking_path: str, waiting_path: str): def __init__(self, speaking_path: str, waiting_path: str):
@@ -72,25 +67,25 @@ class ImageSyncAggregator(FrameProcessor):
await self.push_frame(frame) await self.push_frame(frame)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
camera_out_enabled=True, camera_out_enabled=True,
camera_out_width=1024, camera_out_width=1024,
camera_out_height=1024, camera_out_height=1024,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -116,6 +111,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -135,20 +131,26 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
participant_name = participant.get("info", {}).get("userName", "") logger.info(f"Client connected")
await transport.capture_participant_transcription(participant["id"]) # Kick off the conversation.
await task.queue_frames([TextFrame(f"Hi there {participant_name}!")]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -1,104 +0,0 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.audio.vad.silero import SileroVAD
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def main():
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
transcription_enabled=True,
),
)
vad = SileroVAD()
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(),
vad,
context_aggregator.user(),
llm,
tts,
transport.output(),
context_aggregator.assistant(),
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
),
)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await transport.capture_participant_transcription(participant["id"])
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
await task.cancel()
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -19,31 +15,31 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -64,6 +60,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -82,21 +79,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -1,106 +0,0 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.anthropic.llm import AnthropicLLMService
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def main():
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
)
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = AnthropicLLMService(
api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-opus-20240229"
)
# todo: think more about how to handle system prompts in a more general way. OpenAI,
# Google, and Anthropic all have slightly different approaches to providing a system
# prompt.
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(), # Transport user input
context_aggregator.user(), # User responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
),
)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await transport.capture_participant_transcription(participant["id"])
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
await task.cancel()
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -0,0 +1,106 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.audio.vad.silero import SileroVAD
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True)
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
),
)
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
vad = SileroVAD()
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(),
stt,
vad,
context_aggregator.user(),
llm,
tts,
transport.output(),
context_aggregator.assistant(),
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
),
)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
if __name__ == "__main__":
from run import main
main()

View File

@@ -4,11 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain_community.chat_message_histories import ChatMessageHistory from langchain_community.chat_message_histories import ChatMessageHistory
@@ -16,7 +13,6 @@ from langchain_core.chat_history import BaseChatMessageHistory
from langchain_core.runnables.history import RunnableWithMessageHistory from langchain_core.runnables.history import RunnableWithMessageHistory
from langchain_openai import ChatOpenAI from langchain_openai import ChatOpenAI
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import LLMMessagesFrame from pipecat.frames.frames import LLMMessagesFrame
@@ -29,14 +25,14 @@ from pipecat.processors.aggregators.llm_response import (
) )
from pipecat.processors.frameworks.langchain import LangchainProcessor from pipecat.processors.frameworks.langchain import LangchainProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
message_store = {} message_store = {}
@@ -46,22 +42,22 @@ def get_session_history(session_id: str) -> BaseChatMessageHistory:
return message_store[session_id] return message_store[session_id]
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -93,6 +89,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
tma_in, # User responses tma_in, # User responses
lc, # Langchain lc, # Langchain
tts, # TTS tts, # TTS
@@ -111,10 +108,9 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
lc.set_participant_id(participant["id"])
# Kick off the conversation. # Kick off the conversation.
# the `LLMMessagesFrame` will be picked up by the LangchainProcessor using # the `LLMMessagesFrame` will be picked up by the LangchainProcessor using
# only the content of the last message to inject it in the prompt defined # only the content of the last message to inject it in the prompt defined
@@ -122,14 +118,21 @@ async def main():
messages = [({"content": "Please briefly introduce yourself to the user."})] messages = [({"content": "Please briefly introduce yourself to the user."})]
await task.queue_frames([LLMMessagesFrame(messages)]) await task.queue_frames([LLMMessagesFrame(messages)])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from deepgram import LiveOptions from deepgram import LiveOptions
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import ( from pipecat.frames.frames import (
BotInterruptionFrame, BotInterruptionFrame,
@@ -27,23 +23,19 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.deepgram.tts import DeepgramTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, _) = await configure(session) params=TransportParams(
transport = DailyTransport(
room_url,
None,
"Respond bot",
DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
), ),
@@ -98,20 +90,28 @@ async def main():
async def on_utterance_end(stt, *args, **kwargs): async def on_utterance_end(stt, *args, **kwargs):
await task.queue_frames([StopInterruptionFrame(), UserStoppedSpeakingFrame()]) await task.queue_frames([StopInterruptionFrame(), UserStoppedSpeakingFrame()])
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.deepgram.tts import DeepgramTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, _) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
None,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -83,20 +76,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,45 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
# Create an HTTP session
async with aiohttp.ClientSession() as session:
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = ElevenLabsHttpTTSService( tts = ElevenLabsHttpTTSService(
api_key=os.getenv("ELEVENLABS_API_KEY", ""), api_key=os.getenv("ELEVENLABS_API_KEY", ""),
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
@@ -65,6 +64,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -83,21 +83,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
api_key=os.getenv("ELEVENLABS_API_KEY", ""), api_key=os.getenv("ELEVENLABS_API_KEY", ""),
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
@@ -64,6 +60,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -82,21 +79,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.playht.tts import PlayHTHttpTTSService from pipecat.services.playht.tts import PlayHTHttpTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = PlayHTHttpTTSService( tts = PlayHTHttpTTSService(
user_id=os.getenv("PLAYHT_USER_ID"), user_id=os.getenv("PLAYHT_USER_ID"),
api_key=os.getenv("PLAYHT_API_KEY"), api_key=os.getenv("PLAYHT_API_KEY"),
@@ -65,6 +61,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -83,21 +80,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,47 +4,43 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.playht.tts import PlayHTTTSService from pipecat.services.playht.tts import PlayHTTTSService
from pipecat.transcriptions.language import Language from pipecat.transcriptions.language import Language
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = PlayHTTTSService( tts = PlayHTTTSService(
user_id=os.getenv("PLAYHT_USER_ID"), user_id=os.getenv("PLAYHT_USER_ID"),
api_key=os.getenv("PLAYHT_API_KEY"), api_key=os.getenv("PLAYHT_API_KEY"),
@@ -67,6 +63,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -85,21 +82,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.azure.llm import AzureLLMService from pipecat.services.azure.llm import AzureLLMService
from pipecat.services.azure.stt import AzureSTTService from pipecat.services.azure.stt import AzureSTTService
from pipecat.services.azure.tts import AzureTTSService from pipecat.services.azure.tts import AzureTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -93,21 +86,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -21,38 +17,27 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.openai.stt import OpenAISTTService from pipecat.services.openai.stt import OpenAISTTService
from pipecat.services.openai.tts import OpenAITTSService from pipecat.services.openai.tts import OpenAITTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
audio_out_sample_rate=24000,
transcription_enabled=False,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True, vad_audio_passthrough=True,
), ),
) )
# You can use the OpenAI compatible API like Groq.
# stt = OpenAISTTService(
# base_url="https://api.groq.com/openai/v1",
# api_key="gsk_***",
# model="whisper-large-v3",
# )
stt = OpenAISTTService( stt = OpenAISTTService(
api_key=os.getenv("OPENAI_API_KEY"), api_key=os.getenv("OPENAI_API_KEY"),
model="gpt-4o-transcribe-latest", model="gpt-4o-transcribe-latest",
@@ -89,27 +74,35 @@ async def main():
pipeline, pipeline,
params=PipelineParams( params=PipelineParams(
allow_interruptions=True, allow_interruptions=True,
audio_out_sample_rate=24000,
enable_metrics=True, enable_metrics=True,
enable_usage_metrics=True, enable_usage_metrics=True,
report_only_initial_ttfb=True, report_only_initial_ttfb=True,
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import time import time
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -20,31 +16,31 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openpipe.llm import OpenPipeLLMService from pipecat.services.openpipe.llm import OpenPipeLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -71,6 +67,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -89,21 +86,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,45 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.xtts.tts import XTTSService from pipecat.services.xtts.tts import XTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
# Create an HTTP session
async with aiohttp.ClientSession() as session:
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = XTTSService( tts = XTTSService(
aiohttp_session=session, aiohttp_session=session,
voice_id="Claribel Dervla", voice_id="Claribel Dervla",
@@ -65,6 +64,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -83,21 +83,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -23,23 +19,20 @@ from pipecat.services.gladia.config import GladiaInputParams, LanguageConfig
from pipecat.services.gladia.stt import GladiaSTTService from pipecat.services.gladia.stt import GladiaSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transcriptions.language import Language from pipecat.transcriptions.language import Language
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -48,7 +41,7 @@ async def main():
) )
stt = GladiaSTTService( stt = GladiaSTTService(
api_key=os.getenv("GLADIA_API_KEY"), api_key=os.getenv("GLADIA_API_KEY", ""),
params=GladiaInputParams( params=GladiaInputParams(
language_config=LanguageConfig( language_config=LanguageConfig(
languages=[Language.EN], languages=[Language.EN],
@@ -57,16 +50,16 @@ async def main():
) )
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY", ""),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
) )
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY", ""), model="gpt-4o")
messages = [ messages = [
{ {
"role": "system", "role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", "content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
}, },
] ]
@@ -95,22 +88,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
# Register an event handler to exit the application when the user leaves. @transport.event_handler("on_client_disconnected")
@transport.event_handler("on_participant_left") async def on_client_disconnected(transport, client):
async def on_participant_left(transport, participant, reason): logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.lmnt.tts import LmntTTSService from pipecat.services.lmnt.tts import LmntTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = LmntTTSService(api_key=os.getenv("LMNT_API_KEY"), voice_id="morgan") tts = LmntTTSService(api_key=os.getenv("LMNT_API_KEY"), voice_id="morgan")
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
@@ -61,6 +57,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User respones context_aggregator.user(), # User respones
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -79,21 +76,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -0,0 +1,102 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.groq.llm import GroqLLMService
from pipecat.services.groq.stt import GroqSTTService
from pipecat.services.groq.tts import GroqTTSService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True)
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
),
)
stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"))
llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile")
tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY"))
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
),
)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
if __name__ == "__main__":
from run import main
main()

View File

@@ -1,115 +0,0 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.together.llm import TogetherLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def main():
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
)
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = TogetherLLMService(
api_key=os.getenv("TOGETHER_API_KEY"),
model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
params=TogetherLLMService.InputParams(
temperature=1.0,
top_p=0.9,
top_k=40,
extra={
"frequency_penalty": 2.0,
"presence_penalty": 0.0,
},
),
)
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond in plain language. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
user_aggregator = context_aggregator.user()
assistant_aggregator = context_aggregator.assistant()
pipeline = Pipeline(
[
transport.input(), # Transport user input
user_aggregator, # User responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output
assistant_aggregator, # Assistant spoken responses
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
),
)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await transport.capture_participant_transcription(participant["id"])
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
await task.cancel()
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.aws.tts import PollyTTSService from pipecat.services.aws.tts import PollyTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, _) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
None,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -89,21 +82,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -22,23 +18,20 @@ from pipecat.services.google.llm import GoogleLLMService
from pipecat.services.google.stt import GoogleSTTService from pipecat.services.google.stt import GoogleSTTService
from pipecat.services.google.tts import GoogleTTSService from pipecat.services.google.tts import GoogleTTSService
from pipecat.transcriptions.language import Language from pipecat.transcriptions.language import Language
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, _) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
None,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -48,11 +41,13 @@ async def main():
stt = GoogleSTTService( stt = GoogleSTTService(
params=GoogleSTTService.InputParams(languages=Language.EN_US), params=GoogleSTTService.InputParams(languages=Language.EN_US),
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
) )
tts = GoogleTTSService( tts = GoogleTTSService(
voice_id="en-US-Journey-F", voice_id="en-US-Chirp3-HD-Charon",
params=GoogleTTSService.InputParams(language=Language.EN_US), params=GoogleTTSService.InputParams(language=Language.EN_US),
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
) )
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
@@ -89,21 +84,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.assemblyai.stt import AssemblyAISTTService from pipecat.services.assemblyai.stt import AssemblyAISTTService
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -88,21 +81,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.filters.krisp_filter import KrispFilter from pipecat.audio.filters.krisp_filter import KrispFilter
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -22,23 +18,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.deepgram.tts import DeepgramTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -85,20 +78,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,45 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.rime.tts import RimeHttpTTSService from pipecat.services.rime.tts import RimeHttpTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
# Create an HTTP session
async with aiohttp.ClientSession() as session:
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = RimeHttpTTSService( tts = RimeHttpTTSService(
api_key=os.getenv("RIME_API_KEY", ""), api_key=os.getenv("RIME_API_KEY", ""),
voice_id="rex", voice_id="rex",
@@ -65,6 +64,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -83,21 +83,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.rime.tts import RimeTTSService from pipecat.services.rime.tts import RimeTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = RimeTTSService( tts = RimeTTSService(
api_key=os.getenv("RIME_API_KEY", ""), api_key=os.getenv("RIME_API_KEY", ""),
voice_id="rex", voice_id="rex",
@@ -64,6 +60,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -82,21 +79,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.nim.llm import NimLLMService from pipecat.services.nim.llm import NimLLMService
from pipecat.services.riva.stt import ParakeetSTTService from pipecat.services.riva.stt import ParakeetSTTService
from pipecat.services.riva.tts import FastPitchTTSService from pipecat.services.riva.tts import FastPitchTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, _) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
None,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -47,9 +40,7 @@ async def main():
stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY")) stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
llm = NimLLMService( llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct")
api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct"
)
tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
@@ -75,22 +66,38 @@ async def main():
] ]
) )
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
),
)
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,16 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from dataclasses import dataclass from dataclasses import dataclass
import aiohttp
import google.ai.generativelanguage as glm import google.ai.generativelanguage as glm
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import ( from pipecat.frames.frames import (
@@ -32,14 +28,15 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameProcessor from pipecat.processors.frame_processor import FrameProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.google.llm import GoogleLLMService from pipecat.services.google.llm import GoogleLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.google.tts import GoogleTTSService
from pipecat.transcriptions.language import Language
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
marker = "|----|" marker = "|----|"
system_message = f""" system_message = f"""
@@ -193,15 +190,13 @@ class TanscriptionContextFixup(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
# No transcription at all. just audio input to Gemini! # No transcription at all. just audio input to Gemini!
# transcription_enabled=True, # transcription_enabled=True,
@@ -211,13 +206,14 @@ async def main():
), ),
) )
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001")
tts = GoogleTTSService(
voice_id="en-US-Chirp3-HD-Charon",
params=GoogleTTSService.InputParams(language=Language.EN_US),
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
)
messages = [ messages = [
{ {
"role": "system", "role": "system",
@@ -258,20 +254,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.fish.tts import FishAudioTTSService from pipecat.services.fish.tts import FishAudioTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = FishAudioTTSService( tts = FishAudioTTSService(
api_key=os.getenv("FISH_API_KEY"), api_key=os.getenv("FISH_API_KEY"),
model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama
@@ -64,6 +60,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -82,21 +79,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -20,7 +16,9 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.ultravox.stt import UltravoxSTTService from pipecat.services.ultravox.stt import UltravoxSTTService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
@@ -28,8 +26,6 @@ load_dotenv(override=True)
# The Ultravox model is compute-intensive and performs best with GPU acceleration. # The Ultravox model is compute-intensive and performs best with GPU acceleration.
# This can be deployed on cloud GPU providers like Cerebrium.ai for optimal performance. # This can be deployed on cloud GPU providers like Cerebrium.ai for optimal performance.
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
# Want to initialize the ultravox processor since it takes time to load the model and dont # Want to initialize the ultravox processor since it takes time to load the model and dont
# want to load it every time the pipeline is run # want to load it every time the pipeline is run
@@ -39,17 +35,14 @@ ultravox_processor = UltravoxSTTService(
) )
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=False,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
vad_audio_passthrough=True, vad_audio_passthrough=True,
@@ -78,14 +71,25 @@ async def main():
), ),
) )
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_connected")
async def on_participant_left(transport, participant, reason): async def on_client_connected(transport, client):
logger.info(f"Client connected")
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.neuphonic.tts import NeuphonicHttpTTSService from pipecat.services.neuphonic.tts import NeuphonicHttpTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = NeuphonicHttpTTSService( tts = NeuphonicHttpTTSService(
api_key=os.getenv("NEUPHONIC_API_KEY"), api_key=os.getenv("NEUPHONIC_API_KEY"),
voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily
@@ -64,6 +60,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -82,21 +79,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,46 +4,42 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.neuphonic.tts import NeuphonicTTSService from pipecat.services.neuphonic.tts import NeuphonicTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = NeuphonicTTSService( tts = NeuphonicTTSService(
api_key=os.getenv("NEUPHONIC_API_KEY"), api_key=os.getenv("NEUPHONIC_API_KEY"),
voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily
@@ -64,6 +60,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -82,21 +79,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left") @transport.event_handler("on_client_disconnected")
async def on_participant_left(transport, participant, reason): async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.fal.stt import FalSTTService from pipecat.services.fal.stt import FalSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -88,22 +81,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
# Register an event handler to exit the application when the user leaves. @transport.event_handler("on_client_disconnected")
@transport.event_handler("on_participant_left") async def on_client_disconnected(transport, client):
async def on_participant_left(transport, participant, reason): logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel() await task.cancel()
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -1,103 +0,0 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.groq.llm import GroqLLMService
from pipecat.services.groq.stt import GroqSTTService
from pipecat.services.groq.tts import GroqTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def main():
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True,
# transcription_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
),
)
stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"))
llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile")
tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY"))
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
),
)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await transport.capture_participant_transcription(participant["id"])
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
await task.cancel()
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -4,8 +4,8 @@ import os
from typing import Tuple from typing import Tuple
import aiohttp import aiohttp
from daily_runner import configure
from dotenv import load_dotenv from dotenv import load_dotenv
from runner import configure
from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -72,7 +72,8 @@ async def main():
async def get_text_and_audio(messages) -> Tuple[str, bytearray]: async def get_text_and_audio(messages) -> Tuple[str, bytearray]:
"""This function streams text from the LLM and uses the TTS service to convert """This function streams text from the LLM and uses the TTS service to convert
that text to speech as it's received.""" that text to speech as it's received.
"""
source_queue = asyncio.Queue() source_queue = asyncio.Queue()
sink_queue = asyncio.Queue() sink_queue = asyncio.Queue()
sentence_aggregator = SentenceAggregator() sentence_aggregator = SentenceAggregator()

View File

@@ -4,13 +4,9 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import ( from pipecat.frames.frames import (
Frame, Frame,
@@ -23,13 +19,12 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class MirrorProcessor(FrameProcessor): class MirrorProcessor(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -44,6 +39,7 @@ class MirrorProcessor(FrameProcessor):
) )
) )
elif isinstance(frame, InputImageRawFrame): elif isinstance(frame, InputImageRawFrame):
print(f"Received image frame: {frame.size} {frame.format}")
await self.push_frame( await self.push_frame(
OutputImageRawFrame(image=frame.image, size=frame.size, format=frame.format) OutputImageRawFrame(image=frame.image, size=frame.size, format=frame.format)
) )
@@ -51,17 +47,15 @@ class MirrorProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Test",
DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
camera_in_enabled=True,
camera_out_enabled=True, camera_out_enabled=True,
camera_out_is_live=True, camera_out_is_live=True,
camera_out_width=1280, camera_out_width=1280,
@@ -69,24 +63,32 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await transport.capture_participant_video(participant["id"])
pipeline = Pipeline([transport.input(), MirrorProcessor(), transport.output()]) pipeline = Pipeline([transport.input(), MirrorProcessor(), transport.output()])
runner = PipelineRunner()
task = PipelineTask( task = PipelineTask(
pipeline, pipeline,
params=PipelineParams( params=PipelineParams(),
audio_in_sample_rate=24000,
audio_out_sample_rate=24000,
),
) )
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -5,13 +5,10 @@
# #
import asyncio import asyncio
import sys
import tkinter as tk import tkinter as tk
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import ( from pipecat.frames.frames import (
Frame, Frame,
@@ -24,14 +21,13 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class MirrorProcessor(FrameProcessor): class MirrorProcessor(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -53,17 +49,25 @@ class MirrorProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
p2p_transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
camera_in_enabled=True,
camera_out_enabled=True,
camera_out_is_live=True,
camera_out_width=1280,
camera_out_height=720,
),
)
tk_root = tk.Tk() tk_root = tk.Tk()
tk_root.title("Local Mirror") tk_root.title("Local Mirror")
daily_transport = DailyTransport(
room_url, token, "Test", DailyParams(audio_in_enabled=True)
)
tk_transport = TkLocalTransport( tk_transport = TkLocalTransport(
tk_root, tk_root,
TkTransportParams( TkTransportParams(
@@ -75,18 +79,15 @@ async def main():
), ),
) )
@daily_transport.event_handler("on_first_participant_joined") @p2p_transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_video(participant["id"]) logger.info(f"Client connected")
pipeline = Pipeline([daily_transport.input(), MirrorProcessor(), tk_transport.output()]) pipeline = Pipeline([p2p_transport.input(), MirrorProcessor(), tk_transport.output()])
task = PipelineTask( task = PipelineTask(
pipeline, pipeline,
params=PipelineParams( params=PipelineParams(),
audio_in_sample_rate=24000,
audio_out_sample_rate=24000,
),
) )
async def run_tk(): async def run_tk():
@@ -95,10 +96,12 @@ async def main():
tk_root.update_idletasks() tk_root.update_idletasks()
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await asyncio.gather(runner.run(task), run_tk()) await asyncio.gather(runner.run(task), run_tk())
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,47 +4,44 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.filters.wake_check_filter import WakeCheckFilter from pipecat.processors.filters.wake_check_filter import WakeCheckFilter
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Robot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -67,6 +64,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt, # STT
hey_robot_filter, # Filter out speech not directed at the robot hey_robot_filter, # Filter out speech not directed at the robot
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
@@ -78,15 +76,27 @@ async def main():
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.") # Kick off the conversation.
await task.queue_frame(TTSSpeakFrame("Hi! If you want to talk to me, just say 'Hey Robot'"))
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,21 +4,18 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import wave import wave
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import ( from pipecat.frames.frames import (
Frame, Frame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
OutputAudioRawFrame, OutputAudioRawFrame,
TTSSpeakFrame,
) )
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -30,14 +27,14 @@ from pipecat.processors.aggregators.openai_llm_context import (
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.logger import FrameLogger from pipecat.processors.logger import FrameLogger
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
sounds = {} sounds = {}
sound_files = ["ding1.wav", "ding2.wav"] sound_files = ["ding1.wav", "ding2.wav"]
@@ -80,22 +77,22 @@ class InboundSoundEffectWrapper(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
tts = CartesiaTTSService( tts = CartesiaTTSService(
@@ -120,6 +117,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
in_sound, in_sound,
fl2, fl2,
@@ -132,18 +130,30 @@ async def main():
] ]
) )
@transport.event_handler("on_first_participant_joined") task = PipelineTask(pipeline)
async def on_first_participant_joined(transport, participant):
await transport.capture_participant_transcription(participant["id"]) @transport.event_handler("on_client_connected")
await tts.say("Hi, I'm listening!") async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frame(TTSSpeakFrame("Hi, I'm listening!"))
await transport.send_audio(sounds["ding1.wav"]) await transport.send_audio(sounds["ding1.wav"])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
task = PipelineTask(pipeline) @transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from typing import Optional from typing import Optional
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
@@ -23,14 +19,14 @@ from pipecat.processors.aggregators.user_response import UserResponseAggregator
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.moondream.vision import MoondreamService from pipecat.services.moondream.vision import MoondreamService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class UserImageRequester(FrameProcessor): class UserImageRequester(FrameProcessor):
def __init__(self, participant_id: Optional[str] = None): def __init__(self, participant_id: Optional[str] = None):
@@ -50,24 +46,27 @@ class UserImageRequester(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: # Get WebRTC peer connection ID
(room_url, token) = await configure(session) webrtc_peer_id = webrtc_connection.pc_id
transport = DailyTransport( logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
room_url,
token, transport = SmallWebRTCTransport(
"Describe participant video", webrtc_connection=webrtc_connection,
DailyParams( params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
user_response = UserResponseAggregator() user_response = UserResponseAggregator()
# Initialize the image requester without setting the participant ID yet
image_requester = UserImageRequester() image_requester = UserImageRequester()
vision_aggregator = VisionImageFrameAggregator() vision_aggregator = VisionImageFrameAggregator()
@@ -75,21 +74,17 @@ async def main():
# If you run into weird description, try with use_cpu=True # If you run into weird description, try with use_cpu=True
moondream = MoondreamService() moondream = MoondreamService()
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
) )
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await tts.say("Hi there! Feel free to ask me what I see.")
await transport.capture_participant_video(participant["id"], framerate=0)
await transport.capture_participant_transcription(participant["id"])
image_requester.set_participant_id(participant["id"])
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
user_response, user_response,
image_requester, image_requester,
vision_aggregator, vision_aggregator,
@@ -101,10 +96,31 @@ async def main():
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner() @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}")
# Welcome message
await tts.say("Hi there! Feel free to ask me what I see.")
# Set the participant ID in the image requester
image_requester.set_participant_id(webrtc_peer_id)
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,33 +4,29 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from typing import Optional from typing import Optional
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.user_response import UserResponseAggregator
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.google.llm import GoogleLLMService from pipecat.services.google.llm import GoogleLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class UserImageRequester(FrameProcessor): class UserImageRequester(FrameProcessor):
def __init__(self, participant_id: Optional[str] = None): def __init__(self, participant_id: Optional[str] = None):
@@ -50,29 +46,34 @@ class UserImageRequester(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: # Get WebRTC peer connection ID
(room_url, token) = await configure(session) webrtc_peer_id = webrtc_connection.pc_id
transport = DailyTransport( logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
room_url,
token, transport = SmallWebRTCTransport(
"Describe participant video", webrtc_connection=webrtc_connection,
DailyParams( params=TransportParams(
audio_in_enabled=True, # This is so Silero VAD can get audio data audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
user_response = UserResponseAggregator() user_response = UserResponseAggregator()
# Initialize the image requester without setting the participant ID yet
image_requester = UserImageRequester() image_requester = UserImageRequester()
vision_aggregator = VisionImageFrameAggregator() vision_aggregator = VisionImageFrameAggregator()
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
# Google Gemini model for vision analysis
google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
@@ -80,16 +81,10 @@ async def main():
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
) )
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await tts.say("Hi there! Feel free to ask me what I see.")
await transport.capture_participant_video(participant["id"], framerate=0)
await transport.capture_participant_transcription(participant["id"])
image_requester.set_participant_id(participant["id"])
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
user_response, user_response,
image_requester, image_requester,
vision_aggregator, vision_aggregator,
@@ -99,12 +94,36 @@ async def main():
] ]
) )
task = PipelineTask(pipeline) task = PipelineTask(
pipeline,
params=PipelineParams(allow_interruptions=True),
)
runner = PipelineRunner() @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}")
# Welcome message
await tts.say("Hi there! Feel free to ask me what I see.")
# Set the participant ID in the image requester
image_requester.set_participant_id(webrtc_peer_id)
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,33 +4,29 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from typing import Optional from typing import Optional
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.user_response import UserResponseAggregator
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class UserImageRequester(FrameProcessor): class UserImageRequester(FrameProcessor):
def __init__(self, participant_id: Optional[str] = None): def __init__(self, participant_id: Optional[str] = None):
@@ -50,28 +46,34 @@ class UserImageRequester(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: # Get WebRTC peer connection ID
(room_url, token) = await configure(session) webrtc_peer_id = webrtc_connection.pc_id
transport = DailyTransport( logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
room_url,
token, transport = SmallWebRTCTransport(
"Describe participant video", webrtc_connection=webrtc_connection,
DailyParams( params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
user_response = UserResponseAggregator() user_response = UserResponseAggregator()
# Initialize the image requester without setting the participant ID yet
image_requester = UserImageRequester() image_requester = UserImageRequester()
vision_aggregator = VisionImageFrameAggregator() vision_aggregator = VisionImageFrameAggregator()
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
# OpenAI GPT-4o for vision analysis
openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
tts = CartesiaTTSService( tts = CartesiaTTSService(
@@ -79,16 +81,10 @@ async def main():
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
) )
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await tts.say("Hi there! Feel free to ask me what I see.")
await transport.capture_participant_video(participant["id"], framerate=0)
await transport.capture_participant_transcription(participant["id"])
image_requester.set_participant_id(participant["id"])
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
user_response, user_response,
image_requester, image_requester,
vision_aggregator, vision_aggregator,
@@ -98,12 +94,36 @@ async def main():
] ]
) )
task = PipelineTask(pipeline) task = PipelineTask(
pipeline,
params=PipelineParams(allow_interruptions=True),
)
runner = PipelineRunner() @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}")
# Welcome message
await tts.say("Hi there! Feel free to ask me what I see.")
# Set the participant ID in the image requester
image_requester.set_participant_id(webrtc_peer_id)
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,33 +4,29 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from typing import Optional from typing import Optional
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.user_response import UserResponseAggregator
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.anthropic.llm import AnthropicLLMService
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class UserImageRequester(FrameProcessor): class UserImageRequester(FrameProcessor):
def __init__(self, participant_id: Optional[str] = None): def __init__(self, participant_id: Optional[str] = None):
@@ -50,28 +46,34 @@ class UserImageRequester(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: # Get WebRTC peer connection ID
(room_url, token) = await configure(session) webrtc_peer_id = webrtc_connection.pc_id
transport = DailyTransport( logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
room_url,
token, transport = SmallWebRTCTransport(
"Describe participant video", webrtc_connection=webrtc_connection,
DailyParams( params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
user_response = UserResponseAggregator() user_response = UserResponseAggregator()
# Initialize the image requester without setting the participant ID yet
image_requester = UserImageRequester() image_requester = UserImageRequester()
vision_aggregator = VisionImageFrameAggregator() vision_aggregator = VisionImageFrameAggregator()
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
# Anthropic for vision analysis
anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
@@ -79,16 +81,10 @@ async def main():
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
) )
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await tts.say("Hi there! Feel free to ask me what I see.")
await transport.capture_participant_video(participant["id"], framerate=0)
await transport.capture_participant_transcription(participant["id"])
image_requester.set_participant_id(participant["id"])
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
user_response, user_response,
image_requester, image_requester,
vision_aggregator, vision_aggregator,
@@ -98,12 +94,36 @@ async def main():
] ]
) )
task = PipelineTask(pipeline) task = PipelineTask(
pipeline,
params=PipelineParams(allow_interruptions=True),
)
runner = PipelineRunner() @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}")
# Welcome message
await tts.say("Hi there! Feel free to ask me what I see.")
# Set the participant ID in the image requester
image_requester.set_participant_id(webrtc_peer_id)
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,13 +4,9 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.frames.frames import Frame, TranscriptionFrame
@@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.whisper.stt import WhisperSTTService from pipecat.services.whisper.stt import WhisperSTTService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -35,15 +30,12 @@ class TranscriptionLogger(FrameProcessor):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
None, params=TransportParams(
"Transcription bot",
DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -59,10 +51,21 @@ async def main():
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os
import sys
import aiohttp import os
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.frames.frames import Frame, TranscriptionFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -19,13 +16,12 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -35,17 +31,17 @@ class TranscriptionLogger(FrameProcessor):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True) webrtc_connection=webrtc_connection,
params=TransportParams(audio_in_enabled=True),
) )
stt = DeepgramSTTService( stt = DeepgramSTTService(
api_key=os.getenv("DEEPGRAM_API_KEY"), api_key=os.getenv("DEEPGRAM_API_KEY"),
# live_options=LiveOptions(language=Language.FR), live_options=LiveOptions(language=Language.EN),
) )
tl = TranscriptionLogger() tl = TranscriptionLogger()
@@ -54,10 +50,21 @@ async def main():
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.frames.frames import Frame, TranscriptionFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.gladia import GladiaSTTService from pipecat.services.gladia import GladiaSTTService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -35,12 +30,12 @@ class TranscriptionLogger(FrameProcessor):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True) webrtc_connection=webrtc_connection,
params=TransportParams(audio_in_enabled=True),
) )
stt = GladiaSTTService( stt = GladiaSTTService(
@@ -54,10 +49,21 @@ async def main():
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.frames.frames import Frame, TranscriptionFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.assemblyai.stt import AssemblyAISTTService from pipecat.services.assemblyai.stt import AssemblyAISTTService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -35,12 +30,12 @@ class TranscriptionLogger(FrameProcessor):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True) webrtc_connection=webrtc_connection,
params=TransportParams(audio_in_enabled=True),
) )
stt = AssemblyAISTTService( stt = AssemblyAISTTService(
@@ -53,10 +48,21 @@ async def main():
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import sys
import time import time
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -21,13 +18,12 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.whisper.stt import MLXModel, WhisperSTTServiceMLX from pipecat.services.whisper.stt import MLXModel, WhisperSTTServiceMLX
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
STOP_SECS = 2.0 STOP_SECS = 2.0
@@ -56,15 +52,12 @@ class TranscriptionLogger(FrameProcessor):
self._last_transcription_time = time.time() self._last_transcription_time = time.time()
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
None, params=TransportParams(
"Transcription bot",
DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)),
@@ -86,10 +79,21 @@ async def main():
), ),
) )
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -94,6 +90,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -112,16 +109,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,35 +18,35 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.anthropic.llm import AnthropicLLMService
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
location = arguments["location"] location = arguments["location"]
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -89,6 +85,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User spoken responses context_aggregator.user(), # User spoken responses
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -105,16 +102,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -6,12 +6,9 @@
import asyncio import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,14 +19,16 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.anthropic.llm import AnthropicLLMService
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
video_participant_id = None # Global variable to store the peer connection ID
webrtc_peer_id = None
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
@@ -39,32 +38,45 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
question = arguments["question"] question = arguments["question"]
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
# Request the image frame
await llm.request_image_frame( await llm.request_image_frame(
user_id=video_participant_id, user_id=webrtc_peer_id,
function_name=function_name, function_name=function_name,
tool_call_id=tool_call_id, tool_call_id=tool_call_id,
text_content=question, text_content=question,
) )
# Wait a short time for the frame to be processed
await asyncio.sleep(0.5)
async def main(): # Return a result to complete the function call
global llm await result_callback(
f"I've captured an image from your camera and I'm analyzing what you asked about: {question}"
)
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport( async def run_bot(webrtc_connection: SmallWebRTCConnection):
room_url, global webrtc_peer_id
token, webrtc_peer_id = webrtc_connection.pc_id
"Respond bot",
DailyParams( logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True, # Make sure camera input is enabled
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -102,8 +114,6 @@ async def main():
) )
tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) tools = ToolsSchema(standard_tools=[weather_function, get_image_function])
# todo: test with very short initial user message
system_prompt = """\ system_prompt = """\
You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions.
@@ -115,12 +125,12 @@ You can respond to questions about the weather using the get_weather tool.
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
indicate you should use the get_image tool are: indicate you should use the get_image tool are:
- What do you see? - What do you see?
- What's in the video? - What's in the video?
- Can you describe the video? - Can you describe the video?
- Tell me about what you see. - Tell me about what you see.
- Tell me something interesting about what you see. - Tell me something interesting about what you see.
- What's happening in the video? - What's happening in the video?
If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise. If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise.
""" """
@@ -144,6 +154,7 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt, # STT
context_aggregator.user(), # User speech to text context_aggregator.user(), # User speech to text
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -160,18 +171,27 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
global video_participant_id logger.info(f"Client connected: {client}")
video_participant_id = participant["id"]
await transport.capture_participant_transcription(video_participant_id)
await transport.capture_participant_video(video_participant_id, framerate=0)
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.together.llm import TogetherLLMService from pipecat.services.together.llm import TogetherLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -95,6 +91,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -105,16 +102,27 @@ async def main():
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -6,12 +6,9 @@
import asyncio import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -21,15 +18,17 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
video_participant_id = None # Global variable to store the peer connection ID
webrtc_peer_id = None
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
@@ -38,32 +37,46 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}")
question = arguments["question"] question = arguments["question"]
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
# Request the image frame
await llm.request_image_frame( await llm.request_image_frame(
user_id=video_participant_id, user_id=webrtc_peer_id,
function_name=function_name, function_name=function_name,
tool_call_id=tool_call_id, tool_call_id=tool_call_id,
text_content=question, text_content=question,
) )
# Wait a short time for the frame to be processed
await asyncio.sleep(0.5)
async def main(): # Return a result to complete the function call
async with aiohttp.ClientSession() as session: await result_callback(
(room_url, token) = await configure(session) f"I've captured an image from your camera and I'm analyzing what you asked about: {question}"
)
transport = DailyTransport(
room_url, async def run_bot(webrtc_connection: SmallWebRTCConnection):
token, global webrtc_peer_id
"Respond bot", webrtc_peer_id = webrtc_connection.pc_id
DailyParams(
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True, # Make sure camera input is enabled
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -113,12 +126,12 @@ You can respond to questions about the weather using the get_weather tool.
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
indicate you should use the get_image tool are: indicate you should use the get_image tool are:
- What do you see? - What do you see?
- What's in the video? - What's in the video?
- Can you describe the video? - Can you describe the video?
- Tell me about what you see. - Tell me about what you see.
- Tell me something interesting about what you see. - Tell me something interesting about what you see.
- What's happening in the video? - What's happening in the video?
""" """
messages = [ messages = [
{"role": "system", "content": system_prompt}, {"role": "system", "content": system_prompt},
@@ -130,6 +143,7 @@ indicate you should use the get_image tool are:
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -140,19 +154,27 @@ indicate you should use the get_image tool are:
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
global video_participant_id logger.info(f"Client connected")
video_participant_id = participant["id"]
await transport.capture_participant_transcription(participant["id"])
await transport.capture_participant_video(video_participant_id, framerate=0)
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -6,12 +6,9 @@
import asyncio import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,15 +19,17 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.google.llm import GoogleLLMService from pipecat.services.google.llm import GoogleLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
video_participant_id = None # Global variable to store the peer connection ID
webrtc_peer_id = None
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
@@ -40,32 +39,46 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}")
question = arguments["question"] question = arguments["question"]
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
# Request the image frame
await llm.request_image_frame( await llm.request_image_frame(
user_id=video_participant_id, user_id=webrtc_peer_id,
function_name=function_name, function_name=function_name,
tool_call_id=tool_call_id, tool_call_id=tool_call_id,
text_content=question, text_content=question,
) )
# Wait a short time for the frame to be processed
await asyncio.sleep(0.5)
async def main(): # Return a result to complete the function call
async with aiohttp.ClientSession() as session: await result_callback(
(room_url, token) = await configure(session) f"I've captured an image from your camera and I'm analyzing what you asked about: {question}"
)
transport = DailyTransport(
room_url, async def run_bot(webrtc_connection: SmallWebRTCConnection):
token, global webrtc_peer_id
"Respond bot", webrtc_peer_id = webrtc_connection.pc_id
DailyParams(
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True, # Make sure camera input is enabled
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -115,12 +128,12 @@ You can respond to questions about the weather using the get_weather tool.
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
indicate you should use the get_image tool are: indicate you should use the get_image tool are:
- What do you see? - What do you see?
- What's in the video? - What's in the video?
- Can you describe the video? - Can you describe the video?
- Tell me about what you see. - Tell me about what you see.
- Tell me something interesting about what you see. - Tell me something interesting about what you see.
- What's happening in the video? - What's happening in the video?
""" """
messages = [ messages = [
{"role": "system", "content": system_prompt}, {"role": "system", "content": system_prompt},
@@ -133,6 +146,7 @@ indicate you should use the get_image tool are:
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -150,19 +164,27 @@ indicate you should use the get_image tool are:
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
global video_participant_id logger.info(f"Client connected: {client}")
video_participant_id = participant["id"]
await transport.capture_participant_transcription(participant["id"])
await transport.capture_participant_video(video_participant_id, framerate=0)
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -24,28 +20,25 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.groq.llm import GroqLLMService from pipecat.services.groq.llm import GroqLLMService
from pipecat.services.groq.stt import GroqSTTService from pipecat.services.groq.stt import GroqSTTService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -113,16 +106,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -21,35 +17,35 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.grok.llm import GrokLLMService from pipecat.services.grok.llm import GrokLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -90,6 +86,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -107,16 +104,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -23,35 +19,35 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.azure.llm import AzureLLMService from pipecat.services.azure.llm import AzureLLMService
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -96,6 +92,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -113,16 +110,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.fireworks.llm import FireworksLLMService from pipecat.services.fireworks.llm import FireworksLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -95,6 +91,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -112,16 +109,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,45 +18,43 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.nim.llm import NimLLMService from pipecat.services.nim.llm import NimLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
# text_filters=[MarkdownTextFilter()], # text_filters=[MarkdownTextFilter()],
) )
llm = NimLLMService( llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct")
api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct"
)
# You can also register a function_name of None to get all functions # You can also register a function_name of None to get all functions
# sent to the same callback with an additional function_name parameter. # sent to the same callback with an additional function_name parameter.
llm.register_function("get_current_weather", fetch_weather_from_api) llm.register_function("get_current_weather", fetch_weather_from_api)
@@ -95,6 +89,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -112,16 +107,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -23,35 +19,35 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.cerebras.llm import CerebrasLLMService from pipecat.services.cerebras.llm import CerebrasLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -102,6 +98,7 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -120,16 +117,27 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.deepseek.llm import DeepSeekLLMService from pipecat.services.deepseek.llm import DeepSeekLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -102,6 +98,7 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -120,16 +117,27 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.azure.tts import AzureTTSService from pipecat.services.azure.tts import AzureTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openrouter.llm import OpenRouterLLMService from pipecat.services.openrouter.llm import OpenRouterLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = AzureTTSService( tts = AzureTTSService(
api_key=os.getenv("AZURE_API_KEY"), api_key=os.getenv("AZURE_API_KEY"),
region="eastus", region="eastus",
@@ -96,6 +92,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -114,16 +111,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -11,14 +11,10 @@ currently support function calling. The example shows basic chat completion func
using Perplexity's API while maintaining compatibility with the OpenAI interface. using Perplexity's API while maintaining compatibility with the OpenAI interface.
""" """
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -26,31 +22,31 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.perplexity.llm import PerplexityLLMService from pipecat.services.perplexity.llm import PerplexityLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -71,6 +67,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -89,16 +86,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -21,37 +17,37 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
from pipecat.services.google.llm_openai import GoogleLLMOpenAIBetaService from pipecat.services.google.llm_openai import GoogleLLMOpenAIBetaService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
api_key=os.getenv("ELEVENLABS_API_KEY", ""), api_key=os.getenv("ELEVENLABS_API_KEY", ""),
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
@@ -92,6 +88,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -109,16 +106,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -21,37 +17,37 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
from pipecat.services.google.llm_vertex import GoogleVertexLLMService from pipecat.services.google.llm_vertex import GoogleVertexLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
api_key=os.getenv("ELEVENLABS_API_KEY", ""), api_key=os.getenv("ELEVENLABS_API_KEY", ""),
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
@@ -98,6 +94,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -115,16 +112,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.qwen.llm import QwenLLMService from pipecat.services.qwen.llm import QwenLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await llm.push_frame(TTSSpeakFrame("Let me check on that."))
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -94,6 +90,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
stt,
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
@@ -112,16 +109,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from openai.types.chat import ChatCompletionToolParam from openai.types.chat import ChatCompletionToolParam
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline
@@ -22,13 +18,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.processors.filters.function_filter import FunctionFilter
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
current_voice = "News Lady" current_voice = "News Lady"
@@ -55,22 +52,22 @@ async def barbershop_man_filter(frame) -> bool:
return current_voice == "Barbershop Man" return current_voice == "Barbershop Man"
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Pipecat", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
news_lady = CartesiaTTSService( news_lady = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady voice_id="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady
@@ -121,6 +118,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses context_aggregator.user(), # User responses
llm, # LLM llm, # LLM
ParallelPipeline( # TTS (one of the following vocies) ParallelPipeline( # TTS (one of the following vocies)
@@ -138,9 +136,9 @@ async def main():
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append( messages.append(
{ {
@@ -150,10 +148,21 @@ async def main():
) )
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,16 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from deepgram import LiveOptions from deepgram import LiveOptions
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from openai.types.chat import ChatCompletionToolParam from openai.types.chat import ChatCompletionToolParam
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline
@@ -25,12 +21,12 @@ from pipecat.processors.filters.function_filter import FunctionFilter
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
current_language = "English" current_language = "English"
@@ -49,15 +45,13 @@ async def spanish_filter(frame) -> bool:
return current_language == "Spanish" return current_language == "Spanish"
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Pipecat", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -76,7 +70,7 @@ async def main():
spanish_tts = CartesiaTTSService( spanish_tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="846d6cb0-2301-48b6-9683-48f5618ea2f6", # Spanish-speaking Lady voice_id="d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", # Spanish-speaking Lady
) )
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
@@ -128,9 +122,9 @@ async def main():
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append( messages.append(
{ {
@@ -140,10 +134,21 @@ async def main():
) )
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,50 +4,46 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.deepgram.tts import DeepgramTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import ( from pipecat.transports.base_transport import TransportParams
DailyParams, from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
DailyTransport, from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
DailyTransportMessageFrame, from pipecat.transports.services.daily import DailyTransportMessageFrame
)
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
# Create an HTTP session
async with aiohttp.ClientSession() as session:
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = DeepgramTTSService( tts = DeepgramTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("DEEPGRAM_API_KEY"), api_key=os.getenv("DEEPGRAM_API_KEY"),
@@ -77,6 +73,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt, # STT
context_aggregator.user(), context_aggregator.user(),
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
@@ -93,15 +90,11 @@ async def main():
), ),
) )
# When a participant joins, start transcription for that participant so the
# bot can "hear" and respond to them.
@transport.event_handler("on_participant_joined")
async def on_participant_joined(transport, participant):
await transport.capture_participant_transcription(participant["id"])
# When the first participant joins, the bot should introduce itself. # When the first participant joins, the bot should introduce itself.
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@@ -134,9 +127,21 @@ async def main():
except Exception as e: except Exception as e:
logger.debug(f"message handling error: {e} - {message}") logger.debug(f"message handling error: {e} - {message}")
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import EndFrame, LLMMessagesFrame, TTSSpeakFrame from pipecat.frames.frames import EndFrame, LLMMessagesFrame, TTSSpeakFrame
@@ -21,31 +17,31 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.user_idle_processor import UserIdleProcessor from pipecat.processors.user_idle_processor import UserIdleProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, token) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -97,6 +93,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt,
user_idle, # Idle user check-in user_idle, # Idle user check-in
context_aggregator.user(), context_aggregator.user(),
llm, # LLM llm, # LLM
@@ -115,17 +112,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -5,43 +5,45 @@
# #
import argparse import argparse
import asyncio
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure_with_args
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
# Parse command line arguments
async def main(): # This will be used to pass the input video file to the bot
async with aiohttp.ClientSession() as session: # You can run the bot with a command like:
# python 18-gstreamer-filesrc.py -i path/to/video.mp4
def parse_arguments():
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
parser.add_argument("-i", "--input", type=str, required=True, help="Input video file") parser.add_argument("-i", "--input", type=str, required=True, help="Input video file")
return parser.parse_args()
(room_url, _, args) = await configure_with_args(session, parser)
transport = DailyTransport( args = parse_arguments()
room_url,
None,
"GStreamer", async def run_bot(webrtc_connection: SmallWebRTCConnection):
DailyParams( logger.info(f"Starting bot with video input: {args.input}")
audio_out_enabled=True,
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
camera_out_enabled=True, camera_out_enabled=True,
camera_out_is_live=True,
camera_out_width=1280, camera_out_width=1280,
camera_out_height=720, camera_out_height=720,
camera_out_is_live=True,
), ),
) )
@@ -62,10 +64,12 @@ async def main():
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,39 +4,32 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot with video test source")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, _) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
None,
"GStreamer",
DailyParams(
camera_out_enabled=True, camera_out_enabled=True,
camera_out_is_live=True,
camera_out_width=1280, camera_out_width=1280,
camera_out_height=720, camera_out_height=720,
camera_out_is_live=True,
), ),
) )
@@ -49,17 +42,19 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
gst, # GStreamer file source gst, # GStreamer test source
transport.output(), # Transport bot output transport.output(), # Transport bot output
] ]
) )
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner() runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from datetime import datetime from datetime import datetime
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -29,13 +25,12 @@ from pipecat.services.openai_realtime_beta import (
SemanticTurnDetection, SemanticTurnDetection,
SessionProperties, SessionProperties,
) )
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
temperature = 75 if args["format"] == "fahrenheit" else 24 temperature = 75 if args["format"] == "fahrenheit" else 24
@@ -70,18 +65,14 @@ weather_function = FunctionSchema(
tools = ToolsSchema(standard_tools=[weather_function]) tools = ToolsSchema(standard_tools=[weather_function])
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot",
DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=False,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
vad_audio_passthrough=True, vad_audio_passthrough=True,
@@ -159,20 +150,31 @@ Remember, your responses should be short. Just one or two sentences, usually."""
allow_interruptions=True, allow_interruptions=True,
enable_metrics=True, enable_metrics=True,
enable_usage_metrics=True, enable_usage_metrics=True,
# report_only_initial_ttfb=True, report_only_initial_ttfb=True,
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from datetime import datetime from datetime import datetime
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -27,13 +23,12 @@ from pipecat.services.openai_realtime_beta import (
InputAudioTranscription, InputAudioTranscription,
SessionProperties, SessionProperties,
) )
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
temperature = 75 if args["format"] == "fahrenheit" else 24 temperature = 75 if args["format"] == "fahrenheit" else 24
@@ -69,18 +64,14 @@ weather_function = FunctionSchema(
tools = ToolsSchema(standard_tools=[weather_function]) tools = ToolsSchema(standard_tools=[weather_function])
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot",
DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=False,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
vad_audio_passthrough=True, vad_audio_passthrough=True,
@@ -88,7 +79,7 @@ async def main():
) )
session_properties = SessionProperties( session_properties = SessionProperties(
input_audio_transcription=InputAudioTranscription(), input_audio_transcription=InputAudioTranscription(model="whisper-1"),
# Set openai TurnDetection parameters. Not setting this at all will turn it # Set openai TurnDetection parameters. Not setting this at all will turn it
# on by default # on by default
# turn_detection=TurnDetection(silence_duration_ms=1000), # turn_detection=TurnDetection(silence_duration_ms=1000),
@@ -158,20 +149,31 @@ Remember, your responses should be short. Just one or two sentences, usually."""
allow_interruptions=True, allow_interruptions=True,
enable_metrics=True, enable_metrics=True,
enable_usage_metrics=True, enable_usage_metrics=True,
# report_only_initial_ttfb=True, report_only_initial_ttfb=True,
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,17 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import glob import glob
import json import json
import os import os
import sys
from datetime import datetime from datetime import datetime
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -25,13 +21,14 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext, OpenAILLMContext,
) )
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
BASE_FILENAME = "/tmp/pipecat_conversation_" BASE_FILENAME = "/tmp/pipecat_conversation_"
tts = None tts = None
@@ -165,23 +162,24 @@ tools = [
] ]
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
global tts logger.info(f"Starting bot")
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport( global tts
room_url,
token, transport = SmallWebRTCTransport(
"Respond bot", webrtc_connection=webrtc_connection,
DailyParams( params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -202,6 +200,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt, # STT
context_aggregator.user(), context_aggregator.user(),
llm, # LLM llm, # LLM
tts, tts,
@@ -216,20 +215,31 @@ async def main():
allow_interruptions=True, allow_interruptions=True,
enable_metrics=True, enable_metrics=True,
enable_usage_metrics=True, enable_usage_metrics=True,
# report_only_initial_ttfb=True, report_only_initial_ttfb=True,
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -8,13 +8,10 @@ import asyncio
import glob import glob
import json import json
import os import os
import sys
from datetime import datetime from datetime import datetime
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -24,19 +21,19 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext, OpenAILLMContext,
) )
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai_realtime_beta import ( from pipecat.services.openai_realtime_beta import (
InputAudioTranscription, InputAudioTranscription,
OpenAIRealtimeBetaLLMService, OpenAIRealtimeBetaLLMService,
SessionProperties, SessionProperties,
TurnDetection, TurnDetection,
) )
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
BASE_FILENAME = "/tmp/pipecat_conversation_" BASE_FILENAME = "/tmp/pipecat_conversation_"
@@ -167,24 +164,22 @@ tools = [
] ]
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot",
DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=False,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
vad_audio_passthrough=True, vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
session_properties = SessionProperties( session_properties = SessionProperties(
input_audio_transcription=InputAudioTranscription(), input_audio_transcription=InputAudioTranscription(),
# Set openai TurnDetection parameters. Not setting this at all will turn it # Set openai TurnDetection parameters. Not setting this at all will turn it
@@ -228,6 +223,7 @@ Remember, your responses should be short. Just one or two sentences, usually."""
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt, # STT
context_aggregator.user(), context_aggregator.user(),
llm, # LLM llm, # LLM
transport.output(), # Transport bot output transport.output(), # Transport bot output
@@ -241,20 +237,31 @@ Remember, your responses should be short. Just one or two sentences, usually."""
allow_interruptions=True, allow_interruptions=True,
enable_metrics=True, enable_metrics=True,
enable_usage_metrics=True, enable_usage_metrics=True,
# report_only_initial_ttfb=True, report_only_initial_ttfb=True,
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,17 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import glob import glob
import json import json
import os import os
import sys
from datetime import datetime from datetime import datetime
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -26,12 +22,13 @@ from pipecat.processors.aggregators.openai_llm_context import (
) )
from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.anthropic.llm import AnthropicLLMService
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
BASE_FILENAME = "/tmp/pipecat_conversation_" BASE_FILENAME = "/tmp/pipecat_conversation_"
tts = None tts = None
@@ -160,23 +157,24 @@ tools = [
] ]
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
global tts logger.info(f"Starting bot")
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport( global tts
room_url,
token, transport = SmallWebRTCTransport(
"Respond bot", webrtc_connection=webrtc_connection,
DailyParams( params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -199,6 +197,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt, # STT
context_aggregator.user(), context_aggregator.user(),
llm, # LLM llm, # LLM
tts, tts,
@@ -217,16 +216,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,17 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import glob import glob
import json import json
import os import os
import sys
from datetime import datetime from datetime import datetime
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -25,19 +21,21 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext, OpenAILLMContext,
) )
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.google.llm import GoogleLLMService from pipecat.services.google.llm import GoogleLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
video_participant_id = None video_participant_id = None
BASE_FILENAME = "/tmp/pipecat_conversation_" BASE_FILENAME = "/tmp/pipecat_conversation_"
tts = None tts = None
webrtc_peer_id = None
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
@@ -54,8 +52,11 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
question = arguments["question"] question = arguments["question"]
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
# Request the image frame
await llm.request_image_frame( await llm.request_image_frame(
user_id=video_participant_id, user_id=webrtc_peer_id,
function_name=function_name, function_name=function_name,
tool_call_id=tool_call_id, tool_call_id=tool_call_id,
text_content=question, text_content=question,
@@ -220,23 +221,26 @@ tools = [
] ]
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
global tts global tts, webrtc_peer_id
async with aiohttp.ClientSession() as session: webrtc_peer_id = webrtc_connection.pc_id
(room_url, token) = await configure(session)
transport = DailyTransport( logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
room_url,
token, transport = SmallWebRTCTransport(
"Respond bot", webrtc_connection=webrtc_connection,
DailyParams( params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
transcription_enabled=True, camera_in_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
vad_audio_passthrough=True,
), ),
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService( tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"), api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -258,6 +262,7 @@ async def main():
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
stt, # STT
context_aggregator.user(), context_aggregator.user(),
llm, # LLM llm, # LLM
tts, tts,
@@ -276,19 +281,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
global video_participant_id logger.info(f"Client connected")
video_participant_id = participant["id"]
await transport.capture_participant_transcription(participant["id"])
await transport.capture_participant_video(video_participant_id, framerate=0)
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import TextFrame from pipecat.frames.frames import TextFrame
@@ -28,23 +24,20 @@ from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.sync.event_notifier import EventNotifier from pipecat.sync.event_notifier import EventNotifier
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): transport = SmallWebRTCTransport(
async with aiohttp.ClientSession() as session: webrtc_connection=webrtc_connection,
(room_url, _) = await configure(session) params=TransportParams(
audio_in_enabled=True,
transport = DailyTransport(
room_url,
None,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -99,16 +92,12 @@ async def main():
# This a filter that will wake up the notifier if the given predicate # This a filter that will wake up the notifier if the given predicate
# (wake_check_filter) returns true. # (wake_check_filter) returns true.
completness_check = WakeNotifierFilter( completness_check = WakeNotifierFilter(notifier, types=(TextFrame,), filter=wake_check_filter)
notifier, types=(TextFrame,), filter=wake_check_filter
)
# This processor keeps the last context and will let it through once the # This processor keeps the last context and will let it through once the
# notifier is woken up. We start with the gate open because we send an # notifier is woken up. We start with the gate open because we send an
# initial context frame to start the conversation. # initial context frame to start the conversation.
gated_context_aggregator = GatedOpenAILLMContextAggregator( gated_context_aggregator = GatedOpenAILLMContextAggregator(notifier=notifier, start_open=True)
notifier=notifier, start_open=True
)
# Notify if the user hasn't said anything. # Notify if the user hasn't said anything.
async def user_idle_notifier(frame): async def user_idle_notifier(frame):
@@ -153,17 +142,28 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -6,14 +6,11 @@
import asyncio import asyncio
import os import os
import sys
import time import time
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from openai.types.chat import ChatCompletionToolParam from openai.types.chat import ChatCompletionToolParam
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import ( from pipecat.frames.frames import (
@@ -49,13 +46,12 @@ from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.base_notifier import BaseNotifier
from pipecat.sync.event_notifier import EventNotifier from pipecat.sync.event_notifier import EventNotifier
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
classifier_statement = "Determine if the user's statement ends with a complete thought and you should respond. The user text is transcribed speech. It may contain multiple fragments concatentated together. You are trying to determine only the completeness of the last user statement. The previous assistant statement is provided only for context. Categorize the text as either complete with the user now expecting a response, or incomplete. Return 'YES' if text is likely complete and the user is expecting a response. Return 'NO' if the text seems to be a partial expression or unfinished thought." classifier_statement = "Determine if the user's statement ends with a complete thought and you should respond. The user text is transcribed speech. It may contain multiple fragments concatentated together. You are trying to determine only the completeness of the last user statement. The previous assistant statement is provided only for context. Categorize the text as either complete with the user now expecting a response, or incomplete. Return 'YES' if text is likely complete and the user is expecting a response. Return 'NO' if the text seems to be a partial expression or unfinished thought."
@@ -204,15 +200,13 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
None, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -358,32 +352,42 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."}) messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_app_message") @transport.event_handler("on_app_message")
async def on_app_message(transport, message, sender): async def on_app_message(transport, message):
logger.debug(f"Received app message: {message} - {sender}") logger.debug(f"Received app message: {message}")
if "message" not in message: if "message" not in message:
return return
await task.queue_frames( await task.queue_frames(
[ [
UserStartedSpeakingFrame(), UserStartedSpeakingFrame(),
TranscriptionFrame( TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]),
user_id=sender, timestamp=time.time(), text=message["message"]
),
UserStoppedSpeakingFrame(), UserStoppedSpeakingFrame(),
] ]
) )
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -6,14 +6,11 @@
import asyncio import asyncio
import os import os
import sys
import time import time
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from openai.types.chat import ChatCompletionToolParam from openai.types.chat import ChatCompletionToolParam
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import ( from pipecat.frames.frames import (
@@ -50,13 +47,12 @@ from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.base_notifier import BaseNotifier
from pipecat.sync.event_notifier import EventNotifier from pipecat.sync.event_notifier import EventNotifier
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
classifier_statement = """CRITICAL INSTRUCTION: classifier_statement = """CRITICAL INSTRUCTION:
You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO". You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO".
@@ -408,15 +404,13 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
None, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -566,9 +560,9 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
messages.append( messages.append(
{ {
@@ -579,24 +573,34 @@ async def main():
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_app_message") @transport.event_handler("on_app_message")
async def on_app_message(transport, message, sender): async def on_app_message(transport, message):
logger.debug(f"Received app message: {message} - {sender}") logger.debug(f"Received app message: {message}")
if "message" not in message: if "message" not in message:
return return
await task.queue_frames( await task.queue_frames(
[ [
UserStartedSpeakingFrame(), UserStartedSpeakingFrame(),
TranscriptionFrame( TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]),
user_id=sender, timestamp=time.time(), text=message["message"]
),
UserStoppedSpeakingFrame(), UserStoppedSpeakingFrame(),
] ]
) )
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -6,14 +6,11 @@
import asyncio import asyncio
import os import os
import sys
import time import time
import aiohttp
import google.ai.generativelanguage as glm import google.ai.generativelanguage as glm
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import ( from pipecat.frames.frames import (
@@ -47,12 +44,12 @@ from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService
from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.base_notifier import BaseNotifier
from pipecat.sync.event_notifier import EventNotifier from pipecat.sync.event_notifier import EventNotifier
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
TRANSCRIBER_MODEL = "gemini-2.0-flash-001" TRANSCRIBER_MODEL = "gemini-2.0-flash-001"
CLASSIFIER_MODEL = "gemini-2.0-flash-001" CLASSIFIER_MODEL = "gemini-2.0-flash-001"
@@ -626,15 +623,13 @@ class OutputGate(FrameProcessor):
break break
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
None, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -685,9 +680,7 @@ async def main():
audio_accumulater = AudioAccumulator() audio_accumulater = AudioAccumulator()
# This sends a UserStoppedSpeakingFrame and triggers the notifier event # This sends a UserStoppedSpeakingFrame and triggers the notifier event
completeness_check = CompletenessCheck( completeness_check = CompletenessCheck(notifier=notifier, audio_accumulator=audio_accumulater)
notifier=notifier, audio_accumulator=audio_accumulater
)
async def block_user_stopped_speaking(frame): async def block_user_stopped_speaking(frame):
return not isinstance(frame, UserStoppedSpeakingFrame) return not isinstance(frame, UserStoppedSpeakingFrame)
@@ -744,31 +737,41 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
await transport.capture_participant_transcription(participant["id"]) logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_app_message") @transport.event_handler("on_app_message")
async def on_app_message(transport, message, sender): async def on_app_message(transport, message):
logger.debug(f"Received app message: {message} - {sender}") logger.debug(f"Received app message: {message}")
if "message" not in message: if "message" not in message:
return return
await task.queue_frames( await task.queue_frames(
[ [
UserStartedSpeakingFrame(), UserStartedSpeakingFrame(),
TranscriptionFrame( TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]),
user_id=sender, timestamp=time.time(), text=message["message"]
),
UserStoppedSpeakingFrame(), UserStoppedSpeakingFrame(),
] ]
) )
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -10,9 +10,9 @@ import os
import sys import sys
import aiohttp import aiohttp
from daily_runner import configure_with_args
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure_with_args
from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer

View File

@@ -0,0 +1,140 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""
Usage
-----
Set the path to your background audio file using the `INPUT_AUDIO_PATH` environment variable, then run the bot using:
INPUT_AUDIO_PATH=path/to/your_audio.mp3 python 23-bot-background-sound.py
Example:
INPUT_AUDIO_PATH=my_audio.mp3 python 23-bot-background-sound.py
"""
import asyncio
import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import MixerEnableFrame, MixerUpdateSettingsFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True)
audio_path = os.getenv("INPUT_AUDIO_PATH")
if not audio_path:
raise ValueError("No INPUT_AUDIO_PATH specified in environment variables")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
soundfile_mixer = SoundfileMixer(
sound_files={"office": audio_path},
default_sound="office",
volume=2.0,
)
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
audio_out_mixer=soundfile_mixer,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True,
),
)
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(), # Transport user input
stt, # STT service
context_aggregator.user(), # User responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
),
)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}")
# Show how to use mixer control frames.
await asyncio.sleep(10.0)
await task.queue_frame(MixerUpdateSettingsFrame({"volume": 0.5}))
await asyncio.sleep(5.0)
await task.queue_frame(MixerEnableFrame(False))
await asyncio.sleep(5.0)
await task.queue_frame(MixerEnableFrame(True))
await asyncio.sleep(5.0)
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
if __name__ == "__main__":
from run import main
main()

View File

@@ -6,14 +6,12 @@
import asyncio import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from openai.types.chat import ChatCompletionToolParam
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -23,13 +21,12 @@ from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFil
from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.deepgram.tts import DeepgramTTSService
from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
# Add a delay to test interruption during function calls # Add a delay to test interruption during function calls
@@ -39,15 +36,13 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
await result_callback({"conditions": "nice", "temperature": "75"}) await result_callback({"conditions": "nice", "temperature": "75"})
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, _) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
None, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
@@ -56,6 +51,7 @@ async def main():
) )
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
# Configure the mute processor with both strategies # Configure the mute processor with both strategies
stt_mute_processor = STTMuteFilter( stt_mute_processor = STTMuteFilter(
config=STTMuteConfig( config=STTMuteConfig(
@@ -71,15 +67,10 @@ async def main():
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
llm.register_function("get_current_weather", fetch_weather_from_api) llm.register_function("get_current_weather", fetch_weather_from_api)
tools = [ weather_function = FunctionSchema(
ChatCompletionToolParam( name="get_current_weather",
type="function", description="Get the current weather",
function={ properties={
"name": "get_current_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": { "location": {
"type": "string", "type": "string",
"description": "The city and state, e.g. San Francisco, CA", "description": "The city and state, e.g. San Francisco, CA",
@@ -87,14 +78,12 @@ async def main():
"format": { "format": {
"type": "string", "type": "string",
"enum": ["celsius", "fahrenheit"], "enum": ["celsius", "fahrenheit"],
"description": "The temperature unit to use. Infer this from the users location.", "description": "The temperature unit to use. Infer this from the user's location.",
},
},
"required": ["location", "format"],
}, },
}, },
required=["location", "format"],
) )
] tools = ToolsSchema(standard_tools=[weather_function])
messages = [ messages = [
{ {
@@ -121,8 +110,9 @@ async def main():
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation with a weather-related prompt # Kick off the conversation with a weather-related prompt
messages.append( messages.append(
{ {
@@ -132,10 +122,21 @@ async def main():
) )
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,16 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from dataclasses import dataclass from dataclasses import dataclass
import aiohttp
import google.ai.generativelanguage as glm import google.ai.generativelanguage as glm
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import ( from pipecat.frames.frames import (
@@ -37,13 +33,12 @@ from pipecat.processors.aggregators.openai_llm_context import (
from pipecat.processors.frame_processor import FrameProcessor from pipecat.processors.frame_processor import FrameProcessor
from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
# #
# The system prompt for the main conversation. # The system prompt for the main conversation.
# #
@@ -273,18 +268,14 @@ class TranscriptionContextFixup(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( transport = SmallWebRTCTransport(
room_url, webrtc_connection=webrtc_connection,
token, params=TransportParams(
"Respond bot", audio_in_enabled=True,
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
# No transcription at all. just audio input to Gemini!
# transcription_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(), vad_analyzer=SileroVADAnalyzer(),
vad_audio_passthrough=True, vad_audio_passthrough=True,
@@ -360,15 +351,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation. # Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -20,39 +16,48 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
# Load environment variables
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Initialize the SmallWebRTCTransport with the connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, token) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( audio_in_enabled=True,
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
camera_in_enabled=False,
vad_enabled=True, vad_enabled=True,
vad_audio_passthrough=True, vad_audio_passthrough=True,
# set stop_secs to something roughly similar to the internal setting # set stop_secs to something roughly similar to the internal setting
# of the Multimodal Live api, just to align events. This doesn't really # of the Multimodal Live api, just to align events.
# matter because we can only use the Multimodal Live API's phrase
# endpointing, for now.
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
), ),
) )
# Create the Gemini Multimodal Live LLM service
system_instruction = f"""
You are a helpful AI assistant.
Your goal is to demonstrate your capabilities in a helpful and engaging way.
Your output will be converted to audio so don't include special characters in your answers.
Respond to what the user said in a creative and helpful way.
"""
llm = GeminiMultimodalLiveLLMService( llm = GeminiMultimodalLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"), api_key=os.getenv("GOOGLE_API_KEY"),
# system_instruction="Talk like a pirate." system_instruction=system_instruction,
voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
transcribe_user_audio=True,
) )
# Build the pipeline
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), transport.input(),
@@ -61,6 +66,7 @@ async def main():
] ]
) )
# Configure the pipeline task
task = PipelineTask( task = PipelineTask(
pipeline, pipeline,
params=PipelineParams( params=PipelineParams(
@@ -70,25 +76,40 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") # Handle client connection event
async def on_first_participant_joined(transport, participant): @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frames( await task.queue_frames(
[ [
LLMMessagesAppendFrame( LLMMessagesAppendFrame(
messages=[ messages=[
{ {
"role": "user", "role": "user",
"content": "Greet the user.", "content": f"Greet the user and introduce yourself.",
} }
] ]
) )
] ]
) )
runner = PipelineRunner() # Handle client disconnection events
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
# Run the pipeline
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,14 +4,10 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -20,23 +16,21 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def run_bot(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Starting bot")
async def main(): # Initialize the SmallWebRTCTransport with the connection
async with aiohttp.ClientSession() as session: transport = SmallWebRTCTransport(
(room_url, token) = await configure(session) webrtc_connection=webrtc_connection,
params=TransportParams(
transport = DailyTransport( audio_in_enabled=True,
room_url,
token,
"Respond bot",
DailyParams(
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_audio_passthrough=True, vad_audio_passthrough=True,
@@ -53,7 +47,6 @@ async def main():
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
# system_instruction="Talk like a pirate." # system_instruction="Talk like a pirate."
transcribe_user_audio=True, transcribe_user_audio=True,
transcribe_model_audio=True,
# inference_on_context_initialization=False, # inference_on_context_initialization=False,
) )
@@ -96,14 +89,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -4,15 +4,11 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio
import os import os
import sys
from datetime import datetime from datetime import datetime
import aiohttp
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
@@ -23,13 +19,12 @@ from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True) load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
temperature = 75 if args["format"] == "fahrenheit" else 24 temperature = 75 if args["format"] == "fahrenheit" else 24
@@ -51,15 +46,14 @@ for the weather, call this function.
""" """
async def main(): async def run_bot(webrtc_connection: SmallWebRTCConnection):
async with aiohttp.ClientSession() as session: logger.info(f"Starting bot")
(room_url, token) = await configure(session)
transport = DailyTransport( # Initialize the SmallWebRTCTransport with the connection
room_url, transport = SmallWebRTCTransport(
token, webrtc_connection=webrtc_connection,
"Respond bot", params=TransportParams(
DailyParams( audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
vad_enabled=True, vad_enabled=True,
vad_audio_passthrough=True, vad_audio_passthrough=True,
@@ -124,14 +118,27 @@ async def main():
), ),
) )
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_client_connected")
async def on_first_participant_joined(transport, participant): async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner() @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) from run import main
main()

View File

@@ -9,9 +9,9 @@ import os
import sys import sys
import aiohttp import aiohttp
from daily_runner import configure
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
@@ -53,7 +53,6 @@ async def main():
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
# system_instruction="Talk like a pirate." # system_instruction="Talk like a pirate."
transcribe_user_audio=True, transcribe_user_audio=True,
transcribe_model_audio=True,
# inference_on_context_initialization=False, # inference_on_context_initialization=False,
) )

Some files were not shown because too many files have changed in this diff Show More