Updating foundation examples to use SmallWebRTCTransport and pipecat-ai-small-webrtc-prebuilt (#1534)
Co-authored-by: Filipi Fuchter <filipi@daily.co>
This commit is contained in:
@@ -4,54 +4,54 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
|
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.services.piper.tts import PiperTTSService
|
from pipecat.services.piper.tts import PiperTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Create a transport using the WebRTC connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, _) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
audio_out_enabled=True,
|
||||||
room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True)
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
tts = PiperTTSService(
|
tts = PiperTTSService(
|
||||||
base_url=os.getenv("PIPER_BASE_URL"), aiohttp_session=session, sample_rate=24000
|
base_url=os.getenv("PIPER_BASE_URL"), aiohttp_session=session, sample_rate=24000
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(Pipeline([tts, transport.output()]))
|
task = PipelineTask(Pipeline([tts, transport.output()]))
|
||||||
|
|
||||||
# Register an event handler so we can play the audio when the
|
# Register an event handler so we can play the audio when the client joins
|
||||||
# participant joins.
|
@transport.event_handler("on_client_connected")
|
||||||
@transport.event_handler("on_first_participant_joined")
|
async def on_client_connected(transport, client):
|
||||||
async def on_first_participant_joined(transport, participant):
|
await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
|
||||||
await task.queue_frames(
|
|
||||||
[TTSSpeakFrame(f"Hello there, how are you today ?"), EndFrame()]
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
)
|
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
59
examples/foundational/01-say-one-thing-rime.py
Normal file
59
examples/foundational/01-say-one-thing-rime.py
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
|
||||||
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
|
from pipecat.pipeline.task import PipelineTask
|
||||||
|
from pipecat.services.rime.tts import RimeHttpTTSService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
|
# Create a transport using the WebRTC connection
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_out_enabled=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
tts = RimeHttpTTSService(
|
||||||
|
api_key=os.getenv("RIME_API_KEY", ""),
|
||||||
|
voice_id="rex",
|
||||||
|
aiohttp_session=session,
|
||||||
|
)
|
||||||
|
|
||||||
|
task = PipelineTask(Pipeline([tts, transport.output()]))
|
||||||
|
|
||||||
|
# Register an event handler so we can play the audio when the client joins
|
||||||
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
@@ -4,34 +4,32 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
|
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Create a transport using the WebRTC connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, _) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
audio_out_enabled=True,
|
||||||
room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True)
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
@@ -39,21 +37,19 @@ async def main():
|
|||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(Pipeline([tts, transport.output()]))
|
task = PipelineTask(Pipeline([tts, transport.output()]))
|
||||||
|
|
||||||
# Register an event handler so we can play the audio when the
|
# Register an event handler so we can play the audio when the client joins
|
||||||
# participant joins.
|
@transport.event_handler("on_client_connected")
|
||||||
@transport.event_handler("on_first_participant_joined")
|
async def on_client_connected(transport, client):
|
||||||
async def on_first_participant_joined(transport, participant):
|
await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
|
||||||
participant_name = participant.get("info", {}).get("userName", "")
|
|
||||||
await task.queue_frames(
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
[TTSSpeakFrame(f"Hello there, {participant_name}!"), EndFrame()]
|
|
||||||
)
|
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,51 +4,49 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
|
from pipecat.frames.frames import EndFrame, TTSSpeakFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.services.riva.tts import FastPitchTTSService
|
from pipecat.services.riva.tts import FastPitchTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Create a transport using the WebRTC connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, _) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
audio_out_enabled=True,
|
||||||
room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True)
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
|
tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(Pipeline([tts, transport.output()]))
|
task = PipelineTask(Pipeline([tts, transport.output()]))
|
||||||
|
|
||||||
# Register an event handler so we can play the audio when the
|
# Register an event handler so we can play the audio when the client joins
|
||||||
# participant joins.
|
@transport.event_handler("on_client_connected")
|
||||||
@transport.event_handler("on_first_participant_joined")
|
async def on_client_connected(transport, client):
|
||||||
async def on_first_participant_joined(transport, participant):
|
await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()])
|
||||||
participant_name = participant.get("info", {}).get("userName", "")
|
|
||||||
await task.queue_frames([TTSSpeakFrame(f"Aloha, {participant_name}!"), EndFrame()])
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import EndFrame, LLMMessagesFrame
|
from pipecat.frames.frames import EndFrame, LLMMessagesFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -19,20 +15,22 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Create a transport using the WebRTC connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, _) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
audio_out_enabled=True,
|
||||||
room_url, None, "Say One Thing From an LLM", DailyParams(audio_out_enabled=True)
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
@@ -49,16 +47,19 @@ async def main():
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(Pipeline([llm, tts, transport.output()]))
|
task = PipelineTask(Pipeline([llm, tts, transport.output()]))
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
# Register an event handler so we can play the audio when the client joins
|
||||||
async def on_first_participant_joined(transport, participant):
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
await task.queue_frames([LLMMessagesFrame(messages), EndFrame()])
|
await task.queue_frames([LLMMessagesFrame(messages), EndFrame()])
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,59 +4,67 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import TextFrame
|
from pipecat.frames.frames import TextFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.services.fal.image import FalImageGenService
|
from pipecat.services.fal.image import FalImageGenService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Create a transport using the WebRTC connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, _) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
camera_out_enabled=True,
|
||||||
room_url,
|
camera_out_width=1024,
|
||||||
None,
|
camera_out_height=1024,
|
||||||
"Show a still frame image",
|
),
|
||||||
DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
imagegen = FalImageGenService(
|
imagegen = FalImageGenService(
|
||||||
params=FalImageGenService.InputParams(image_size="square_hd"),
|
params=FalImageGenService.InputParams(image_size="square_hd"),
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
key=os.getenv("FAL_KEY"),
|
key=os.getenv("FAL_KEY"),
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(Pipeline([imagegen, transport.output()]))
|
task = PipelineTask(Pipeline([imagegen, transport.output()]))
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
# Register an event handler so we can play the audio when the client joins
|
||||||
async def on_first_participant_joined(transport, participant):
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
await task.queue_frame(TextFrame("a cat in the style of picasso"))
|
await task.queue_frame(TextFrame("a cat in the style of picasso"))
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,62 +4,67 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import EndFrame, TextFrame
|
from pipecat.frames.frames import TextFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.services.google.image import GoogleImageGenService
|
from pipecat.services.google.image import GoogleImageGenService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Create a transport using the WebRTC connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, _) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
camera_out_enabled=True,
|
||||||
room_url,
|
camera_out_width=1024,
|
||||||
None,
|
camera_out_height=1024,
|
||||||
"Show a still frame image",
|
),
|
||||||
DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
imagegen = GoogleImageGenService(
|
imagegen = GoogleImageGenService(
|
||||||
api_key=os.getenv("GOOGLE_API_KEY"),
|
api_key=os.getenv("GOOGLE_API_KEY"),
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(
|
task = PipelineTask(
|
||||||
Pipeline([imagegen, transport.output()]),
|
Pipeline([imagegen, transport.output()]),
|
||||||
params=PipelineParams(enable_metrics=True),
|
params=PipelineParams(enable_metrics=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
# Register an event handler so we can play the audio when the client joins
|
||||||
async def on_first_participant_joined(transport, participant):
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
await task.queue_frame(TextFrame("a cat in the style of picasso"))
|
await task.queue_frame(TextFrame("a cat in the style of picasso"))
|
||||||
await task.queue_frame(TextFrame("a dog in the style of picasso"))
|
await task.queue_frame(TextFrame("a dog in the style of picasso"))
|
||||||
await task.queue_frame(TextFrame("a fish in the style of picasso"))
|
await task.queue_frame(TextFrame("a fish in the style of picasso"))
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
await task.queue_frame(EndFrame())
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
from daily_runner import configure
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
|
from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
|
||||||
from pipecat.pipeline.merge_pipeline import SequentialMergePipeline
|
from pipecat.pipeline.merge_pipeline import SequentialMergePipeline
|
||||||
|
|||||||
@@ -4,15 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
DataFrame,
|
DataFrame,
|
||||||
@@ -30,13 +27,12 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
|||||||
from pipecat.services.cartesia.tts import CartesiaHttpTTSService
|
from pipecat.services.cartesia.tts import CartesiaHttpTTSService
|
||||||
from pipecat.services.fal.image import FalImageGenService
|
from pipecat.services.fal.image import FalImageGenService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class MonthFrame(DataFrame):
|
class MonthFrame(DataFrame):
|
||||||
@@ -67,15 +63,19 @@ class MonthPrepender(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
"""Run the Calendar Month Narration bot using WebRTC transport.
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
Args:
|
||||||
room_url,
|
webrtc_connection: The WebRTC connection to use
|
||||||
None,
|
room_name: Optional room name for display purposes
|
||||||
"Month Narration Bot",
|
"""
|
||||||
DailyParams(
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
|
# Create a transport using the WebRTC connection
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
camera_out_enabled=True,
|
camera_out_enabled=True,
|
||||||
camera_out_width=1024,
|
camera_out_width=1024,
|
||||||
@@ -83,6 +83,8 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session for API calls
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
|
|
||||||
tts = CartesiaHttpTTSService(
|
tts = CartesiaHttpTTSService(
|
||||||
@@ -144,14 +146,30 @@ async def main():
|
|||||||
frames.append(MonthFrame(month=month))
|
frames.append(MonthFrame(month=month))
|
||||||
frames.append(LLMMessagesFrame(messages))
|
frames.append(LLMMessagesFrame(messages))
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
|
# Set up transport event handlers
|
||||||
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Start the month narration once connected
|
||||||
await task.queue_frames(frames)
|
await task.queue_frames(frames)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
# Run the pipeline
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import Frame, MetricsFrame
|
from pipecat.frames.frames import Frame, MetricsFrame
|
||||||
@@ -27,14 +23,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class MetricsLogger(FrameProcessor):
|
class MetricsLogger(FrameProcessor):
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -56,22 +52,22 @@ class MetricsLogger(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -94,6 +90,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -111,21 +108,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -20,7 +16,6 @@ from pipecat.frames.frames import (
|
|||||||
BotStoppedSpeakingFrame,
|
BotStoppedSpeakingFrame,
|
||||||
Frame,
|
Frame,
|
||||||
OutputImageRawFrame,
|
OutputImageRawFrame,
|
||||||
TextFrame,
|
|
||||||
)
|
)
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
@@ -28,14 +23,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class ImageSyncAggregator(FrameProcessor):
|
class ImageSyncAggregator(FrameProcessor):
|
||||||
def __init__(self, speaking_path: str, waiting_path: str):
|
def __init__(self, speaking_path: str, waiting_path: str):
|
||||||
@@ -72,25 +67,25 @@ class ImageSyncAggregator(FrameProcessor):
|
|||||||
await self.push_frame(frame)
|
await self.push_frame(frame)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
camera_out_enabled=True,
|
camera_out_enabled=True,
|
||||||
camera_out_width=1024,
|
camera_out_width=1024,
|
||||||
camera_out_height=1024,
|
camera_out_height=1024,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -116,6 +111,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -135,20 +131,26 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
participant_name = participant.get("info", {}).get("userName", "")
|
logger.info(f"Client connected")
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
# Kick off the conversation.
|
||||||
await task.queue_frames([TextFrame(f"Hi there {participant_name}!")])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -1,104 +0,0 @@
|
|||||||
#
|
|
||||||
# Copyright (c) 2024–2025, Daily
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
|
||||||
#
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from loguru import logger
|
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|
||||||
from pipecat.processors.audio.vad.silero import SileroVAD
|
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
|
||||||
|
|
||||||
load_dotenv(override=True)
|
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
|
||||||
audio_out_enabled=True,
|
|
||||||
transcription_enabled=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
vad = SileroVAD()
|
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
|
||||||
)
|
|
||||||
|
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
|
||||||
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "system",
|
|
||||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
context = OpenAILLMContext(messages)
|
|
||||||
context_aggregator = llm.create_context_aggregator(context)
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
|
||||||
[
|
|
||||||
transport.input(),
|
|
||||||
vad,
|
|
||||||
context_aggregator.user(),
|
|
||||||
llm,
|
|
||||||
tts,
|
|
||||||
transport.output(),
|
|
||||||
context_aggregator.assistant(),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
task = PipelineTask(
|
|
||||||
pipeline,
|
|
||||||
params=PipelineParams(
|
|
||||||
allow_interruptions=True,
|
|
||||||
enable_metrics=True,
|
|
||||||
enable_usage_metrics=True,
|
|
||||||
report_only_initial_ttfb=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
# Kick off the conversation.
|
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
|
||||||
async def on_participant_left(transport, participant, reason):
|
|
||||||
await task.cancel()
|
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
await runner.run(task)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -19,31 +15,31 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -64,6 +60,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -82,21 +79,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -1,106 +0,0 @@
|
|||||||
#
|
|
||||||
# Copyright (c) 2024–2025, Daily
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
|
||||||
#
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from loguru import logger
|
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|
||||||
from pipecat.services.anthropic.llm import AnthropicLLMService
|
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
|
||||||
|
|
||||||
load_dotenv(override=True)
|
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
|
||||||
)
|
|
||||||
|
|
||||||
llm = AnthropicLLMService(
|
|
||||||
api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-opus-20240229"
|
|
||||||
)
|
|
||||||
|
|
||||||
# todo: think more about how to handle system prompts in a more general way. OpenAI,
|
|
||||||
# Google, and Anthropic all have slightly different approaches to providing a system
|
|
||||||
# prompt.
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "system",
|
|
||||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
context = OpenAILLMContext(messages)
|
|
||||||
context_aggregator = llm.create_context_aggregator(context)
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
|
||||||
[
|
|
||||||
transport.input(), # Transport user input
|
|
||||||
context_aggregator.user(), # User responses
|
|
||||||
llm, # LLM
|
|
||||||
tts, # TTS
|
|
||||||
transport.output(), # Transport bot output
|
|
||||||
context_aggregator.assistant(), # Assistant spoken responses
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
task = PipelineTask(
|
|
||||||
pipeline,
|
|
||||||
params=PipelineParams(
|
|
||||||
allow_interruptions=True,
|
|
||||||
enable_metrics=True,
|
|
||||||
enable_usage_metrics=True,
|
|
||||||
report_only_initial_ttfb=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
# Kick off the conversation.
|
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
|
||||||
async def on_participant_left(transport, participant, reason):
|
|
||||||
await task.cancel()
|
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
await runner.run(task)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
106
examples/foundational/07a-interruptible-vad.py
Normal file
106
examples/foundational/07a-interruptible-vad.py
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.processors.audio.vad.silero import SileroVAD
|
||||||
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
|
vad = SileroVAD()
|
||||||
|
|
||||||
|
tts = CartesiaTTSService(
|
||||||
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
|
)
|
||||||
|
|
||||||
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
context = OpenAILLMContext(messages)
|
||||||
|
context_aggregator = llm.create_context_aggregator(context)
|
||||||
|
|
||||||
|
pipeline = Pipeline(
|
||||||
|
[
|
||||||
|
transport.input(),
|
||||||
|
stt,
|
||||||
|
vad,
|
||||||
|
context_aggregator.user(),
|
||||||
|
llm,
|
||||||
|
tts,
|
||||||
|
transport.output(),
|
||||||
|
context_aggregator.assistant(),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(
|
||||||
|
allow_interruptions=True,
|
||||||
|
enable_metrics=True,
|
||||||
|
enable_usage_metrics=True,
|
||||||
|
report_only_initial_ttfb=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
@@ -4,11 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
|
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
|
||||||
from langchain_community.chat_message_histories import ChatMessageHistory
|
from langchain_community.chat_message_histories import ChatMessageHistory
|
||||||
@@ -16,7 +13,6 @@ from langchain_core.chat_history import BaseChatMessageHistory
|
|||||||
from langchain_core.runnables.history import RunnableWithMessageHistory
|
from langchain_core.runnables.history import RunnableWithMessageHistory
|
||||||
from langchain_openai import ChatOpenAI
|
from langchain_openai import ChatOpenAI
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import LLMMessagesFrame
|
from pipecat.frames.frames import LLMMessagesFrame
|
||||||
@@ -29,14 +25,14 @@ from pipecat.processors.aggregators.llm_response import (
|
|||||||
)
|
)
|
||||||
from pipecat.processors.frameworks.langchain import LangchainProcessor
|
from pipecat.processors.frameworks.langchain import LangchainProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
message_store = {}
|
message_store = {}
|
||||||
|
|
||||||
|
|
||||||
@@ -46,22 +42,22 @@ def get_session_history(session_id: str) -> BaseChatMessageHistory:
|
|||||||
return message_store[session_id]
|
return message_store[session_id]
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -93,6 +89,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
tma_in, # User responses
|
tma_in, # User responses
|
||||||
lc, # Langchain
|
lc, # Langchain
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -111,10 +108,9 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
lc.set_participant_id(participant["id"])
|
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
# the `LLMMessagesFrame` will be picked up by the LangchainProcessor using
|
# the `LLMMessagesFrame` will be picked up by the LangchainProcessor using
|
||||||
# only the content of the last message to inject it in the prompt defined
|
# only the content of the last message to inject it in the prompt defined
|
||||||
@@ -122,14 +118,21 @@ async def main():
|
|||||||
messages = [({"content": "Please briefly introduce yourself to the user."})]
|
messages = [({"content": "Please briefly introduce yourself to the user."})]
|
||||||
await task.queue_frames([LLMMessagesFrame(messages)])
|
await task.queue_frames([LLMMessagesFrame(messages)])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from deepgram import LiveOptions
|
from deepgram import LiveOptions
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
BotInterruptionFrame,
|
BotInterruptionFrame,
|
||||||
@@ -27,23 +23,19 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.deepgram.tts import DeepgramTTSService
|
from pipecat.services.deepgram.tts import DeepgramTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, _) = await configure(session)
|
params=TransportParams(
|
||||||
|
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
),
|
),
|
||||||
@@ -98,20 +90,28 @@ async def main():
|
|||||||
async def on_utterance_end(stt, *args, **kwargs):
|
async def on_utterance_end(stt, *args, **kwargs):
|
||||||
await task.queue_frames([StopInterruptionFrame(), UserStoppedSpeakingFrame()])
|
await task.queue_frames([StopInterruptionFrame(), UserStoppedSpeakingFrame()])
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.deepgram.tts import DeepgramTTSService
|
from pipecat.services.deepgram.tts import DeepgramTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, _) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -83,20 +76,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,45 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService
|
from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = ElevenLabsHttpTTSService(
|
tts = ElevenLabsHttpTTSService(
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
||||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
||||||
@@ -65,6 +64,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -83,21 +83,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
||||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
||||||
@@ -64,6 +60,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -82,21 +79,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.services.playht.tts import PlayHTHttpTTSService
|
from pipecat.services.playht.tts import PlayHTHttpTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = PlayHTHttpTTSService(
|
tts = PlayHTHttpTTSService(
|
||||||
user_id=os.getenv("PLAYHT_USER_ID"),
|
user_id=os.getenv("PLAYHT_USER_ID"),
|
||||||
api_key=os.getenv("PLAYHT_API_KEY"),
|
api_key=os.getenv("PLAYHT_API_KEY"),
|
||||||
@@ -65,6 +61,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -83,21 +80,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,47 +4,43 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.services.playht.tts import PlayHTTTSService
|
from pipecat.services.playht.tts import PlayHTTTSService
|
||||||
from pipecat.transcriptions.language import Language
|
from pipecat.transcriptions.language import Language
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = PlayHTTTSService(
|
tts = PlayHTTTSService(
|
||||||
user_id=os.getenv("PLAYHT_USER_ID"),
|
user_id=os.getenv("PLAYHT_USER_ID"),
|
||||||
api_key=os.getenv("PLAYHT_API_KEY"),
|
api_key=os.getenv("PLAYHT_API_KEY"),
|
||||||
@@ -67,6 +63,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -85,21 +82,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.azure.llm import AzureLLMService
|
from pipecat.services.azure.llm import AzureLLMService
|
||||||
from pipecat.services.azure.stt import AzureSTTService
|
from pipecat.services.azure.stt import AzureSTTService
|
||||||
from pipecat.services.azure.tts import AzureTTSService
|
from pipecat.services.azure.tts import AzureTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -93,21 +86,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -21,38 +17,27 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.services.openai.stt import OpenAISTTService
|
from pipecat.services.openai.stt import OpenAISTTService
|
||||||
from pipecat.services.openai.tts import OpenAITTSService
|
from pipecat.services.openai.tts import OpenAITTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
audio_out_sample_rate=24000,
|
|
||||||
transcription_enabled=False,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# You can use the OpenAI compatible API like Groq.
|
|
||||||
# stt = OpenAISTTService(
|
|
||||||
# base_url="https://api.groq.com/openai/v1",
|
|
||||||
# api_key="gsk_***",
|
|
||||||
# model="whisper-large-v3",
|
|
||||||
# )
|
|
||||||
stt = OpenAISTTService(
|
stt = OpenAISTTService(
|
||||||
api_key=os.getenv("OPENAI_API_KEY"),
|
api_key=os.getenv("OPENAI_API_KEY"),
|
||||||
model="gpt-4o-transcribe-latest",
|
model="gpt-4o-transcribe-latest",
|
||||||
@@ -89,27 +74,35 @@ async def main():
|
|||||||
pipeline,
|
pipeline,
|
||||||
params=PipelineParams(
|
params=PipelineParams(
|
||||||
allow_interruptions=True,
|
allow_interruptions=True,
|
||||||
|
audio_out_sample_rate=24000,
|
||||||
enable_metrics=True,
|
enable_metrics=True,
|
||||||
enable_usage_metrics=True,
|
enable_usage_metrics=True,
|
||||||
report_only_initial_ttfb=True,
|
report_only_initial_ttfb=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -20,31 +16,31 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openpipe.llm import OpenPipeLLMService
|
from pipecat.services.openpipe.llm import OpenPipeLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -71,6 +67,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -89,21 +86,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,45 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.services.xtts.tts import XTTSService
|
from pipecat.services.xtts.tts import XTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = XTTSService(
|
tts = XTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
voice_id="Claribel Dervla",
|
voice_id="Claribel Dervla",
|
||||||
@@ -65,6 +64,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -83,21 +83,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -23,23 +19,20 @@ from pipecat.services.gladia.config import GladiaInputParams, LanguageConfig
|
|||||||
from pipecat.services.gladia.stt import GladiaSTTService
|
from pipecat.services.gladia.stt import GladiaSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transcriptions.language import Language
|
from pipecat.transcriptions.language import Language
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -48,7 +41,7 @@ async def main():
|
|||||||
)
|
)
|
||||||
|
|
||||||
stt = GladiaSTTService(
|
stt = GladiaSTTService(
|
||||||
api_key=os.getenv("GLADIA_API_KEY"),
|
api_key=os.getenv("GLADIA_API_KEY", ""),
|
||||||
params=GladiaInputParams(
|
params=GladiaInputParams(
|
||||||
language_config=LanguageConfig(
|
language_config=LanguageConfig(
|
||||||
languages=[Language.EN],
|
languages=[Language.EN],
|
||||||
@@ -57,16 +50,16 @@ async def main():
|
|||||||
)
|
)
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY", ""),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
)
|
)
|
||||||
|
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY", ""), model="gpt-4o")
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
"content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -95,22 +88,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
# Register an event handler to exit the application when the user leaves.
|
@transport.event_handler("on_client_disconnected")
|
||||||
@transport.event_handler("on_participant_left")
|
async def on_client_disconnected(transport, client):
|
||||||
async def on_participant_left(transport, participant, reason):
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.lmnt.tts import LmntTTSService
|
from pipecat.services.lmnt.tts import LmntTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = LmntTTSService(api_key=os.getenv("LMNT_API_KEY"), voice_id="morgan")
|
tts = LmntTTSService(api_key=os.getenv("LMNT_API_KEY"), voice_id="morgan")
|
||||||
|
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
@@ -61,6 +57,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User respones
|
context_aggregator.user(), # User respones
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -79,21 +76,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
102
examples/foundational/07l-interruptible-groq.py
Normal file
102
examples/foundational/07l-interruptible-groq.py
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.groq.llm import GroqLLMService
|
||||||
|
from pipecat.services.groq.stt import GroqSTTService
|
||||||
|
from pipecat.services.groq.tts import GroqTTSService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
vad_enabled=True,
|
||||||
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"))
|
||||||
|
|
||||||
|
llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile")
|
||||||
|
|
||||||
|
tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY"))
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
context = OpenAILLMContext(messages)
|
||||||
|
context_aggregator = llm.create_context_aggregator(context)
|
||||||
|
|
||||||
|
pipeline = Pipeline(
|
||||||
|
[
|
||||||
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
|
context_aggregator.user(), # User responses
|
||||||
|
llm, # LLM
|
||||||
|
tts, # TTS
|
||||||
|
transport.output(), # Transport bot output
|
||||||
|
context_aggregator.assistant(), # Assistant spoken responses
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(
|
||||||
|
allow_interruptions=True,
|
||||||
|
enable_metrics=True,
|
||||||
|
enable_usage_metrics=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
#
|
|
||||||
# Copyright (c) 2024–2025, Daily
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
|
||||||
#
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from loguru import logger
|
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
|
||||||
from pipecat.services.together.llm import TogetherLLMService
|
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
|
||||||
|
|
||||||
load_dotenv(override=True)
|
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
|
||||||
)
|
|
||||||
|
|
||||||
llm = TogetherLLMService(
|
|
||||||
api_key=os.getenv("TOGETHER_API_KEY"),
|
|
||||||
model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
|
|
||||||
params=TogetherLLMService.InputParams(
|
|
||||||
temperature=1.0,
|
|
||||||
top_p=0.9,
|
|
||||||
top_k=40,
|
|
||||||
extra={
|
|
||||||
"frequency_penalty": 2.0,
|
|
||||||
"presence_penalty": 0.0,
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "system",
|
|
||||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond in plain language. Respond to what the user said in a creative and helpful way.",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
context = OpenAILLMContext(messages)
|
|
||||||
context_aggregator = llm.create_context_aggregator(context)
|
|
||||||
user_aggregator = context_aggregator.user()
|
|
||||||
assistant_aggregator = context_aggregator.assistant()
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
|
||||||
[
|
|
||||||
transport.input(), # Transport user input
|
|
||||||
user_aggregator, # User responses
|
|
||||||
llm, # LLM
|
|
||||||
tts, # TTS
|
|
||||||
transport.output(), # Transport bot output
|
|
||||||
assistant_aggregator, # Assistant spoken responses
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
task = PipelineTask(
|
|
||||||
pipeline,
|
|
||||||
params=PipelineParams(
|
|
||||||
allow_interruptions=True,
|
|
||||||
enable_metrics=True,
|
|
||||||
enable_usage_metrics=True,
|
|
||||||
report_only_initial_ttfb=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
# Kick off the conversation.
|
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
|
||||||
async def on_participant_left(transport, participant, reason):
|
|
||||||
await task.cancel()
|
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
await runner.run(task)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.aws.tts import PollyTTSService
|
from pipecat.services.aws.tts import PollyTTSService
|
||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, _) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -89,21 +82,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -22,23 +18,20 @@ from pipecat.services.google.llm import GoogleLLMService
|
|||||||
from pipecat.services.google.stt import GoogleSTTService
|
from pipecat.services.google.stt import GoogleSTTService
|
||||||
from pipecat.services.google.tts import GoogleTTSService
|
from pipecat.services.google.tts import GoogleTTSService
|
||||||
from pipecat.transcriptions.language import Language
|
from pipecat.transcriptions.language import Language
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, _) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -48,11 +41,13 @@ async def main():
|
|||||||
|
|
||||||
stt = GoogleSTTService(
|
stt = GoogleSTTService(
|
||||||
params=GoogleSTTService.InputParams(languages=Language.EN_US),
|
params=GoogleSTTService.InputParams(languages=Language.EN_US),
|
||||||
|
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
|
||||||
)
|
)
|
||||||
|
|
||||||
tts = GoogleTTSService(
|
tts = GoogleTTSService(
|
||||||
voice_id="en-US-Journey-F",
|
voice_id="en-US-Chirp3-HD-Charon",
|
||||||
params=GoogleTTSService.InputParams(language=Language.EN_US),
|
params=GoogleTTSService.InputParams(language=Language.EN_US),
|
||||||
|
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
|
||||||
)
|
)
|
||||||
|
|
||||||
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
|
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
|
||||||
@@ -89,21 +84,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.assemblyai.stt import AssemblyAISTTService
|
from pipecat.services.assemblyai.stt import AssemblyAISTTService
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -88,21 +81,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.filters.krisp_filter import KrispFilter
|
from pipecat.audio.filters.krisp_filter import KrispFilter
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
@@ -22,23 +18,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.deepgram.tts import DeepgramTTSService
|
from pipecat.services.deepgram.tts import DeepgramTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -85,20 +78,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,45 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.services.rime.tts import RimeHttpTTSService
|
from pipecat.services.rime.tts import RimeHttpTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = RimeHttpTTSService(
|
tts = RimeHttpTTSService(
|
||||||
api_key=os.getenv("RIME_API_KEY", ""),
|
api_key=os.getenv("RIME_API_KEY", ""),
|
||||||
voice_id="rex",
|
voice_id="rex",
|
||||||
@@ -65,6 +64,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -83,21 +83,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.services.rime.tts import RimeTTSService
|
from pipecat.services.rime.tts import RimeTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = RimeTTSService(
|
tts = RimeTTSService(
|
||||||
api_key=os.getenv("RIME_API_KEY", ""),
|
api_key=os.getenv("RIME_API_KEY", ""),
|
||||||
voice_id="rex",
|
voice_id="rex",
|
||||||
@@ -64,6 +60,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -82,21 +79,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.nim.llm import NimLLMService
|
from pipecat.services.nim.llm import NimLLMService
|
||||||
from pipecat.services.riva.stt import ParakeetSTTService
|
from pipecat.services.riva.stt import ParakeetSTTService
|
||||||
from pipecat.services.riva.tts import FastPitchTTSService
|
from pipecat.services.riva.tts import FastPitchTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, _) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -47,9 +40,7 @@ async def main():
|
|||||||
|
|
||||||
stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
|
stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
|
||||||
|
|
||||||
llm = NimLLMService(
|
llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct")
|
||||||
api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct"
|
|
||||||
)
|
|
||||||
|
|
||||||
tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
|
tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
|
||||||
|
|
||||||
@@ -75,22 +66,38 @@ async def main():
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(
|
||||||
|
allow_interruptions=True,
|
||||||
|
enable_metrics=True,
|
||||||
|
enable_usage_metrics=True,
|
||||||
|
report_only_initial_ttfb=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,16 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
import google.ai.generativelanguage as glm
|
import google.ai.generativelanguage as glm
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -32,14 +28,15 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.frame_processor import FrameProcessor
|
from pipecat.processors.frame_processor import FrameProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
|
||||||
from pipecat.services.google.llm import GoogleLLMService
|
from pipecat.services.google.llm import GoogleLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.google.tts import GoogleTTSService
|
||||||
|
from pipecat.transcriptions.language import Language
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
marker = "|----|"
|
marker = "|----|"
|
||||||
system_message = f"""
|
system_message = f"""
|
||||||
@@ -193,15 +190,13 @@ class TanscriptionContextFixup(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
# No transcription at all. just audio input to Gemini!
|
# No transcription at all. just audio input to Gemini!
|
||||||
# transcription_enabled=True,
|
# transcription_enabled=True,
|
||||||
@@ -211,13 +206,14 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
|
||||||
)
|
|
||||||
|
|
||||||
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001")
|
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001")
|
||||||
|
|
||||||
|
tts = GoogleTTSService(
|
||||||
|
voice_id="en-US-Chirp3-HD-Charon",
|
||||||
|
params=GoogleTTSService.InputParams(language=Language.EN_US),
|
||||||
|
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
|
||||||
|
)
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -258,20 +254,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.fish.tts import FishAudioTTSService
|
from pipecat.services.fish.tts import FishAudioTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = FishAudioTTSService(
|
tts = FishAudioTTSService(
|
||||||
api_key=os.getenv("FISH_API_KEY"),
|
api_key=os.getenv("FISH_API_KEY"),
|
||||||
model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama
|
model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama
|
||||||
@@ -64,6 +60,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -82,21 +79,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -20,7 +16,9 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.ultravox.stt import UltravoxSTTService
|
from pipecat.services.ultravox.stt import UltravoxSTTService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
@@ -28,8 +26,6 @@ load_dotenv(override=True)
|
|||||||
# The Ultravox model is compute-intensive and performs best with GPU acceleration.
|
# The Ultravox model is compute-intensive and performs best with GPU acceleration.
|
||||||
# This can be deployed on cloud GPU providers like Cerebrium.ai for optimal performance.
|
# This can be deployed on cloud GPU providers like Cerebrium.ai for optimal performance.
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
# Want to initialize the ultravox processor since it takes time to load the model and dont
|
# Want to initialize the ultravox processor since it takes time to load the model and dont
|
||||||
# want to load it every time the pipeline is run
|
# want to load it every time the pipeline is run
|
||||||
@@ -39,17 +35,14 @@ ultravox_processor = UltravoxSTTService(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=False,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
@@ -78,14 +71,25 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.neuphonic.tts import NeuphonicHttpTTSService
|
from pipecat.services.neuphonic.tts import NeuphonicHttpTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = NeuphonicHttpTTSService(
|
tts = NeuphonicHttpTTSService(
|
||||||
api_key=os.getenv("NEUPHONIC_API_KEY"),
|
api_key=os.getenv("NEUPHONIC_API_KEY"),
|
||||||
voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily
|
voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily
|
||||||
@@ -64,6 +60,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -82,21 +79,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,46 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.neuphonic.tts import NeuphonicTTSService
|
from pipecat.services.neuphonic.tts import NeuphonicTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = NeuphonicTTSService(
|
tts = NeuphonicTTSService(
|
||||||
api_key=os.getenv("NEUPHONIC_API_KEY"),
|
api_key=os.getenv("NEUPHONIC_API_KEY"),
|
||||||
voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily
|
voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily
|
||||||
@@ -64,6 +60,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -82,21 +79,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
@transport.event_handler("on_client_disconnected")
|
||||||
async def on_participant_left(transport, participant, reason):
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -21,23 +17,20 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.fal.stt import FalSTTService
|
from pipecat.services.fal.stt import FalSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -88,22 +81,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
# Register an event handler to exit the application when the user leaves.
|
@transport.event_handler("on_client_disconnected")
|
||||||
@transport.event_handler("on_participant_left")
|
async def on_client_disconnected(transport, client):
|
||||||
async def on_participant_left(transport, participant, reason):
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -1,103 +0,0 @@
|
|||||||
#
|
|
||||||
# Copyright (c) 2024–2025, Daily
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
|
||||||
#
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from loguru import logger
|
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|
||||||
from pipecat.services.groq.llm import GroqLLMService
|
|
||||||
from pipecat.services.groq.stt import GroqSTTService
|
|
||||||
from pipecat.services.groq.tts import GroqTTSService
|
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
|
||||||
|
|
||||||
load_dotenv(override=True)
|
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
|
||||||
# transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
|
||||||
vad_audio_passthrough=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"))
|
|
||||||
|
|
||||||
llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile")
|
|
||||||
|
|
||||||
tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY"))
|
|
||||||
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "system",
|
|
||||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
context = OpenAILLMContext(messages)
|
|
||||||
context_aggregator = llm.create_context_aggregator(context)
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
|
||||||
[
|
|
||||||
transport.input(), # Transport user input
|
|
||||||
stt,
|
|
||||||
context_aggregator.user(), # User responses
|
|
||||||
llm, # LLM
|
|
||||||
tts, # TTS
|
|
||||||
transport.output(), # Transport bot output
|
|
||||||
context_aggregator.assistant(), # Assistant spoken responses
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
task = PipelineTask(
|
|
||||||
pipeline,
|
|
||||||
params=PipelineParams(
|
|
||||||
allow_interruptions=True,
|
|
||||||
enable_metrics=True,
|
|
||||||
enable_usage_metrics=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
# Kick off the conversation.
|
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
|
||||||
|
|
||||||
@transport.event_handler("on_participant_left")
|
|
||||||
async def on_participant_left(transport, participant, reason):
|
|
||||||
await task.cancel()
|
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
await runner.run(task)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
@@ -4,8 +4,8 @@ import os
|
|||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
from daily_runner import configure
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
|
from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -72,7 +72,8 @@ async def main():
|
|||||||
|
|
||||||
async def get_text_and_audio(messages) -> Tuple[str, bytearray]:
|
async def get_text_and_audio(messages) -> Tuple[str, bytearray]:
|
||||||
"""This function streams text from the LLM and uses the TTS service to convert
|
"""This function streams text from the LLM and uses the TTS service to convert
|
||||||
that text to speech as it's received."""
|
that text to speech as it's received.
|
||||||
|
"""
|
||||||
source_queue = asyncio.Queue()
|
source_queue = asyncio.Queue()
|
||||||
sink_queue = asyncio.Queue()
|
sink_queue = asyncio.Queue()
|
||||||
sentence_aggregator = SentenceAggregator()
|
sentence_aggregator = SentenceAggregator()
|
||||||
|
|||||||
@@ -4,13 +4,9 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
Frame,
|
Frame,
|
||||||
@@ -23,13 +19,12 @@ from pipecat.pipeline.pipeline import Pipeline
|
|||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class MirrorProcessor(FrameProcessor):
|
class MirrorProcessor(FrameProcessor):
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -44,6 +39,7 @@ class MirrorProcessor(FrameProcessor):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
elif isinstance(frame, InputImageRawFrame):
|
elif isinstance(frame, InputImageRawFrame):
|
||||||
|
print(f"Received image frame: {frame.size} {frame.format}")
|
||||||
await self.push_frame(
|
await self.push_frame(
|
||||||
OutputImageRawFrame(image=frame.image, size=frame.size, format=frame.format)
|
OutputImageRawFrame(image=frame.image, size=frame.size, format=frame.format)
|
||||||
)
|
)
|
||||||
@@ -51,17 +47,15 @@ class MirrorProcessor(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Test",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
|
camera_in_enabled=True,
|
||||||
camera_out_enabled=True,
|
camera_out_enabled=True,
|
||||||
camera_out_is_live=True,
|
camera_out_is_live=True,
|
||||||
camera_out_width=1280,
|
camera_out_width=1280,
|
||||||
@@ -69,24 +63,32 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await transport.capture_participant_video(participant["id"])
|
|
||||||
|
|
||||||
pipeline = Pipeline([transport.input(), MirrorProcessor(), transport.output()])
|
pipeline = Pipeline([transport.input(), MirrorProcessor(), transport.output()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
|
||||||
|
|
||||||
task = PipelineTask(
|
task = PipelineTask(
|
||||||
pipeline,
|
pipeline,
|
||||||
params=PipelineParams(
|
params=PipelineParams(),
|
||||||
audio_in_sample_rate=24000,
|
|
||||||
audio_out_sample_rate=24000,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -5,13 +5,10 @@
|
|||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import sys
|
|
||||||
import tkinter as tk
|
import tkinter as tk
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
Frame,
|
Frame,
|
||||||
@@ -24,14 +21,13 @@ from pipecat.pipeline.pipeline import Pipeline
|
|||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams
|
from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class MirrorProcessor(FrameProcessor):
|
class MirrorProcessor(FrameProcessor):
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -53,17 +49,25 @@ class MirrorProcessor(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
p2p_transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
camera_in_enabled=True,
|
||||||
|
camera_out_enabled=True,
|
||||||
|
camera_out_is_live=True,
|
||||||
|
camera_out_width=1280,
|
||||||
|
camera_out_height=720,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
tk_root = tk.Tk()
|
tk_root = tk.Tk()
|
||||||
tk_root.title("Local Mirror")
|
tk_root.title("Local Mirror")
|
||||||
|
|
||||||
daily_transport = DailyTransport(
|
|
||||||
room_url, token, "Test", DailyParams(audio_in_enabled=True)
|
|
||||||
)
|
|
||||||
|
|
||||||
tk_transport = TkLocalTransport(
|
tk_transport = TkLocalTransport(
|
||||||
tk_root,
|
tk_root,
|
||||||
TkTransportParams(
|
TkTransportParams(
|
||||||
@@ -75,18 +79,15 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@daily_transport.event_handler("on_first_participant_joined")
|
@p2p_transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_video(participant["id"])
|
logger.info(f"Client connected")
|
||||||
|
|
||||||
pipeline = Pipeline([daily_transport.input(), MirrorProcessor(), tk_transport.output()])
|
pipeline = Pipeline([p2p_transport.input(), MirrorProcessor(), tk_transport.output()])
|
||||||
|
|
||||||
task = PipelineTask(
|
task = PipelineTask(
|
||||||
pipeline,
|
pipeline,
|
||||||
params=PipelineParams(
|
params=PipelineParams(),
|
||||||
audio_in_sample_rate=24000,
|
|
||||||
audio_out_sample_rate=24000,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
async def run_tk():
|
async def run_tk():
|
||||||
@@ -95,10 +96,12 @@ async def main():
|
|||||||
tk_root.update_idletasks()
|
tk_root.update_idletasks()
|
||||||
await asyncio.sleep(0.1)
|
await asyncio.sleep(0.1)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await asyncio.gather(runner.run(task), run_tk())
|
await asyncio.gather(runner.run(task), run_tk())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,47 +4,44 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
|
from pipecat.frames.frames import TTSSpeakFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.filters.wake_check_filter import WakeCheckFilter
|
from pipecat.processors.filters.wake_check_filter import WakeCheckFilter
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Robot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -67,6 +64,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT
|
||||||
hey_robot_filter, # Filter out speech not directed at the robot
|
hey_robot_filter, # Filter out speech not directed at the robot
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
@@ -78,15 +76,27 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.")
|
# Kick off the conversation.
|
||||||
|
await task.queue_frame(TTSSpeakFrame("Hi! If you want to talk to me, just say 'Hey Robot'"))
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,21 +4,18 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import wave
|
import wave
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
Frame,
|
Frame,
|
||||||
LLMFullResponseEndFrame,
|
LLMFullResponseEndFrame,
|
||||||
OutputAudioRawFrame,
|
OutputAudioRawFrame,
|
||||||
|
TTSSpeakFrame,
|
||||||
)
|
)
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
@@ -30,14 +27,14 @@ from pipecat.processors.aggregators.openai_llm_context import (
|
|||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.processors.logger import FrameLogger
|
from pipecat.processors.logger import FrameLogger
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
sounds = {}
|
sounds = {}
|
||||||
sound_files = ["ding1.wav", "ding2.wav"]
|
sound_files = ["ding1.wav", "ding2.wav"]
|
||||||
@@ -80,22 +77,22 @@ class InboundSoundEffectWrapper(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
@@ -120,6 +117,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
in_sound,
|
in_sound,
|
||||||
fl2,
|
fl2,
|
||||||
@@ -132,18 +130,30 @@ async def main():
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
task = PipelineTask(pipeline)
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
@transport.event_handler("on_client_connected")
|
||||||
await tts.say("Hi, I'm listening!")
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
|
await task.queue_frame(TTSSpeakFrame("Hi, I'm listening!"))
|
||||||
await transport.send_audio(sounds["ding1.wav"])
|
await transport.send_audio(sounds["ding1.wav"])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
task = PipelineTask(pipeline)
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
||||||
@@ -23,14 +19,14 @@ from pipecat.processors.aggregators.user_response import UserResponseAggregator
|
|||||||
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.moondream.vision import MoondreamService
|
from pipecat.services.moondream.vision import MoondreamService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class UserImageRequester(FrameProcessor):
|
class UserImageRequester(FrameProcessor):
|
||||||
def __init__(self, participant_id: Optional[str] = None):
|
def __init__(self, participant_id: Optional[str] = None):
|
||||||
@@ -50,24 +46,27 @@ class UserImageRequester(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
# Get WebRTC peer connection ID
|
||||||
(room_url, token) = await configure(session)
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
|
|
||||||
transport = DailyTransport(
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
room_url,
|
|
||||||
token,
|
transport = SmallWebRTCTransport(
|
||||||
"Describe participant video",
|
webrtc_connection=webrtc_connection,
|
||||||
DailyParams(
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
user_response = UserResponseAggregator()
|
user_response = UserResponseAggregator()
|
||||||
|
|
||||||
|
# Initialize the image requester without setting the participant ID yet
|
||||||
image_requester = UserImageRequester()
|
image_requester = UserImageRequester()
|
||||||
|
|
||||||
vision_aggregator = VisionImageFrameAggregator()
|
vision_aggregator = VisionImageFrameAggregator()
|
||||||
@@ -75,21 +74,17 @@ async def main():
|
|||||||
# If you run into weird description, try with use_cpu=True
|
# If you run into weird description, try with use_cpu=True
|
||||||
moondream = MoondreamService()
|
moondream = MoondreamService()
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await tts.say("Hi there! Feel free to ask me what I see.")
|
|
||||||
await transport.capture_participant_video(participant["id"], framerate=0)
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
image_requester.set_participant_id(participant["id"])
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
user_response,
|
user_response,
|
||||||
image_requester,
|
image_requester,
|
||||||
vision_aggregator,
|
vision_aggregator,
|
||||||
@@ -101,10 +96,31 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected: {client}")
|
||||||
|
|
||||||
|
# Welcome message
|
||||||
|
await tts.say("Hi there! Feel free to ask me what I see.")
|
||||||
|
|
||||||
|
# Set the participant ID in the image requester
|
||||||
|
image_requester.set_participant_id(webrtc_peer_id)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,33 +4,29 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.user_response import UserResponseAggregator
|
from pipecat.processors.aggregators.user_response import UserResponseAggregator
|
||||||
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.google.llm import GoogleLLMService
|
from pipecat.services.google.llm import GoogleLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class UserImageRequester(FrameProcessor):
|
class UserImageRequester(FrameProcessor):
|
||||||
def __init__(self, participant_id: Optional[str] = None):
|
def __init__(self, participant_id: Optional[str] = None):
|
||||||
@@ -50,29 +46,34 @@ class UserImageRequester(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
# Get WebRTC peer connection ID
|
||||||
(room_url, token) = await configure(session)
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
|
|
||||||
transport = DailyTransport(
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
room_url,
|
|
||||||
token,
|
transport = SmallWebRTCTransport(
|
||||||
"Describe participant video",
|
webrtc_connection=webrtc_connection,
|
||||||
DailyParams(
|
params=TransportParams(
|
||||||
audio_in_enabled=True, # This is so Silero VAD can get audio data
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
user_response = UserResponseAggregator()
|
user_response = UserResponseAggregator()
|
||||||
|
|
||||||
|
# Initialize the image requester without setting the participant ID yet
|
||||||
image_requester = UserImageRequester()
|
image_requester = UserImageRequester()
|
||||||
|
|
||||||
vision_aggregator = VisionImageFrameAggregator()
|
vision_aggregator = VisionImageFrameAggregator()
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
|
# Google Gemini model for vision analysis
|
||||||
google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY"))
|
google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
@@ -80,16 +81,10 @@ async def main():
|
|||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await tts.say("Hi there! Feel free to ask me what I see.")
|
|
||||||
await transport.capture_participant_video(participant["id"], framerate=0)
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
image_requester.set_participant_id(participant["id"])
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
user_response,
|
user_response,
|
||||||
image_requester,
|
image_requester,
|
||||||
vision_aggregator,
|
vision_aggregator,
|
||||||
@@ -99,12 +94,36 @@ async def main():
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(allow_interruptions=True),
|
||||||
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected: {client}")
|
||||||
|
|
||||||
|
# Welcome message
|
||||||
|
await tts.say("Hi there! Feel free to ask me what I see.")
|
||||||
|
|
||||||
|
# Set the participant ID in the image requester
|
||||||
|
image_requester.set_participant_id(webrtc_peer_id)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,33 +4,29 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.user_response import UserResponseAggregator
|
from pipecat.processors.aggregators.user_response import UserResponseAggregator
|
||||||
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class UserImageRequester(FrameProcessor):
|
class UserImageRequester(FrameProcessor):
|
||||||
def __init__(self, participant_id: Optional[str] = None):
|
def __init__(self, participant_id: Optional[str] = None):
|
||||||
@@ -50,28 +46,34 @@ class UserImageRequester(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
# Get WebRTC peer connection ID
|
||||||
(room_url, token) = await configure(session)
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
|
|
||||||
transport = DailyTransport(
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
room_url,
|
|
||||||
token,
|
transport = SmallWebRTCTransport(
|
||||||
"Describe participant video",
|
webrtc_connection=webrtc_connection,
|
||||||
DailyParams(
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
user_response = UserResponseAggregator()
|
user_response = UserResponseAggregator()
|
||||||
|
|
||||||
|
# Initialize the image requester without setting the participant ID yet
|
||||||
image_requester = UserImageRequester()
|
image_requester = UserImageRequester()
|
||||||
|
|
||||||
vision_aggregator = VisionImageFrameAggregator()
|
vision_aggregator = VisionImageFrameAggregator()
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
|
# OpenAI GPT-4o for vision analysis
|
||||||
openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
@@ -79,16 +81,10 @@ async def main():
|
|||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await tts.say("Hi there! Feel free to ask me what I see.")
|
|
||||||
await transport.capture_participant_video(participant["id"], framerate=0)
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
image_requester.set_participant_id(participant["id"])
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
user_response,
|
user_response,
|
||||||
image_requester,
|
image_requester,
|
||||||
vision_aggregator,
|
vision_aggregator,
|
||||||
@@ -98,12 +94,36 @@ async def main():
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(allow_interruptions=True),
|
||||||
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected: {client}")
|
||||||
|
|
||||||
|
# Welcome message
|
||||||
|
await tts.say("Hi there! Feel free to ask me what I see.")
|
||||||
|
|
||||||
|
# Set the participant ID in the image requester
|
||||||
|
image_requester.set_participant_id(webrtc_peer_id)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,33 +4,29 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.user_response import UserResponseAggregator
|
from pipecat.processors.aggregators.user_response import UserResponseAggregator
|
||||||
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.anthropic.llm import AnthropicLLMService
|
from pipecat.services.anthropic.llm import AnthropicLLMService
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class UserImageRequester(FrameProcessor):
|
class UserImageRequester(FrameProcessor):
|
||||||
def __init__(self, participant_id: Optional[str] = None):
|
def __init__(self, participant_id: Optional[str] = None):
|
||||||
@@ -50,28 +46,34 @@ class UserImageRequester(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
# Get WebRTC peer connection ID
|
||||||
(room_url, token) = await configure(session)
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
|
|
||||||
transport = DailyTransport(
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
room_url,
|
|
||||||
token,
|
transport = SmallWebRTCTransport(
|
||||||
"Describe participant video",
|
webrtc_connection=webrtc_connection,
|
||||||
DailyParams(
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
user_response = UserResponseAggregator()
|
user_response = UserResponseAggregator()
|
||||||
|
|
||||||
|
# Initialize the image requester without setting the participant ID yet
|
||||||
image_requester = UserImageRequester()
|
image_requester = UserImageRequester()
|
||||||
|
|
||||||
vision_aggregator = VisionImageFrameAggregator()
|
vision_aggregator = VisionImageFrameAggregator()
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
|
# Anthropic for vision analysis
|
||||||
anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY"))
|
anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
@@ -79,16 +81,10 @@ async def main():
|
|||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
|
||||||
async def on_first_participant_joined(transport, participant):
|
|
||||||
await tts.say("Hi there! Feel free to ask me what I see.")
|
|
||||||
await transport.capture_participant_video(participant["id"], framerate=0)
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
image_requester.set_participant_id(participant["id"])
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
user_response,
|
user_response,
|
||||||
image_requester,
|
image_requester,
|
||||||
vision_aggregator,
|
vision_aggregator,
|
||||||
@@ -98,12 +94,36 @@ async def main():
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(allow_interruptions=True),
|
||||||
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected: {client}")
|
||||||
|
|
||||||
|
# Welcome message
|
||||||
|
await tts.say("Hi there! Feel free to ask me what I see.")
|
||||||
|
|
||||||
|
# Set the participant ID in the image requester
|
||||||
|
image_requester.set_participant_id(webrtc_peer_id)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,13 +4,9 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import Frame, TranscriptionFrame
|
from pipecat.frames.frames import Frame, TranscriptionFrame
|
||||||
@@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.whisper.stt import WhisperSTTService
|
from pipecat.services.whisper.stt import WhisperSTTService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class TranscriptionLogger(FrameProcessor):
|
class TranscriptionLogger(FrameProcessor):
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -35,15 +30,12 @@ class TranscriptionLogger(FrameProcessor):
|
|||||||
print(f"Transcription: {frame.text}")
|
print(f"Transcription: {frame.text}")
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
None,
|
params=TransportParams(
|
||||||
"Transcription bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -59,10 +51,21 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
import os
|
||||||
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import Frame, TranscriptionFrame
|
from pipecat.frames.frames import Frame, TranscriptionFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -19,13 +16,12 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions
|
from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class TranscriptionLogger(FrameProcessor):
|
class TranscriptionLogger(FrameProcessor):
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -35,17 +31,17 @@ class TranscriptionLogger(FrameProcessor):
|
|||||||
print(f"Transcription: {frame.text}")
|
print(f"Transcription: {frame.text}")
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(audio_in_enabled=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
stt = DeepgramSTTService(
|
stt = DeepgramSTTService(
|
||||||
api_key=os.getenv("DEEPGRAM_API_KEY"),
|
api_key=os.getenv("DEEPGRAM_API_KEY"),
|
||||||
# live_options=LiveOptions(language=Language.FR),
|
live_options=LiveOptions(language=Language.EN),
|
||||||
)
|
)
|
||||||
|
|
||||||
tl = TranscriptionLogger()
|
tl = TranscriptionLogger()
|
||||||
@@ -54,10 +50,21 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import Frame, TranscriptionFrame
|
from pipecat.frames.frames import Frame, TranscriptionFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.gladia import GladiaSTTService
|
from pipecat.services.gladia import GladiaSTTService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class TranscriptionLogger(FrameProcessor):
|
class TranscriptionLogger(FrameProcessor):
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -35,12 +30,12 @@ class TranscriptionLogger(FrameProcessor):
|
|||||||
print(f"Transcription: {frame.text}")
|
print(f"Transcription: {frame.text}")
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(audio_in_enabled=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
stt = GladiaSTTService(
|
stt = GladiaSTTService(
|
||||||
@@ -54,10 +49,21 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.frames.frames import Frame, TranscriptionFrame
|
from pipecat.frames.frames import Frame, TranscriptionFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -19,13 +15,12 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.assemblyai.stt import AssemblyAISTTService
|
from pipecat.services.assemblyai.stt import AssemblyAISTTService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
class TranscriptionLogger(FrameProcessor):
|
class TranscriptionLogger(FrameProcessor):
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -35,12 +30,12 @@ class TranscriptionLogger(FrameProcessor):
|
|||||||
print(f"Transcription: {frame.text}")
|
print(f"Transcription: {frame.text}")
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url, None, "Transcription bot", DailyParams(audio_in_enabled=True)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(audio_in_enabled=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
stt = AssemblyAISTTService(
|
stt = AssemblyAISTTService(
|
||||||
@@ -53,10 +48,21 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -21,13 +18,12 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.services.whisper.stt import MLXModel, WhisperSTTServiceMLX
|
from pipecat.services.whisper.stt import MLXModel, WhisperSTTServiceMLX
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
STOP_SECS = 2.0
|
STOP_SECS = 2.0
|
||||||
|
|
||||||
@@ -56,15 +52,12 @@ class TranscriptionLogger(FrameProcessor):
|
|||||||
self._last_transcription_time = time.time()
|
self._last_transcription_time = time.time()
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
None,
|
params=TransportParams(
|
||||||
"Transcription bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)),
|
||||||
@@ -86,10 +79,21 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -94,6 +90,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -112,16 +109,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,35 +18,35 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.anthropic.llm import AnthropicLLMService
|
from pipecat.services.anthropic.llm import AnthropicLLMService
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
location = arguments["location"]
|
location = arguments["location"]
|
||||||
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
|
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -89,6 +85,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User spoken responses
|
context_aggregator.user(), # User spoken responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -105,16 +102,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -6,12 +6,9 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,14 +19,16 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.anthropic.llm import AnthropicLLMService
|
from pipecat.services.anthropic.llm import AnthropicLLMService
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
video_participant_id = None
|
# Global variable to store the peer connection ID
|
||||||
|
webrtc_peer_id = None
|
||||||
|
|
||||||
|
|
||||||
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
@@ -39,32 +38,45 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
|
|||||||
|
|
||||||
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
question = arguments["question"]
|
question = arguments["question"]
|
||||||
|
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
|
||||||
|
|
||||||
|
# Request the image frame
|
||||||
await llm.request_image_frame(
|
await llm.request_image_frame(
|
||||||
user_id=video_participant_id,
|
user_id=webrtc_peer_id,
|
||||||
function_name=function_name,
|
function_name=function_name,
|
||||||
tool_call_id=tool_call_id,
|
tool_call_id=tool_call_id,
|
||||||
text_content=question,
|
text_content=question,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Wait a short time for the frame to be processed
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
|
||||||
async def main():
|
# Return a result to complete the function call
|
||||||
global llm
|
await result_callback(
|
||||||
|
f"I've captured an image from your camera and I'm analyzing what you asked about: {question}"
|
||||||
|
)
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
room_url,
|
global webrtc_peer_id
|
||||||
token,
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
|
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True, # Make sure camera input is enabled
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -102,8 +114,6 @@ async def main():
|
|||||||
)
|
)
|
||||||
tools = ToolsSchema(standard_tools=[weather_function, get_image_function])
|
tools = ToolsSchema(standard_tools=[weather_function, get_image_function])
|
||||||
|
|
||||||
# todo: test with very short initial user message
|
|
||||||
|
|
||||||
system_prompt = """\
|
system_prompt = """\
|
||||||
You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions.
|
You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions.
|
||||||
|
|
||||||
@@ -115,12 +125,12 @@ You can respond to questions about the weather using the get_weather tool.
|
|||||||
|
|
||||||
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
|
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
|
||||||
indicate you should use the get_image tool are:
|
indicate you should use the get_image tool are:
|
||||||
- What do you see?
|
- What do you see?
|
||||||
- What's in the video?
|
- What's in the video?
|
||||||
- Can you describe the video?
|
- Can you describe the video?
|
||||||
- Tell me about what you see.
|
- Tell me about what you see.
|
||||||
- Tell me something interesting about what you see.
|
- Tell me something interesting about what you see.
|
||||||
- What's happening in the video?
|
- What's happening in the video?
|
||||||
|
|
||||||
If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise.
|
If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise.
|
||||||
"""
|
"""
|
||||||
@@ -144,6 +154,7 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT
|
||||||
context_aggregator.user(), # User speech to text
|
context_aggregator.user(), # User speech to text
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -160,18 +171,27 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
global video_participant_id
|
logger.info(f"Client connected: {client}")
|
||||||
video_participant_id = participant["id"]
|
|
||||||
await transport.capture_participant_transcription(video_participant_id)
|
|
||||||
await transport.capture_participant_video(video_participant_id, framerate=0)
|
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.together.llm import TogetherLLMService
|
from pipecat.services.together.llm import TogetherLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -95,6 +91,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -105,16 +102,27 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -6,12 +6,9 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -21,15 +18,17 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
video_participant_id = None
|
# Global variable to store the peer connection ID
|
||||||
|
webrtc_peer_id = None
|
||||||
|
|
||||||
|
|
||||||
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
@@ -38,32 +37,46 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
|
|||||||
|
|
||||||
|
|
||||||
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}")
|
|
||||||
question = arguments["question"]
|
question = arguments["question"]
|
||||||
|
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
|
||||||
|
|
||||||
|
# Request the image frame
|
||||||
await llm.request_image_frame(
|
await llm.request_image_frame(
|
||||||
user_id=video_participant_id,
|
user_id=webrtc_peer_id,
|
||||||
function_name=function_name,
|
function_name=function_name,
|
||||||
tool_call_id=tool_call_id,
|
tool_call_id=tool_call_id,
|
||||||
text_content=question,
|
text_content=question,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Wait a short time for the frame to be processed
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
|
||||||
async def main():
|
# Return a result to complete the function call
|
||||||
async with aiohttp.ClientSession() as session:
|
await result_callback(
|
||||||
(room_url, token) = await configure(session)
|
f"I've captured an image from your camera and I'm analyzing what you asked about: {question}"
|
||||||
|
)
|
||||||
|
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
token,
|
global webrtc_peer_id
|
||||||
"Respond bot",
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
DailyParams(
|
|
||||||
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
|
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True, # Make sure camera input is enabled
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -113,12 +126,12 @@ You can respond to questions about the weather using the get_weather tool.
|
|||||||
|
|
||||||
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
|
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
|
||||||
indicate you should use the get_image tool are:
|
indicate you should use the get_image tool are:
|
||||||
- What do you see?
|
- What do you see?
|
||||||
- What's in the video?
|
- What's in the video?
|
||||||
- Can you describe the video?
|
- Can you describe the video?
|
||||||
- Tell me about what you see.
|
- Tell me about what you see.
|
||||||
- Tell me something interesting about what you see.
|
- Tell me something interesting about what you see.
|
||||||
- What's happening in the video?
|
- What's happening in the video?
|
||||||
"""
|
"""
|
||||||
messages = [
|
messages = [
|
||||||
{"role": "system", "content": system_prompt},
|
{"role": "system", "content": system_prompt},
|
||||||
@@ -130,6 +143,7 @@ indicate you should use the get_image tool are:
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -140,19 +154,27 @@ indicate you should use the get_image tool are:
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
global video_participant_id
|
logger.info(f"Client connected")
|
||||||
video_participant_id = participant["id"]
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
await transport.capture_participant_video(video_participant_id, framerate=0)
|
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -6,12 +6,9 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,15 +19,17 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.google.llm import GoogleLLMService
|
from pipecat.services.google.llm import GoogleLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
video_participant_id = None
|
# Global variable to store the peer connection ID
|
||||||
|
webrtc_peer_id = None
|
||||||
|
|
||||||
|
|
||||||
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
@@ -40,32 +39,46 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu
|
|||||||
|
|
||||||
|
|
||||||
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}")
|
|
||||||
question = arguments["question"]
|
question = arguments["question"]
|
||||||
|
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
|
||||||
|
|
||||||
|
# Request the image frame
|
||||||
await llm.request_image_frame(
|
await llm.request_image_frame(
|
||||||
user_id=video_participant_id,
|
user_id=webrtc_peer_id,
|
||||||
function_name=function_name,
|
function_name=function_name,
|
||||||
tool_call_id=tool_call_id,
|
tool_call_id=tool_call_id,
|
||||||
text_content=question,
|
text_content=question,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Wait a short time for the frame to be processed
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
|
||||||
async def main():
|
# Return a result to complete the function call
|
||||||
async with aiohttp.ClientSession() as session:
|
await result_callback(
|
||||||
(room_url, token) = await configure(session)
|
f"I've captured an image from your camera and I'm analyzing what you asked about: {question}"
|
||||||
|
)
|
||||||
|
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
token,
|
global webrtc_peer_id
|
||||||
"Respond bot",
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
DailyParams(
|
|
||||||
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
|
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True, # Make sure camera input is enabled
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -115,12 +128,12 @@ You can respond to questions about the weather using the get_weather tool.
|
|||||||
|
|
||||||
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
|
You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \
|
||||||
indicate you should use the get_image tool are:
|
indicate you should use the get_image tool are:
|
||||||
- What do you see?
|
- What do you see?
|
||||||
- What's in the video?
|
- What's in the video?
|
||||||
- Can you describe the video?
|
- Can you describe the video?
|
||||||
- Tell me about what you see.
|
- Tell me about what you see.
|
||||||
- Tell me something interesting about what you see.
|
- Tell me something interesting about what you see.
|
||||||
- What's happening in the video?
|
- What's happening in the video?
|
||||||
"""
|
"""
|
||||||
messages = [
|
messages = [
|
||||||
{"role": "system", "content": system_prompt},
|
{"role": "system", "content": system_prompt},
|
||||||
@@ -133,6 +146,7 @@ indicate you should use the get_image tool are:
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -150,19 +164,27 @@ indicate you should use the get_image tool are:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
global video_participant_id
|
logger.info(f"Client connected: {client}")
|
||||||
video_participant_id = participant["id"]
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
await transport.capture_participant_video(video_participant_id, framerate=0)
|
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -24,28 +20,25 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.groq.llm import GroqLLMService
|
from pipecat.services.groq.llm import GroqLLMService
|
||||||
from pipecat.services.groq.stt import GroqSTTService
|
from pipecat.services.groq.stt import GroqSTTService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -113,16 +106,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -21,35 +17,35 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.grok.llm import GrokLLMService
|
from pipecat.services.grok.llm import GrokLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -90,6 +86,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -107,16 +104,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -23,35 +19,35 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.azure.llm import AzureLLMService
|
from pipecat.services.azure.llm import AzureLLMService
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -96,6 +92,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -113,16 +110,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.fireworks.llm import FireworksLLMService
|
from pipecat.services.fireworks.llm import FireworksLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -95,6 +91,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -112,16 +109,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,45 +18,43 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.nim.llm import NimLLMService
|
from pipecat.services.nim.llm import NimLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
# text_filters=[MarkdownTextFilter()],
|
# text_filters=[MarkdownTextFilter()],
|
||||||
)
|
)
|
||||||
|
|
||||||
llm = NimLLMService(
|
llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct")
|
||||||
api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct"
|
|
||||||
)
|
|
||||||
# You can also register a function_name of None to get all functions
|
# You can also register a function_name of None to get all functions
|
||||||
# sent to the same callback with an additional function_name parameter.
|
# sent to the same callback with an additional function_name parameter.
|
||||||
llm.register_function("get_current_weather", fetch_weather_from_api)
|
llm.register_function("get_current_weather", fetch_weather_from_api)
|
||||||
@@ -95,6 +89,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -112,16 +107,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -23,35 +19,35 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.cerebras.llm import CerebrasLLMService
|
from pipecat.services.cerebras.llm import CerebrasLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -102,6 +98,7 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -120,16 +117,27 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.deepseek.llm import DeepSeekLLMService
|
from pipecat.services.deepseek.llm import DeepSeekLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -102,6 +98,7 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -120,16 +117,27 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.azure.tts import AzureTTSService
|
from pipecat.services.azure.tts import AzureTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openrouter.llm import OpenRouterLLMService
|
from pipecat.services.openrouter.llm import OpenRouterLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = AzureTTSService(
|
tts = AzureTTSService(
|
||||||
api_key=os.getenv("AZURE_API_KEY"),
|
api_key=os.getenv("AZURE_API_KEY"),
|
||||||
region="eastus",
|
region="eastus",
|
||||||
@@ -96,6 +92,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -114,16 +111,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -11,14 +11,10 @@ currently support function calling. The example shows basic chat completion func
|
|||||||
using Perplexity's API while maintaining compatibility with the OpenAI interface.
|
using Perplexity's API while maintaining compatibility with the OpenAI interface.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
@@ -26,31 +22,31 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.perplexity.llm import PerplexityLLMService
|
from pipecat.services.perplexity.llm import PerplexityLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -71,6 +67,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -89,16 +86,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -21,37 +17,37 @@ from pipecat.pipeline.pipeline import Pipeline
|
|||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
||||||
from pipecat.services.google.llm_openai import GoogleLLMOpenAIBetaService
|
from pipecat.services.google.llm_openai import GoogleLLMOpenAIBetaService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
||||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
||||||
@@ -92,6 +88,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -109,16 +106,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -21,37 +17,37 @@ from pipecat.pipeline.pipeline import Pipeline
|
|||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
||||||
from pipecat.services.google.llm_vertex import GoogleVertexLLMService
|
from pipecat.services.google.llm_vertex import GoogleVertexLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
api_key=os.getenv("ELEVENLABS_API_KEY", ""),
|
||||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
|
||||||
@@ -98,6 +94,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -115,16 +112,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -22,36 +18,36 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.qwen.llm import QwenLLMService
|
from pipecat.services.qwen.llm import QwenLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -94,6 +90,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
stt,
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm,
|
llm,
|
||||||
tts,
|
tts,
|
||||||
@@ -112,16 +109,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from openai.types.chat import ChatCompletionToolParam
|
from openai.types.chat import ChatCompletionToolParam
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
|
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
|
||||||
@@ -22,13 +18,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.filters.function_filter import FunctionFilter
|
from pipecat.processors.filters.function_filter import FunctionFilter
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
current_voice = "News Lady"
|
current_voice = "News Lady"
|
||||||
|
|
||||||
@@ -55,22 +52,22 @@ async def barbershop_man_filter(frame) -> bool:
|
|||||||
return current_voice == "Barbershop Man"
|
return current_voice == "Barbershop Man"
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Pipecat",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
news_lady = CartesiaTTSService(
|
news_lady = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady
|
voice_id="bf991597-6c13-47e4-8411-91ec2de5c466", # Newslady
|
||||||
@@ -121,6 +118,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
ParallelPipeline( # TTS (one of the following vocies)
|
ParallelPipeline( # TTS (one of the following vocies)
|
||||||
@@ -138,9 +136,9 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
@@ -150,10 +148,21 @@ async def main():
|
|||||||
)
|
)
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,16 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from deepgram import LiveOptions
|
from deepgram import LiveOptions
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from openai.types.chat import ChatCompletionToolParam
|
from openai.types.chat import ChatCompletionToolParam
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
|
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
|
||||||
@@ -25,12 +21,12 @@ from pipecat.processors.filters.function_filter import FunctionFilter
|
|||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
current_language = "English"
|
current_language = "English"
|
||||||
|
|
||||||
@@ -49,15 +45,13 @@ async def spanish_filter(frame) -> bool:
|
|||||||
return current_language == "Spanish"
|
return current_language == "Spanish"
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Pipecat",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -76,7 +70,7 @@ async def main():
|
|||||||
|
|
||||||
spanish_tts = CartesiaTTSService(
|
spanish_tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="846d6cb0-2301-48b6-9683-48f5618ea2f6", # Spanish-speaking Lady
|
voice_id="d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", # Spanish-speaking Lady
|
||||||
)
|
)
|
||||||
|
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
@@ -128,9 +122,9 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
@@ -140,10 +134,21 @@ async def main():
|
|||||||
)
|
)
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,50 +4,46 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.deepgram.tts import DeepgramTTSService
|
from pipecat.services.deepgram.tts import DeepgramTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import (
|
from pipecat.transports.base_transport import TransportParams
|
||||||
DailyParams,
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
DailyTransport,
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
DailyTransportMessageFrame,
|
from pipecat.transports.services.daily import DailyTransportMessageFrame
|
||||||
)
|
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create an HTTP session
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = DeepgramTTSService(
|
tts = DeepgramTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
api_key=os.getenv("DEEPGRAM_API_KEY"),
|
api_key=os.getenv("DEEPGRAM_API_KEY"),
|
||||||
@@ -77,6 +73,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -93,15 +90,11 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# When a participant joins, start transcription for that participant so the
|
|
||||||
# bot can "hear" and respond to them.
|
|
||||||
@transport.event_handler("on_participant_joined")
|
|
||||||
async def on_participant_joined(transport, participant):
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
|
|
||||||
# When the first participant joins, the bot should introduce itself.
|
# When the first participant joins, the bot should introduce itself.
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@@ -134,9 +127,21 @@ async def main():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"message handling error: {e} - {message}")
|
logger.debug(f"message handling error: {e} - {message}")
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import EndFrame, LLMMessagesFrame, TTSSpeakFrame
|
from pipecat.frames.frames import EndFrame, LLMMessagesFrame, TTSSpeakFrame
|
||||||
@@ -21,31 +17,31 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.user_idle_processor import UserIdleProcessor
|
from pipecat.processors.user_idle_processor import UserIdleProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, token) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -97,6 +93,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt,
|
||||||
user_idle, # Idle user check-in
|
user_idle, # Idle user check-in
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
@@ -115,17 +112,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -5,43 +5,45 @@
|
|||||||
#
|
#
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure_with_args
|
|
||||||
|
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
|
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
# Parse command line arguments
|
||||||
async def main():
|
# This will be used to pass the input video file to the bot
|
||||||
async with aiohttp.ClientSession() as session:
|
# You can run the bot with a command like:
|
||||||
|
# python 18-gstreamer-filesrc.py -i path/to/video.mp4
|
||||||
|
def parse_arguments():
|
||||||
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
|
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
|
||||||
parser.add_argument("-i", "--input", type=str, required=True, help="Input video file")
|
parser.add_argument("-i", "--input", type=str, required=True, help="Input video file")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
(room_url, _, args) = await configure_with_args(session, parser)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
args = parse_arguments()
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"GStreamer",
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
DailyParams(
|
logger.info(f"Starting bot with video input: {args.input}")
|
||||||
audio_out_enabled=True,
|
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
camera_out_enabled=True,
|
camera_out_enabled=True,
|
||||||
|
camera_out_is_live=True,
|
||||||
camera_out_width=1280,
|
camera_out_width=1280,
|
||||||
camera_out_height=720,
|
camera_out_height=720,
|
||||||
camera_out_is_live=True,
|
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -62,10 +64,12 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,39 +4,32 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineTask
|
from pipecat.pipeline.task import PipelineTask
|
||||||
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
|
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot with video test source")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, _) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"GStreamer",
|
|
||||||
DailyParams(
|
|
||||||
camera_out_enabled=True,
|
camera_out_enabled=True,
|
||||||
|
camera_out_is_live=True,
|
||||||
camera_out_width=1280,
|
camera_out_width=1280,
|
||||||
camera_out_height=720,
|
camera_out_height=720,
|
||||||
camera_out_is_live=True,
|
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -49,17 +42,19 @@ async def main():
|
|||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
gst, # GStreamer file source
|
gst, # GStreamer test source
|
||||||
transport.output(), # Transport bot output
|
transport.output(), # Transport bot output
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
task = PipelineTask(pipeline)
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -29,13 +25,12 @@ from pipecat.services.openai_realtime_beta import (
|
|||||||
SemanticTurnDetection,
|
SemanticTurnDetection,
|
||||||
SessionProperties,
|
SessionProperties,
|
||||||
)
|
)
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
temperature = 75 if args["format"] == "fahrenheit" else 24
|
temperature = 75 if args["format"] == "fahrenheit" else 24
|
||||||
@@ -70,18 +65,14 @@ weather_function = FunctionSchema(
|
|||||||
tools = ToolsSchema(standard_tools=[weather_function])
|
tools = ToolsSchema(standard_tools=[weather_function])
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=False,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
@@ -159,20 +150,31 @@ Remember, your responses should be short. Just one or two sentences, usually."""
|
|||||||
allow_interruptions=True,
|
allow_interruptions=True,
|
||||||
enable_metrics=True,
|
enable_metrics=True,
|
||||||
enable_usage_metrics=True,
|
enable_usage_metrics=True,
|
||||||
# report_only_initial_ttfb=True,
|
report_only_initial_ttfb=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
@@ -27,13 +23,12 @@ from pipecat.services.openai_realtime_beta import (
|
|||||||
InputAudioTranscription,
|
InputAudioTranscription,
|
||||||
SessionProperties,
|
SessionProperties,
|
||||||
)
|
)
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
temperature = 75 if args["format"] == "fahrenheit" else 24
|
temperature = 75 if args["format"] == "fahrenheit" else 24
|
||||||
@@ -69,18 +64,14 @@ weather_function = FunctionSchema(
|
|||||||
tools = ToolsSchema(standard_tools=[weather_function])
|
tools = ToolsSchema(standard_tools=[weather_function])
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=False,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
@@ -88,7 +79,7 @@ async def main():
|
|||||||
)
|
)
|
||||||
|
|
||||||
session_properties = SessionProperties(
|
session_properties = SessionProperties(
|
||||||
input_audio_transcription=InputAudioTranscription(),
|
input_audio_transcription=InputAudioTranscription(model="whisper-1"),
|
||||||
# Set openai TurnDetection parameters. Not setting this at all will turn it
|
# Set openai TurnDetection parameters. Not setting this at all will turn it
|
||||||
# on by default
|
# on by default
|
||||||
# turn_detection=TurnDetection(silence_duration_ms=1000),
|
# turn_detection=TurnDetection(silence_duration_ms=1000),
|
||||||
@@ -158,20 +149,31 @@ Remember, your responses should be short. Just one or two sentences, usually."""
|
|||||||
allow_interruptions=True,
|
allow_interruptions=True,
|
||||||
enable_metrics=True,
|
enable_metrics=True,
|
||||||
enable_usage_metrics=True,
|
enable_usage_metrics=True,
|
||||||
# report_only_initial_ttfb=True,
|
report_only_initial_ttfb=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,17 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import glob
|
import glob
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -25,13 +21,14 @@ from pipecat.processors.aggregators.openai_llm_context import (
|
|||||||
OpenAILLMContext,
|
OpenAILLMContext,
|
||||||
)
|
)
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
||||||
tts = None
|
tts = None
|
||||||
@@ -165,23 +162,24 @@ tools = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
global tts
|
logger.info(f"Starting bot")
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
global tts
|
||||||
room_url,
|
|
||||||
token,
|
transport = SmallWebRTCTransport(
|
||||||
"Respond bot",
|
webrtc_connection=webrtc_connection,
|
||||||
DailyParams(
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -202,6 +200,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts,
|
tts,
|
||||||
@@ -216,20 +215,31 @@ async def main():
|
|||||||
allow_interruptions=True,
|
allow_interruptions=True,
|
||||||
enable_metrics=True,
|
enable_metrics=True,
|
||||||
enable_usage_metrics=True,
|
enable_usage_metrics=True,
|
||||||
# report_only_initial_ttfb=True,
|
report_only_initial_ttfb=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -8,13 +8,10 @@ import asyncio
|
|||||||
import glob
|
import glob
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -24,19 +21,19 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|||||||
from pipecat.processors.aggregators.openai_llm_context import (
|
from pipecat.processors.aggregators.openai_llm_context import (
|
||||||
OpenAILLMContext,
|
OpenAILLMContext,
|
||||||
)
|
)
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai_realtime_beta import (
|
from pipecat.services.openai_realtime_beta import (
|
||||||
InputAudioTranscription,
|
InputAudioTranscription,
|
||||||
OpenAIRealtimeBetaLLMService,
|
OpenAIRealtimeBetaLLMService,
|
||||||
SessionProperties,
|
SessionProperties,
|
||||||
TurnDetection,
|
TurnDetection,
|
||||||
)
|
)
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
||||||
|
|
||||||
|
|
||||||
@@ -167,24 +164,22 @@ tools = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=False,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
session_properties = SessionProperties(
|
session_properties = SessionProperties(
|
||||||
input_audio_transcription=InputAudioTranscription(),
|
input_audio_transcription=InputAudioTranscription(),
|
||||||
# Set openai TurnDetection parameters. Not setting this at all will turn it
|
# Set openai TurnDetection parameters. Not setting this at all will turn it
|
||||||
@@ -228,6 +223,7 @@ Remember, your responses should be short. Just one or two sentences, usually."""
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
transport.output(), # Transport bot output
|
transport.output(), # Transport bot output
|
||||||
@@ -241,20 +237,31 @@ Remember, your responses should be short. Just one or two sentences, usually."""
|
|||||||
allow_interruptions=True,
|
allow_interruptions=True,
|
||||||
enable_metrics=True,
|
enable_metrics=True,
|
||||||
enable_usage_metrics=True,
|
enable_usage_metrics=True,
|
||||||
# report_only_initial_ttfb=True,
|
report_only_initial_ttfb=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,17 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import glob
|
import glob
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -26,12 +22,13 @@ from pipecat.processors.aggregators.openai_llm_context import (
|
|||||||
)
|
)
|
||||||
from pipecat.services.anthropic.llm import AnthropicLLMService
|
from pipecat.services.anthropic.llm import AnthropicLLMService
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
||||||
tts = None
|
tts = None
|
||||||
@@ -160,23 +157,24 @@ tools = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
global tts
|
logger.info(f"Starting bot")
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
global tts
|
||||||
room_url,
|
|
||||||
token,
|
transport = SmallWebRTCTransport(
|
||||||
"Respond bot",
|
webrtc_connection=webrtc_connection,
|
||||||
DailyParams(
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -199,6 +197,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts,
|
tts,
|
||||||
@@ -217,16 +216,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,17 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import glob
|
import glob
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -25,19 +21,21 @@ from pipecat.processors.aggregators.openai_llm_context import (
|
|||||||
OpenAILLMContext,
|
OpenAILLMContext,
|
||||||
)
|
)
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.google.llm import GoogleLLMService
|
from pipecat.services.google.llm import GoogleLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
video_participant_id = None
|
video_participant_id = None
|
||||||
|
|
||||||
|
|
||||||
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
BASE_FILENAME = "/tmp/pipecat_conversation_"
|
||||||
tts = None
|
tts = None
|
||||||
|
webrtc_peer_id = None
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
@@ -54,8 +52,11 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
|
|||||||
|
|
||||||
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||||
question = arguments["question"]
|
question = arguments["question"]
|
||||||
|
logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}")
|
||||||
|
|
||||||
|
# Request the image frame
|
||||||
await llm.request_image_frame(
|
await llm.request_image_frame(
|
||||||
user_id=video_participant_id,
|
user_id=webrtc_peer_id,
|
||||||
function_name=function_name,
|
function_name=function_name,
|
||||||
tool_call_id=tool_call_id,
|
tool_call_id=tool_call_id,
|
||||||
text_content=question,
|
text_content=question,
|
||||||
@@ -220,23 +221,26 @@ tools = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
global tts
|
global tts, webrtc_peer_id
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_peer_id = webrtc_connection.pc_id
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
logger.info(f"Starting bot with peer_id: {webrtc_peer_id}")
|
||||||
room_url,
|
|
||||||
token,
|
transport = SmallWebRTCTransport(
|
||||||
"Respond bot",
|
webrtc_connection=webrtc_connection,
|
||||||
DailyParams(
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
transcription_enabled=True,
|
camera_in_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
tts = CartesiaTTSService(
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
@@ -258,6 +262,7 @@ async def main():
|
|||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts,
|
tts,
|
||||||
@@ -276,19 +281,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
global video_participant_id
|
logger.info(f"Client connected")
|
||||||
video_participant_id = participant["id"]
|
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
|
||||||
await transport.capture_participant_video(video_participant_id, framerate=0)
|
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import TextFrame
|
from pipecat.frames.frames import TextFrame
|
||||||
@@ -28,23 +24,20 @@ from pipecat.services.cartesia.tts import CartesiaTTSService
|
|||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.sync.event_notifier import EventNotifier
|
from pipecat.sync.event_notifier import EventNotifier
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
transport = SmallWebRTCTransport(
|
||||||
async with aiohttp.ClientSession() as session:
|
webrtc_connection=webrtc_connection,
|
||||||
(room_url, _) = await configure(session)
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
transport = DailyTransport(
|
|
||||||
room_url,
|
|
||||||
None,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -99,16 +92,12 @@ async def main():
|
|||||||
|
|
||||||
# This a filter that will wake up the notifier if the given predicate
|
# This a filter that will wake up the notifier if the given predicate
|
||||||
# (wake_check_filter) returns true.
|
# (wake_check_filter) returns true.
|
||||||
completness_check = WakeNotifierFilter(
|
completness_check = WakeNotifierFilter(notifier, types=(TextFrame,), filter=wake_check_filter)
|
||||||
notifier, types=(TextFrame,), filter=wake_check_filter
|
|
||||||
)
|
|
||||||
|
|
||||||
# This processor keeps the last context and will let it through once the
|
# This processor keeps the last context and will let it through once the
|
||||||
# notifier is woken up. We start with the gate open because we send an
|
# notifier is woken up. We start with the gate open because we send an
|
||||||
# initial context frame to start the conversation.
|
# initial context frame to start the conversation.
|
||||||
gated_context_aggregator = GatedOpenAILLMContextAggregator(
|
gated_context_aggregator = GatedOpenAILLMContextAggregator(notifier=notifier, start_open=True)
|
||||||
notifier=notifier, start_open=True
|
|
||||||
)
|
|
||||||
|
|
||||||
# Notify if the user hasn't said anything.
|
# Notify if the user hasn't said anything.
|
||||||
async def user_idle_notifier(frame):
|
async def user_idle_notifier(frame):
|
||||||
@@ -153,17 +142,28 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -6,14 +6,11 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from openai.types.chat import ChatCompletionToolParam
|
from openai.types.chat import ChatCompletionToolParam
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -49,13 +46,12 @@ from pipecat.services.deepgram.stt import DeepgramSTTService
|
|||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.sync.base_notifier import BaseNotifier
|
from pipecat.sync.base_notifier import BaseNotifier
|
||||||
from pipecat.sync.event_notifier import EventNotifier
|
from pipecat.sync.event_notifier import EventNotifier
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
classifier_statement = "Determine if the user's statement ends with a complete thought and you should respond. The user text is transcribed speech. It may contain multiple fragments concatentated together. You are trying to determine only the completeness of the last user statement. The previous assistant statement is provided only for context. Categorize the text as either complete with the user now expecting a response, or incomplete. Return 'YES' if text is likely complete and the user is expecting a response. Return 'NO' if the text seems to be a partial expression or unfinished thought."
|
classifier_statement = "Determine if the user's statement ends with a complete thought and you should respond. The user text is transcribed speech. It may contain multiple fragments concatentated together. You are trying to determine only the completeness of the last user statement. The previous assistant statement is provided only for context. Categorize the text as either complete with the user now expecting a response, or incomplete. Return 'YES' if text is likely complete and the user is expecting a response. Return 'NO' if the text seems to be a partial expression or unfinished thought."
|
||||||
|
|
||||||
@@ -204,15 +200,13 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
|
|||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
None,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -358,32 +352,42 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_app_message")
|
@transport.event_handler("on_app_message")
|
||||||
async def on_app_message(transport, message, sender):
|
async def on_app_message(transport, message):
|
||||||
logger.debug(f"Received app message: {message} - {sender}")
|
logger.debug(f"Received app message: {message}")
|
||||||
if "message" not in message:
|
if "message" not in message:
|
||||||
return
|
return
|
||||||
|
|
||||||
await task.queue_frames(
|
await task.queue_frames(
|
||||||
[
|
[
|
||||||
UserStartedSpeakingFrame(),
|
UserStartedSpeakingFrame(),
|
||||||
TranscriptionFrame(
|
TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]),
|
||||||
user_id=sender, timestamp=time.time(), text=message["message"]
|
|
||||||
),
|
|
||||||
UserStoppedSpeakingFrame(),
|
UserStoppedSpeakingFrame(),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -6,14 +6,11 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from openai.types.chat import ChatCompletionToolParam
|
from openai.types.chat import ChatCompletionToolParam
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -50,13 +47,12 @@ from pipecat.services.deepgram.stt import DeepgramSTTService
|
|||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.sync.base_notifier import BaseNotifier
|
from pipecat.sync.base_notifier import BaseNotifier
|
||||||
from pipecat.sync.event_notifier import EventNotifier
|
from pipecat.sync.event_notifier import EventNotifier
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
classifier_statement = """CRITICAL INSTRUCTION:
|
classifier_statement = """CRITICAL INSTRUCTION:
|
||||||
You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO".
|
You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO".
|
||||||
@@ -408,15 +404,13 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
|
|||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
None,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -566,9 +560,9 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
@@ -579,24 +573,34 @@ async def main():
|
|||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_app_message")
|
@transport.event_handler("on_app_message")
|
||||||
async def on_app_message(transport, message, sender):
|
async def on_app_message(transport, message):
|
||||||
logger.debug(f"Received app message: {message} - {sender}")
|
logger.debug(f"Received app message: {message}")
|
||||||
if "message" not in message:
|
if "message" not in message:
|
||||||
return
|
return
|
||||||
|
|
||||||
await task.queue_frames(
|
await task.queue_frames(
|
||||||
[
|
[
|
||||||
UserStartedSpeakingFrame(),
|
UserStartedSpeakingFrame(),
|
||||||
TranscriptionFrame(
|
TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]),
|
||||||
user_id=sender, timestamp=time.time(), text=message["message"]
|
|
||||||
),
|
|
||||||
UserStoppedSpeakingFrame(),
|
UserStoppedSpeakingFrame(),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -6,14 +6,11 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
import google.ai.generativelanguage as glm
|
import google.ai.generativelanguage as glm
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -47,12 +44,12 @@ from pipecat.services.cartesia.tts import CartesiaTTSService
|
|||||||
from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService
|
from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService
|
||||||
from pipecat.sync.base_notifier import BaseNotifier
|
from pipecat.sync.base_notifier import BaseNotifier
|
||||||
from pipecat.sync.event_notifier import EventNotifier
|
from pipecat.sync.event_notifier import EventNotifier
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
TRANSCRIBER_MODEL = "gemini-2.0-flash-001"
|
TRANSCRIBER_MODEL = "gemini-2.0-flash-001"
|
||||||
CLASSIFIER_MODEL = "gemini-2.0-flash-001"
|
CLASSIFIER_MODEL = "gemini-2.0-flash-001"
|
||||||
@@ -626,15 +623,13 @@ class OutputGate(FrameProcessor):
|
|||||||
break
|
break
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
None,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -685,9 +680,7 @@ async def main():
|
|||||||
|
|
||||||
audio_accumulater = AudioAccumulator()
|
audio_accumulater = AudioAccumulator()
|
||||||
# This sends a UserStoppedSpeakingFrame and triggers the notifier event
|
# This sends a UserStoppedSpeakingFrame and triggers the notifier event
|
||||||
completeness_check = CompletenessCheck(
|
completeness_check = CompletenessCheck(notifier=notifier, audio_accumulator=audio_accumulater)
|
||||||
notifier=notifier, audio_accumulator=audio_accumulater
|
|
||||||
)
|
|
||||||
|
|
||||||
async def block_user_stopped_speaking(frame):
|
async def block_user_stopped_speaking(frame):
|
||||||
return not isinstance(frame, UserStoppedSpeakingFrame)
|
return not isinstance(frame, UserStoppedSpeakingFrame)
|
||||||
@@ -744,31 +737,41 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
await transport.capture_participant_transcription(participant["id"])
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
@transport.event_handler("on_app_message")
|
@transport.event_handler("on_app_message")
|
||||||
async def on_app_message(transport, message, sender):
|
async def on_app_message(transport, message):
|
||||||
logger.debug(f"Received app message: {message} - {sender}")
|
logger.debug(f"Received app message: {message}")
|
||||||
if "message" not in message:
|
if "message" not in message:
|
||||||
return
|
return
|
||||||
|
|
||||||
await task.queue_frames(
|
await task.queue_frames(
|
||||||
[
|
[
|
||||||
UserStartedSpeakingFrame(),
|
UserStartedSpeakingFrame(),
|
||||||
TranscriptionFrame(
|
TranscriptionFrame(user_id="", timestamp=time.time(), text=message["message"]),
|
||||||
user_id=sender, timestamp=time.time(), text=message["message"]
|
|
||||||
),
|
|
||||||
UserStoppedSpeakingFrame(),
|
UserStoppedSpeakingFrame(),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
from daily_runner import configure_with_args
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure_with_args
|
|
||||||
|
|
||||||
from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer
|
from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
140
examples/foundational/23-bot-background-sound-p2p.py
Normal file
140
examples/foundational/23-bot-background-sound-p2p.py
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""
|
||||||
|
Usage
|
||||||
|
-----
|
||||||
|
Set the path to your background audio file using the `INPUT_AUDIO_PATH` environment variable, then run the bot using:
|
||||||
|
|
||||||
|
INPUT_AUDIO_PATH=path/to/your_audio.mp3 python 23-bot-background-sound.py
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
INPUT_AUDIO_PATH=my_audio.mp3 python 23-bot-background-sound.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer
|
||||||
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
|
from pipecat.frames.frames import MixerEnableFrame, MixerUpdateSettingsFrame
|
||||||
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
audio_path = os.getenv("INPUT_AUDIO_PATH")
|
||||||
|
if not audio_path:
|
||||||
|
raise ValueError("No INPUT_AUDIO_PATH specified in environment variables")
|
||||||
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
|
soundfile_mixer = SoundfileMixer(
|
||||||
|
sound_files={"office": audio_path},
|
||||||
|
default_sound="office",
|
||||||
|
volume=2.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
transport = SmallWebRTCTransport(
|
||||||
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
audio_out_mixer=soundfile_mixer,
|
||||||
|
vad_enabled=True,
|
||||||
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
vad_audio_passthrough=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
|
tts = CartesiaTTSService(
|
||||||
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||||
|
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||||
|
)
|
||||||
|
|
||||||
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
context = OpenAILLMContext(messages)
|
||||||
|
context_aggregator = llm.create_context_aggregator(context)
|
||||||
|
|
||||||
|
pipeline = Pipeline(
|
||||||
|
[
|
||||||
|
transport.input(), # Transport user input
|
||||||
|
stt, # STT service
|
||||||
|
context_aggregator.user(), # User responses
|
||||||
|
llm, # LLM
|
||||||
|
tts, # TTS
|
||||||
|
transport.output(), # Transport bot output
|
||||||
|
context_aggregator.assistant(), # Assistant spoken responses
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
task = PipelineTask(
|
||||||
|
pipeline,
|
||||||
|
params=PipelineParams(
|
||||||
|
allow_interruptions=True,
|
||||||
|
enable_metrics=True,
|
||||||
|
enable_usage_metrics=True,
|
||||||
|
report_only_initial_ttfb=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected: {client}")
|
||||||
|
# Show how to use mixer control frames.
|
||||||
|
await asyncio.sleep(10.0)
|
||||||
|
await task.queue_frame(MixerUpdateSettingsFrame({"volume": 0.5}))
|
||||||
|
await asyncio.sleep(5.0)
|
||||||
|
await task.queue_frame(MixerEnableFrame(False))
|
||||||
|
await asyncio.sleep(5.0)
|
||||||
|
await task.queue_frame(MixerEnableFrame(True))
|
||||||
|
await asyncio.sleep(5.0)
|
||||||
|
# Kick off the conversation.
|
||||||
|
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
@@ -6,14 +6,12 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from openai.types.chat import ChatCompletionToolParam
|
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
@@ -23,13 +21,12 @@ from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFil
|
|||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||||
from pipecat.services.deepgram.tts import DeepgramTTSService
|
from pipecat.services.deepgram.tts import DeepgramTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
# Add a delay to test interruption during function calls
|
# Add a delay to test interruption during function calls
|
||||||
@@ -39,15 +36,13 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
|
|||||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, _) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
None,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
@@ -56,6 +51,7 @@ async def main():
|
|||||||
)
|
)
|
||||||
|
|
||||||
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
||||||
|
|
||||||
# Configure the mute processor with both strategies
|
# Configure the mute processor with both strategies
|
||||||
stt_mute_processor = STTMuteFilter(
|
stt_mute_processor = STTMuteFilter(
|
||||||
config=STTMuteConfig(
|
config=STTMuteConfig(
|
||||||
@@ -71,15 +67,10 @@ async def main():
|
|||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||||
llm.register_function("get_current_weather", fetch_weather_from_api)
|
llm.register_function("get_current_weather", fetch_weather_from_api)
|
||||||
|
|
||||||
tools = [
|
weather_function = FunctionSchema(
|
||||||
ChatCompletionToolParam(
|
name="get_current_weather",
|
||||||
type="function",
|
description="Get the current weather",
|
||||||
function={
|
properties={
|
||||||
"name": "get_current_weather",
|
|
||||||
"description": "Get the current weather",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"location": {
|
"location": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "The city and state, e.g. San Francisco, CA",
|
"description": "The city and state, e.g. San Francisco, CA",
|
||||||
@@ -87,14 +78,12 @@ async def main():
|
|||||||
"format": {
|
"format": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": ["celsius", "fahrenheit"],
|
"enum": ["celsius", "fahrenheit"],
|
||||||
"description": "The temperature unit to use. Infer this from the users location.",
|
"description": "The temperature unit to use. Infer this from the user's location.",
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["location", "format"],
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
required=["location", "format"],
|
||||||
)
|
)
|
||||||
]
|
tools = ToolsSchema(standard_tools=[weather_function])
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
@@ -121,8 +110,9 @@ async def main():
|
|||||||
|
|
||||||
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation with a weather-related prompt
|
# Kick off the conversation with a weather-related prompt
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
@@ -132,10 +122,21 @@ async def main():
|
|||||||
)
|
)
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,16 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
import google.ai.generativelanguage as glm
|
import google.ai.generativelanguage as glm
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -37,13 +33,12 @@ from pipecat.processors.aggregators.openai_llm_context import (
|
|||||||
from pipecat.processors.frame_processor import FrameProcessor
|
from pipecat.processors.frame_processor import FrameProcessor
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService
|
from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# The system prompt for the main conversation.
|
# The system prompt for the main conversation.
|
||||||
#
|
#
|
||||||
@@ -273,18 +268,14 @@ class TranscriptionContextFixup(FrameProcessor):
|
|||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
transport = SmallWebRTCTransport(
|
||||||
room_url,
|
webrtc_connection=webrtc_connection,
|
||||||
token,
|
params=TransportParams(
|
||||||
"Respond bot",
|
audio_in_enabled=True,
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
# No transcription at all. just audio input to Gemini!
|
|
||||||
# transcription_enabled=True,
|
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_analyzer=SileroVADAnalyzer(),
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
@@ -360,15 +351,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -20,39 +16,48 @@ from pipecat.pipeline.pipeline import Pipeline
|
|||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
|
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
|
# Load environment variables
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Initialize the SmallWebRTCTransport with the connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, token) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
audio_in_enabled=True,
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
|
camera_in_enabled=False,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
# set stop_secs to something roughly similar to the internal setting
|
# set stop_secs to something roughly similar to the internal setting
|
||||||
# of the Multimodal Live api, just to align events. This doesn't really
|
# of the Multimodal Live api, just to align events.
|
||||||
# matter because we can only use the Multimodal Live API's phrase
|
|
||||||
# endpointing, for now.
|
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
|
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Create the Gemini Multimodal Live LLM service
|
||||||
|
system_instruction = f"""
|
||||||
|
You are a helpful AI assistant.
|
||||||
|
Your goal is to demonstrate your capabilities in a helpful and engaging way.
|
||||||
|
Your output will be converted to audio so don't include special characters in your answers.
|
||||||
|
Respond to what the user said in a creative and helpful way.
|
||||||
|
"""
|
||||||
|
|
||||||
llm = GeminiMultimodalLiveLLMService(
|
llm = GeminiMultimodalLiveLLMService(
|
||||||
api_key=os.getenv("GOOGLE_API_KEY"),
|
api_key=os.getenv("GOOGLE_API_KEY"),
|
||||||
# system_instruction="Talk like a pirate."
|
system_instruction=system_instruction,
|
||||||
|
voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
|
||||||
|
transcribe_user_audio=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Build the pipeline
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
@@ -61,6 +66,7 @@ async def main():
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Configure the pipeline task
|
||||||
task = PipelineTask(
|
task = PipelineTask(
|
||||||
pipeline,
|
pipeline,
|
||||||
params=PipelineParams(
|
params=PipelineParams(
|
||||||
@@ -70,25 +76,40 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
# Handle client connection event
|
||||||
async def on_first_participant_joined(transport, participant):
|
@transport.event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
await task.queue_frames(
|
await task.queue_frames(
|
||||||
[
|
[
|
||||||
LLMMessagesAppendFrame(
|
LLMMessagesAppendFrame(
|
||||||
messages=[
|
messages=[
|
||||||
{
|
{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": "Greet the user.",
|
"content": f"Greet the user and introduce yourself.",
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
runner = PipelineRunner()
|
# Handle client disconnection events
|
||||||
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
# Run the pipeline
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -20,23 +16,21 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
|
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
logger.info(f"Starting bot")
|
||||||
|
|
||||||
async def main():
|
# Initialize the SmallWebRTCTransport with the connection
|
||||||
async with aiohttp.ClientSession() as session:
|
transport = SmallWebRTCTransport(
|
||||||
(room_url, token) = await configure(session)
|
webrtc_connection=webrtc_connection,
|
||||||
|
params=TransportParams(
|
||||||
transport = DailyTransport(
|
audio_in_enabled=True,
|
||||||
room_url,
|
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
DailyParams(
|
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
@@ -53,7 +47,6 @@ async def main():
|
|||||||
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
|
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
|
||||||
# system_instruction="Talk like a pirate."
|
# system_instruction="Talk like a pirate."
|
||||||
transcribe_user_audio=True,
|
transcribe_user_audio=True,
|
||||||
transcribe_model_audio=True,
|
|
||||||
# inference_on_context_initialization=False,
|
# inference_on_context_initialization=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -96,14 +89,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -4,15 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
|
||||||
@@ -23,13 +19,12 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
|
from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.base_transport import TransportParams
|
||||||
|
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||||
|
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
logger.remove(0)
|
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||||
temperature = 75 if args["format"] == "fahrenheit" else 24
|
temperature = 75 if args["format"] == "fahrenheit" else 24
|
||||||
@@ -51,15 +46,14 @@ for the weather, call this function.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def run_bot(webrtc_connection: SmallWebRTCConnection):
|
||||||
async with aiohttp.ClientSession() as session:
|
logger.info(f"Starting bot")
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
|
|
||||||
transport = DailyTransport(
|
# Initialize the SmallWebRTCTransport with the connection
|
||||||
room_url,
|
transport = SmallWebRTCTransport(
|
||||||
token,
|
webrtc_connection=webrtc_connection,
|
||||||
"Respond bot",
|
params=TransportParams(
|
||||||
DailyParams(
|
audio_in_enabled=True,
|
||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
vad_enabled=True,
|
vad_enabled=True,
|
||||||
vad_audio_passthrough=True,
|
vad_audio_passthrough=True,
|
||||||
@@ -124,14 +118,27 @@ async def main():
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_participant_joined")
|
@transport.event_handler("on_client_connected")
|
||||||
async def on_first_participant_joined(transport, participant):
|
async def on_client_connected(transport, client):
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
# Kick off the conversation.
|
||||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||||
|
|
||||||
runner = PipelineRunner()
|
@transport.event_handler("on_client_disconnected")
|
||||||
|
async def on_client_disconnected(transport, client):
|
||||||
|
logger.info(f"Client disconnected")
|
||||||
|
|
||||||
|
@transport.event_handler("on_client_closed")
|
||||||
|
async def on_client_closed(transport, client):
|
||||||
|
logger.info(f"Client closed connection")
|
||||||
|
await task.cancel()
|
||||||
|
|
||||||
|
runner = PipelineRunner(handle_sigint=False)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
from run import main
|
||||||
|
|
||||||
|
main()
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
from daily_runner import configure
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from runner import configure
|
|
||||||
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
@@ -53,7 +53,6 @@ async def main():
|
|||||||
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
|
voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
|
||||||
# system_instruction="Talk like a pirate."
|
# system_instruction="Talk like a pirate."
|
||||||
transcribe_user_audio=True,
|
transcribe_user_audio=True,
|
||||||
transcribe_model_audio=True,
|
|
||||||
# inference_on_context_initialization=False,
|
# inference_on_context_initialization=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user