d779a5b4ea
These messages are developer instructions to the assistant (e.g. "Please introduce yourself to the user"), not simulated user input. The "developer" role is semantically correct for this purpose.
243 lines
8.3 KiB
Python
243 lines
8.3 KiB
Python
#
|
|
# Copyright (c) 2024-2026, Daily
|
|
#
|
|
# SPDX-License-Identifier: BSD 2-Clause License
|
|
#
|
|
|
|
|
|
import asyncio
|
|
import os
|
|
|
|
from dotenv import load_dotenv
|
|
from loguru import logger
|
|
|
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
|
from pipecat.frames.frames import (
|
|
EndTaskFrame,
|
|
LLMMessagesAppendFrame,
|
|
LLMRunFrame,
|
|
TTSSpeakFrame,
|
|
UserIdleTimeoutUpdateFrame,
|
|
)
|
|
from pipecat.pipeline.pipeline import Pipeline
|
|
from pipecat.pipeline.runner import PipelineRunner
|
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.aggregators.llm_response_universal import (
|
|
LLMContextAggregatorPair,
|
|
LLMUserAggregatorParams,
|
|
)
|
|
from pipecat.processors.frame_processor import FrameDirection
|
|
from pipecat.runner.types import RunnerArguments
|
|
from pipecat.runner.utils import create_transport
|
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
|
from pipecat.services.deepgram.stt import DeepgramSTTService
|
|
from pipecat.services.llm_service import FunctionCallParams
|
|
from pipecat.services.openai.llm import OpenAILLMService
|
|
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
|
from pipecat.transports.daily.transport import DailyParams
|
|
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
|
|
|
|
load_dotenv(override=True)
|
|
|
|
|
|
class IdleHandler:
|
|
"""Helper class to manage user idle retry logic."""
|
|
|
|
def __init__(self):
|
|
self._retry_count = 0
|
|
|
|
def reset(self):
|
|
"""Reset the retry count when user becomes active."""
|
|
self._retry_count = 0
|
|
|
|
async def handle_idle(self, aggregator):
|
|
"""Handle user idle event with escalating prompts."""
|
|
self._retry_count += 1
|
|
|
|
if self._retry_count == 1:
|
|
# First attempt: Add a gentle prompt to the conversation
|
|
message = {
|
|
"role": "user",
|
|
"content": "The user has been quiet. Politely and briefly ask if they're still there.",
|
|
}
|
|
await aggregator.push_frame(LLMMessagesAppendFrame([message], run_llm=True))
|
|
elif self._retry_count == 2:
|
|
# Second attempt: More direct prompt
|
|
message = {
|
|
"role": "user",
|
|
"content": "The user is still inactive. Ask if they'd like to continue our conversation.",
|
|
}
|
|
await aggregator.push_frame(LLMMessagesAppendFrame([message], run_llm=True))
|
|
else:
|
|
# Third attempt: End the conversation
|
|
await aggregator.push_frame(
|
|
TTSSpeakFrame("It seems like you're busy right now. Have a nice day!")
|
|
)
|
|
await aggregator.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
|
|
|
|
|
|
async def fetch_weather_from_api(params: FunctionCallParams):
|
|
# Simulate a slow API call, waiting longer than the user idle timeout.
|
|
await asyncio.sleep(3)
|
|
await params.result_callback({"conditions": "nice", "temperature": "75"})
|
|
|
|
|
|
async def fetch_restaurant_recommendation(params: FunctionCallParams):
|
|
await asyncio.sleep(6)
|
|
await params.result_callback({"name": "The Golden Dragon"})
|
|
|
|
|
|
# We use lambdas to defer transport parameter creation until the transport
|
|
# type is selected at runtime.
|
|
transport_params = {
|
|
"daily": lambda: DailyParams(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
),
|
|
"twilio": lambda: FastAPIWebsocketParams(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
),
|
|
"webrtc": lambda: TransportParams(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
),
|
|
}
|
|
|
|
|
|
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|
logger.info(f"Starting bot")
|
|
|
|
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
|
|
|
tts = CartesiaTTSService(
|
|
api_key=os.getenv("CARTESIA_API_KEY"),
|
|
settings=CartesiaTTSService.Settings(
|
|
voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
|
),
|
|
)
|
|
|
|
llm = OpenAILLMService(
|
|
api_key=os.getenv("OPENAI_API_KEY"),
|
|
settings=OpenAILLMService.Settings(
|
|
system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.",
|
|
),
|
|
)
|
|
|
|
llm.register_function("get_current_weather", fetch_weather_from_api)
|
|
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
|
|
|
|
@llm.event_handler("on_function_calls_started")
|
|
async def on_function_calls_started(service, function_calls):
|
|
await tts.queue_frame(TTSSpeakFrame("Let me check on that."))
|
|
|
|
weather_function = FunctionSchema(
|
|
name="get_current_weather",
|
|
description="Get the current weather",
|
|
properties={
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city and state, e.g. San Francisco, CA",
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use. Infer this from the user's location.",
|
|
},
|
|
},
|
|
required=["location", "format"],
|
|
)
|
|
restaurant_function = FunctionSchema(
|
|
name="get_restaurant_recommendation",
|
|
description="Get a restaurant recommendation",
|
|
properties={
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city and state, e.g. San Francisco, CA",
|
|
},
|
|
},
|
|
required=["location"],
|
|
)
|
|
tools = ToolsSchema(standard_tools=[weather_function, restaurant_function])
|
|
|
|
context = LLMContext(tools=tools)
|
|
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
|
context,
|
|
user_params=LLMUserAggregatorParams(
|
|
user_idle_timeout=5.0, # Detect user idle after 5 seconds
|
|
vad_analyzer=SileroVADAnalyzer(),
|
|
),
|
|
)
|
|
|
|
pipeline = Pipeline(
|
|
[
|
|
transport.input(), # Transport user input
|
|
stt,
|
|
user_aggregator, # User aggregator with built-in idle detection
|
|
llm, # LLM
|
|
tts, # TTS
|
|
transport.output(), # Transport bot output
|
|
assistant_aggregator,
|
|
]
|
|
)
|
|
|
|
task = PipelineTask(
|
|
pipeline,
|
|
params=PipelineParams(
|
|
enable_metrics=True,
|
|
enable_usage_metrics=True,
|
|
),
|
|
idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
|
|
)
|
|
|
|
# Set up idle handling with retry logic
|
|
idle_handler = IdleHandler()
|
|
|
|
@user_aggregator.event_handler("on_user_turn_idle")
|
|
async def on_user_turn_idle(aggregator):
|
|
logger.info(f"User turn idle")
|
|
await idle_handler.handle_idle(aggregator)
|
|
|
|
@user_aggregator.event_handler("on_user_turn_started")
|
|
async def on_user_turn_started(aggregator, strategy):
|
|
idle_handler.reset()
|
|
|
|
@transport.event_handler("on_client_connected")
|
|
async def on_client_connected(transport, client):
|
|
logger.info(f"Client connected")
|
|
# Kick off the conversation.
|
|
context.add_message(
|
|
{"role": "developer", "content": "Please introduce yourself to the user."}
|
|
)
|
|
await task.queue_frames([LLMRunFrame()])
|
|
await asyncio.sleep(30)
|
|
logger.info(f"Disabling idle detection")
|
|
await task.queue_frames([UserIdleTimeoutUpdateFrame(timeout=0)])
|
|
await asyncio.sleep(30)
|
|
logger.info(f"Enabling idle detection")
|
|
await task.queue_frames([UserIdleTimeoutUpdateFrame(timeout=5)])
|
|
|
|
@transport.event_handler("on_client_disconnected")
|
|
async def on_client_disconnected(transport, client):
|
|
logger.info(f"Client disconnected")
|
|
await task.cancel()
|
|
|
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
|
|
|
await runner.run(task)
|
|
|
|
|
|
async def bot(runner_args: RunnerArguments):
|
|
"""Main bot entry point compatible with Pipecat Cloud."""
|
|
transport = await create_transport(runner_args, transport_params)
|
|
await run_bot(transport, runner_args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from pipecat.runner.run import main
|
|
|
|
main()
|