# # Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio import os import sys import aiohttp from dotenv import load_dotenv from loguru import logger from PIL import Image from pipecatcloud.agent import DailySessionArguments from pipecat.audio.turn.smart_turn.fal_smart_turn import FalSmartTurnAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, Frame, MetricsFrame, OutputImageRawFrame, SpriteFrame, ) from pipecat.metrics.metrics import SmartTurnMetricsData from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import ( RTVIConfig, RTVIObserver, RTVIProcessor, RTVIServerMessageFrame, ) from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) # Check if we're in local development mode LOCAL = os.getenv("LOCAL") logger.remove() logger.add(sys.stderr, level="DEBUG") sprites = [] script_dir = os.path.dirname(__file__) # Load sequential animation frames for i in range(1, 26): # Build the full path to the image file full_path = os.path.join(script_dir, f"assets/robot0{i}.png") # Get the filename without the extension to use as the dictionary key # Open the image and convert it to bytes with Image.open(full_path) as img: sprites.append(OutputImageRawFrame(image=img.tobytes(), size=img.size, format=img.format)) # Create a smooth animation by adding reversed frames flipped = sprites[::-1] sprites.extend(flipped) # Define static and animated states quiet_frame = sprites[0] # Static frame for when bot is listening talking_frame = SpriteFrame(images=sprites) # Animation sequence for when bot is talking class TalkingAnimation(FrameProcessor): """Manages the bot's visual animation states. Switches between static (listening) and animated (talking) states based on the bot's current speaking status. """ def __init__(self): super().__init__() self._is_talking = False async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames and update animation state. Args: frame: The incoming frame to process direction: The direction of frame flow in the pipeline """ await super().process_frame(frame, direction) # Switch to talking animation when bot starts speaking if isinstance(frame, BotStartedSpeakingFrame): if not self._is_talking: await self.push_frame(talking_frame) self._is_talking = True # Return to static frame when bot stops speaking elif isinstance(frame, BotStoppedSpeakingFrame): await self.push_frame(quiet_frame) self._is_talking = False await self.push_frame(frame, direction) class SmartTurnMetricsProcessor(FrameProcessor): """Processes the metrics data from Smart Turn Analyzer. This processor is responsible for handling smart turn metrics data and forwarding it to the client UI via RTVI. """ async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames and handle Smart Turn metrics. Args: frame: The incoming frame to process direction: The direction of frame flow in the pipeline """ await super().process_frame(frame, direction) # Handle Smart Turn metrics if isinstance(frame, MetricsFrame): for metrics in frame.data: if isinstance(metrics, SmartTurnMetricsData): logger.info(f"Smart Turn metrics: {metrics}") # Create a payload with the smart turn prediction data smart_turn_data = { "type": "smart_turn_result", "is_complete": metrics.is_complete, "probability": metrics.probability, "inference_time_ms": metrics.inference_time_ms, "server_total_time_ms": metrics.server_total_time_ms, "e2e_processing_time_ms": metrics.e2e_processing_time_ms, } # Send the data to the client via RTVI rtvi_frame = RTVIServerMessageFrame(data=smart_turn_data) await self.push_frame(rtvi_frame) await self.push_frame(frame, direction) async def main(transport: DailyTransport): # Configure your STT, LLM, and TTS services here # Swap out different processors or properties to customize your bot stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # Set up the initial context for the conversation # You can specified initial system and assistant messages here messages = [ { "role": "system", "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.", }, ] # This sets up the LLM context by providing messages and tools context = OpenAILLMContext(messages) context_aggregator = llm.create_context_aggregator(context) ta = TalkingAnimation() smart_turn_metrics_processor = SmartTurnMetricsProcessor() # RTVI events for Pipecat client UI rtvi = RTVIProcessor(config=RTVIConfig(config=[])) # A core voice AI pipeline # Add additional processors to customize the bot's behavior pipeline = Pipeline( [ transport.input(), rtvi, smart_turn_metrics_processor, stt, context_aggregator.user(), llm, tts, ta, transport.output(), context_aggregator.assistant(), ] ) task = PipelineTask( pipeline, params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, ), observers=[RTVIObserver(rtvi)], ) @rtvi.event_handler("on_client_ready") async def on_client_ready(rtvi): logger.debug("Client ready event received") await rtvi.set_bot_ready() # Kick off the conversation await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): logger.info("First participant joined: {}", participant["id"]) # Push a static frame to show the bot is listening await task.queue_frame(quiet_frame) @transport.event_handler("on_participant_left") async def on_participant_left(transport, participant, reason): logger.info("Participant left: {}", participant) await task.cancel() runner = PipelineRunner(handle_sigint=False, force_gc=True) await runner.run(task) async def bot(args: DailySessionArguments): """Main bot entry point compatible with the FastAPI route handler. Args: room_url: The Daily room URL token: The Daily room token body: The configuration object from the request body session_id: The session ID for logging """ from pipecat.audio.filters.krisp_filter import KrispFilter logger.info(f"Bot process initialized {args.room_url} {args.token}") async with aiohttp.ClientSession() as session: transport = DailyTransport( args.room_url, args.token, "Smart Turn Bot", params=DailyParams( audio_in_enabled=True, audio_in_filter=KrispFilter(), audio_out_enabled=True, video_out_enabled=True, video_out_width=1024, video_out_height=576, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), turn_analyzer=FalSmartTurnAnalyzer( api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=session ), ), ) try: await main(transport) logger.info("Bot process completed") except Exception as e: logger.exception(f"Error in bot process: {str(e)}") raise # Local development async def local_daily(): """Daily transport for local development.""" from runner import configure try: async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) transport = DailyTransport( room_url, token, "Smart Turn Bot", params=DailyParams( audio_in_enabled=True, audio_out_enabled=True, video_out_enabled=True, video_out_width=1024, video_out_height=576, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), turn_analyzer=FalSmartTurnAnalyzer( api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=session ), ), ) await main(transport) except Exception as e: logger.exception(f"Error in local development mode: {e}") # Local development entry point if LOCAL and __name__ == "__main__": try: asyncio.run(local_daily()) except Exception as e: logger.exception(f"Failed to run in local mode: {e}")