# # Copyright (c) 2024–2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """OpenAI Bot Implementation. This module implements a chatbot using OpenAI's GPT-4 model for natural language processing. It includes: - Real-time audio/video interaction through Daily - Animated robot avatar - Text-to-speech using ElevenLabs - Support for both English and Spanish The bot runs as part of a pipeline that processes audio/video frames and manages the conversation flow. """ import os import sys import aiohttp from dotenv import load_dotenv from loguru import logger from PIL import Image from pipecatcloud.agent import DailySessionArguments, SessionArguments, WebSocketSessionArguments from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, Frame, OutputImageRawFrame, SpriteFrame, TTSSpeakFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.gladia.stt import GladiaSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.network.fastapi_websocket import ( FastAPIWebsocketParams, FastAPIWebsocketTransport, ) from pipecat.transports.network.small_webrtc import SmallWebRTCTransport from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) # Check if we're in local development mode LOCAL_RUN = os.getenv("LOCAL_RUN") if LOCAL_RUN: import asyncio import webbrowser try: from local_runner import configure except ImportError: logger.error("Could not import local_runner module. Local development mode may not work.") # Logger for local dev # logger.add(sys.stderr, level="DEBUG") async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): """Fetch weather data dummy function. This function simulates fetching weather data from an external API. It demonstrates how to call an external service from the language model. """ await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) async def main(transport: BaseTransport): """Main bot execution function. Sets up and runs the bot pipeline including: - Speech-to-text and text-to-speech services - Language model integration - Animation processing - RTVI event handling Uses the transport defined by the calling function. See below for various ways to start the bot with different transports. """ tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="c45bc5ec-dc68-4feb-8829-6e6b2748095d", # Movieman ) stt = GladiaSTTService(api_key=os.getenv("GLADIA_API_KEY")) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") # Register your function call providing the function name and callback llm.register_function("get_current_weather", fetch_weather_from_api) # Define your function call using the FunctionSchema # Learn more about function calling in Pipecat: # https://docs.pipecat.ai/guides/features/function-calling weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", properties={ "location": { "type": "string", "description": "The city and state, e.g. San Francisco, CA", }, "format": { "type": "string", "enum": ["celsius", "fahrenheit"], "description": "The temperature unit to use. Infer this from the user's location.", }, }, required=["location", "format"], ) # Set up the tools schema with your weather function call tools = ToolsSchema(standard_tools=[weather_function]) # Set up initial messages for the bot messages = [ { "role": "system", "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.", }, ] # Set up conversation context and management # The context_aggregator will automatically collect conversation context # Pass your initial messages and tools to the context to initialize the context context = OpenAILLMContext(messages, tools) context_aggregator = llm.create_context_aggregator(context) # RTVI events for Pipecat client UI rtvi = RTVIProcessor(config=RTVIConfig(config=[])) # Add your processors to the pipeline pipeline = Pipeline( [ transport.input(), stt, rtvi, context_aggregator.user(), llm, tts, transport.output(), context_aggregator.assistant(), ] ) # Create a PipelineTask to manage the pipeline task = PipelineTask( pipeline, params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, ), observers=[RTVIObserver(rtvi)], ) @rtvi.event_handler("on_client_ready") async def on_client_ready(rtvi): # Notify the client that the bot is ready await rtvi.set_bot_ready() @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): # Kick off the conversation by pushing a context frame to the pipeline await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): # Cancel the PipelineTask to stop processing await task.cancel() runner = PipelineRunner() await runner.run(task) shared_params = { "audio_in_enabled": True, "audio_out_enabled": True, "video_in_enabled": False, "video_out_enabled": False, "vad_enabled": True, "vad_analyzer": SileroVADAnalyzer(), "vad_audio_passthrough": True, } async def bot(args: SessionArguments): """Bot entry point compatible with Pipecat Cloud. SessionArguments will be a different subclass depending on how the session is started. args: either DailySessionArguments or WebsocketSessionArguments DailySessionArguments: room_url: The Daily room URL token: The Daily room token body: The configuration object from the request body session_id: The session ID for logging WebsocketSessionArguments: websocket: The websocket for connecting to Twilio """ logger.info(f"Starting PCC bot. args: {args}") if isinstance(args, WebSocketSessionArguments): logger.debug("Starting WebSocket bot") start_data = args.websocket.iter_text() await start_data.__anext__() call_data = json.loads(await start_data.__anext__()) stream_sid = call_data["start"]["streamSid"] transport = FastAPIWebsocketTransport( websocket=args.websocket, params=FastAPIWebsocketParams( **shared_params, serializer=TwilioFrameSerializer(stream_sid), ), ) elif isinstance(args, DailySessionArguments): logger.debug("Starting Daily bot") transport = DailyTransport( args.room_url, args.token, "Respond bot", DailyParams(**shared_params, transcription_enabled=False), ) try: await main(transport) logger.info("Bot process completed") except Exception as e: logger.exception(f"Error in bot process: {str(e)}") raise # Local development async def local_daily(): """This is an entrypoint for running your bot locally but using Daily for the transport. To use this, you'll need to have DAILY_API_KEY set in your .env file. """ try: async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) logger.warning(f"Talk to your voice agent here: {room_url}") webbrowser.open(room_url) transport = DailyTransport( room_url=room_url, token=token, bot_name="Bot", params=DailyParams(**shared_params, transcription_enabled=False), ) await main(transport) except Exception as e: logger.exception(f"Error in local development mode: {e}") async def local_webrtc(webrtc_connection): """An entrypoint for using the SmallWebRTCTransport, which doesn't require a Daily account or API key. You'll need to run the web client and small API server included with this example to use this transport. Run `python server.py` to use it. """ transport = SmallWebRTCTransport( webrtc_connection=webrtc_connection, params=TransportParams(**shared_params) ) await main(transport) # Local development entry point if LOCAL_RUN and __name__ == "__main__": try: # Change this line to run whichever entrypoint you want to use for your bot. asyncio.run(local_daily()) except Exception as e: logger.exception(f"Failed to run in local mode: {e}")