# # Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import datetime import io import os import sys import wave import aiofiles from dotenv import load_dotenv from fastapi import WebSocket from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.dtmf_aggregator import DTMFAggregator from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.serializers.twilio import TwilioFrameSerializer from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.network.fastapi_websocket import ( FastAPIWebsocketParams, FastAPIWebsocketTransport, ) load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") async def save_audio(server_name: str, audio: bytes, sample_rate: int, num_channels: int): if len(audio) > 0: filename = ( f"{server_name}_recording_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.wav" ) with io.BytesIO() as buffer: with wave.open(buffer, "wb") as wf: wf.setsampwidth(2) wf.setnchannels(num_channels) wf.setframerate(sample_rate) wf.writeframes(audio) async with aiofiles.open(filename, "wb") as file: await file.write(buffer.getvalue()) logger.info(f"Merged audio saved to {filename}") else: logger.info("No audio data to save") async def run_bot(websocket_client: WebSocket, stream_sid: str, call_sid: str, testing: bool): serializer = TwilioFrameSerializer( stream_sid=stream_sid, call_sid=call_sid, account_sid=os.getenv("TWILIO_ACCOUNT_SID", ""), auth_token=os.getenv("TWILIO_AUTH_TOKEN", ""), ) transport = FastAPIWebsocketTransport( websocket=websocket_client, params=FastAPIWebsocketParams( audio_in_enabled=True, audio_out_enabled=True, add_wav_header=False, vad_analyzer=SileroVADAnalyzer(), serializer=serializer, ), ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady push_silence_after_stop=testing, ) # Create DTMF aggregator dtmf_aggregator = DTMFAggregator( timeout=3.0, # 3 second timeout prefix="Menu selection: ", # Helpful prefix for LLM ) messages = [ { "role": "system", "content": """You are an elementary teacher in an audio call. Your output will be converted to audio so don't include special characters in your answers. When you receive input starting with "Menu selection:", this represents button presses on the phone keypad. For example: - "Menu selection: 1" means they pressed button 1 - "Menu selection: 123#" means they pressed 1, 2, 3, then # (pound) - Common patterns: single digits for menu choices, sequences ending with # for completed entries Respond to both voice and keypad input in short sentences.""", }, ] context = OpenAILLMContext(messages) context_aggregator = llm.create_context_aggregator(context) # NOTE: Watch out! This will save all the conversation in memory. You can # pass `buffer_size` to get periodic callbacks. audiobuffer = AudioBufferProcessor(user_continuous_stream=not testing) pipeline = Pipeline( [ transport.input(), # Websocket input from client dtmf_aggregator, # DTMF aggregator (processes DTMF before STT) stt, # Speech-To-Text context_aggregator.user(), llm, # LLM tts, # Text-To-Speech transport.output(), # Websocket output to client audiobuffer, # Used to buffer the audio in the pipeline context_aggregator.assistant(), ] ) task = PipelineTask( pipeline, params=PipelineParams( audio_in_sample_rate=8000, audio_out_sample_rate=8000, allow_interruptions=True, ), ) @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): # Start recording. await audiobuffer.start_recording() # Kick off the conversation. messages.append( { "role": "system", "content": "Please introduce yourself to the user and mention they can use voice or press numbers on their phone keypad to interact.", } ) await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): await task.cancel() @audiobuffer.event_handler("on_audio_data") async def on_audio_data(buffer, audio, sample_rate, num_channels): server_name = f"server_{websocket_client.client.port}" await save_audio(server_name, audio, sample_rate, num_channels) # We use `handle_sigint=False` because `uvicorn` is controlling keyboard # interruptions. We use `force_gc=True` to force garbage collection after # the runner finishes running a task which could be useful for long running # applications with multiple clients connecting. runner = PipelineRunner(handle_sigint=False, force_gc=True) await runner.run(task)