# # Copyright (c) 2024, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio import aiohttp import os import sys from PIL import Image from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import ( LLMAssistantResponseAggregator, LLMUserResponseAggregator, ) from pipecat.frames.frames import ( OutputImageRawFrame, SpriteFrame, Frame, LLMMessagesFrame, TTSAudioRawFrame, TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from runner import configure from loguru import logger from dotenv import load_dotenv load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") sprites = [] script_dir = os.path.dirname(__file__) for i in range(1, 26): # Build the full path to the image file full_path = os.path.join(script_dir, f"assets/robot0{i}.png") # Get the filename without the extension to use as the dictionary key # Open the image and convert it to bytes with Image.open(full_path) as img: sprites.append(OutputImageRawFrame(image=img.tobytes(), size=img.size, format=img.format)) flipped = sprites[::-1] sprites.extend(flipped) # When the bot isn't talking, show a static image of the cat listening quiet_frame = sprites[0] talking_frame = SpriteFrame(images=sprites) class TalkingAnimation(FrameProcessor): """ This class starts a talking animation when it receives an first AudioFrame, and then returns to a "quiet" sprite when it sees a TTSStoppedFrame. """ def __init__(self): super().__init__() self._is_talking = False async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, TTSAudioRawFrame): if not self._is_talking: await self.push_frame(talking_frame) self._is_talking = True elif isinstance(frame, TTSStoppedFrame): await self.push_frame(quiet_frame) self._is_talking = False await self.push_frame(frame) async def main(): async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) transport = DailyTransport( room_url, token, "Chatbot", DailyParams( audio_out_enabled=True, camera_out_enabled=True, camera_out_width=1024, camera_out_height=576, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), transcription_enabled=True, # # Spanish # # transcription_settings=DailyTranscriptionSettings( # language="es", # tier="nova", # model="2-general" # ) ), ) tts = ElevenLabsTTSService( api_key=os.getenv("ELEVENLABS_API_KEY"), # # English # voice_id="pNInz6obpgDQGcFmaJgB", # # Spanish # # model="eleven_multilingual_v2", # voice_id="gD1IexrzCvsXPHUuT0s3", ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") messages = [ { "role": "system", # # English # "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.", # # Spanish # # "content": "Eres Chatbot, un amigable y útil robot. Tu objetivo es demostrar tus capacidades de una manera breve. Tus respuestas se convertiran a audio así que nunca no debes incluir caracteres especiales. Contesta a lo que el usuario pregunte de una manera creativa, útil y breve. Empieza por presentarte a ti mismo.", }, ] user_response = LLMUserResponseAggregator() assistant_response = LLMAssistantResponseAggregator() ta = TalkingAnimation() pipeline = Pipeline( [ transport.input(), user_response, llm, tts, ta, transport.output(), assistant_response, ] ) task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) await task.queue_frame(quiet_frame) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): transport.capture_participant_transcription(participant["id"]) await task.queue_frames([LLMMessagesFrame(messages)]) runner = PipelineRunner() await runner.run(task) if __name__ == "__main__": asyncio.run(main())