import asyncio import aiohttp import os import sys from PIL import Image from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator from pipecat.frames.frames import ( AudioRawFrame, ImageRawFrame, SpriteFrame, Frame, LLMMessagesFrame, TTSStoppedFrame ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from runner import configure from loguru import logger from dotenv import load_dotenv load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") sprites = [] script_dir = os.path.dirname(__file__) for i in range(1, 26): # Build the full path to the image file full_path = os.path.join(script_dir, f"assets/robot0{i}.png") # Get the filename without the extension to use as the dictionary key # Open the image and convert it to bytes with Image.open(full_path) as img: sprites.append(ImageRawFrame(image=img.tobytes(), size=img.size, format=img.format)) flipped = sprites[::-1] sprites.extend(flipped) # When the bot isn't talking, show a static image of the cat listening quiet_frame = sprites[0] talking_frame = SpriteFrame(images=sprites) class TalkingAnimation(FrameProcessor): """ This class starts a talking animation when it receives an first AudioFrame, and then returns to a "quiet" sprite when it sees a LLMResponseEndFrame. """ def __init__(self): super().__init__() self._is_talking = False async def process_frame(self, frame: Frame, direction: FrameDirection): if isinstance(frame, AudioRawFrame): if not self._is_talking: await self.push_frame(talking_frame) self._is_talking = True elif isinstance(frame, TTSStoppedFrame): await self.push_frame(quiet_frame) self._is_talking = False await self.push_frame(frame) async def main(room_url: str, token): async with aiohttp.ClientSession() as session: transport = DailyTransport( room_url, token, "Chatbot", DailyParams( audio_out_enabled=True, camera_out_enabled=True, camera_out_width=1024, camera_out_height=576, transcription_enabled=True, vad_enabled=True ) ) tts = ElevenLabsTTSService( aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="pNInz6obpgDQGcFmaJgB", ) llm = OpenAILLMService( api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4-turbo-preview") messages = [ { "role": "system", "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.", }, ] user_response = LLMUserResponseAggregator() ta = TalkingAnimation() pipeline = Pipeline([transport.input(), user_response, llm, tts, ta, transport.output()]) task = PipelineTask(pipeline) await task.queue_frame(quiet_frame) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): transport.capture_participant_transcription(participant["id"]) await task.queue_frames([LLMMessagesFrame(messages)]) runner = PipelineRunner() await runner.run(task) if __name__ == "__main__": (url, token) = configure() asyncio.run(main(url, token))