From 22cd1ac5f2e92b462782bedec9265c168c66fdb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 13 May 2024 15:28:11 -0700 Subject: [PATCH] examples: fix moondream-chatbot --- examples/moondream-chatbot/README.md | 4 +- examples/moondream-chatbot/bot.py | 156 +++++++++----------- examples/moondream-chatbot/requirements.txt | 2 +- examples/moondream-chatbot/server.py | 2 +- examples/simple-chatbot/bot.py | 10 +- 5 files changed, 82 insertions(+), 92 deletions(-) diff --git a/examples/moondream-chatbot/README.md b/examples/moondream-chatbot/README.md index 8fca00e58..a1ec5377a 100644 --- a/examples/moondream-chatbot/README.md +++ b/examples/moondream-chatbot/README.md @@ -10,8 +10,8 @@ This app connects you to a chatbot powered by GPT-4, complete with animations ge ## Get started ```python -python3 -m venv env -source env/bin/activate +python3 -m venv .venv +source .venv/bin/activate pip install -r requirements.txt cp env.example .env # and add your credentials diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 56d5b72da..13a815ab2 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -1,43 +1,46 @@ import asyncio - import aiohttp -import logging import os +import sys + from PIL import Image from typing import AsyncGenerator -from dailyai.pipeline.aggregators import ( - LLMUserResponseAggregator, - ParallelPipeline, - VisionImageFrameAggregator, - SentenceAggregator -) -from dailyai.pipeline.frames import ( - ImageFrame, +from pipecat.frames.frames import ( + ImageRawFrame, SpriteFrame, Frame, LLMMessagesFrame, - AudioFrame, - PipelineStartedFrame, - TTSEndFrame, + AudioRawFrame, + TTSStoppedFrame, TextFrame, - UserImageFrame, + UserImageRawFrame, UserImageRequestFrame, ) -from dailyai.services.moondream_ai_service import MoondreamService -from dailyai.pipeline.pipeline import FrameProcessor, Pipeline -from dailyai.transports.daily_transport import DailyTransport -from dailyai.services.open_ai_services import OpenAILLMService -from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService + +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator +from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.moondream import MoondreamService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.vad.silero import SileroVAD from runner import configure +from loguru import logger + from dotenv import load_dotenv load_dotenv(override=True) -logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") -logger = logging.getLogger("dailyai") -logger.setLevel(logging.DEBUG) +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") user_request_answer = "Let me take a look." @@ -51,13 +54,13 @@ for i in range(1, 26): # Get the filename without the extension to use as the dictionary key # Open the image and convert it to bytes with Image.open(full_path) as img: - sprites.append(img.tobytes()) + sprites.append(ImageRawFrame(image=img.tobytes(), size=img.size, format=img.format)) flipped = sprites[::-1] sprites.extend(flipped) # When the bot isn't talking, show a static image of the cat listening -quiet_frame = ImageFrame(sprites[0], (1024, 576)) +quiet_frame = sprites[0] talking_frame = SpriteFrame(images=sprites) @@ -71,37 +74,18 @@ class TalkingAnimation(FrameProcessor): super().__init__() self._is_talking = False - async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: - if isinstance(frame, AudioFrame): + async def process_frame(self, frame: Frame, direction: FrameDirection): + if isinstance(frame, AudioRawFrame): if not self._is_talking: - yield talking_frame - yield frame + await self.push_frame(talking_frame) self._is_talking = True - else: - yield frame - elif isinstance(frame, TTSEndFrame): - yield quiet_frame - yield frame + elif isinstance(frame, TTSStoppedFrame): + await self.push_frame(quiet_frame) self._is_talking = False - else: - yield frame - - -class AnimationInitializer(FrameProcessor): - def __init__(self): - super().__init__() - - async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: - if isinstance(frame, PipelineStartedFrame): - yield quiet_frame - yield frame - else: - yield frame + await self.push_frame(frame) class UserImageRequester(FrameProcessor): - participant_id: str | None - def __init__(self): super().__init__() self.participant_id = None @@ -109,33 +93,32 @@ class UserImageRequester(FrameProcessor): def set_participant_id(self, participant_id: str): self.participant_id = participant_id - async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: + async def process_frame(self, frame: Frame, direction: FrameDirection): if self.participant_id and isinstance(frame, TextFrame): if frame.text == user_request_answer: - yield UserImageRequestFrame(self.participant_id) - yield TextFrame("Describe the image in a short sentence.") - elif isinstance(frame, UserImageFrame): - yield frame + await self.push_frame(UserImageRequestFrame(self.participant_id), FrameDirection.UPSTREAM) + await self.push_frame(TextFrame("Describe the image in a short sentence.")) + elif isinstance(frame, UserImageRawFrame): + await self.push_frame(frame) class TextFilterProcessor(FrameProcessor): - text: str - def __init__(self, text: str): + super().__init__() self.text = text - async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: + async def process_frame(self, frame: Frame, direction: FrameDirection): if isinstance(frame, TextFrame): if frame.text != self.text: - yield frame + await self.push_frame(frame) else: - yield frame + await self.push_frame(frame) class ImageFilterProcessor(FrameProcessor): - async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: - if not isinstance(frame, ImageFrame): - yield frame + async def process_frame(self, frame: Frame, direction: FrameDirection): + if not isinstance(frame, ImageRawFrame): + await self.push_frame(frame) async def main(room_url: str, token): @@ -144,17 +127,18 @@ async def main(room_url: str, token): room_url, token, "Chatbot", - duration_minutes=5, - start_transcription=True, - mic_enabled=True, - mic_sample_rate=16000, - camera_enabled=True, - camera_width=1024, - camera_height=576, - vad_enabled=True, - video_rendering_enabled=True + DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_out_enabled=True, + camera_out_width=1024, + camera_out_height=576, + transcription_enabled=True + ) ) + vad = SileroVAD() + tts = ElevenLabsTTSService( aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), @@ -166,11 +150,11 @@ async def main(room_url: str, token): model="gpt-4-turbo-preview") ta = TalkingAnimation() - ai = AnimationInitializer() sa = SentenceAggregator() ir = UserImageRequester() va = VisionImageFrameAggregator() + # If you run into weird description, try with use_cpu=True moondream = MoondreamService() @@ -186,23 +170,25 @@ async def main(room_url: str, token): ura = LLMUserResponseAggregator(messages) - pipeline = Pipeline([ - ai, ura, llm, ParallelPipeline( - [[sa, ir, va, moondream], [tf, imgf]] - ), - tts, ta - ]) + pipeline = Pipeline([transport.input(), vad, ura, llm, + ParallelPipeline( + [sa, ir, va, moondream], + [tf, imgf]), + tts, ta, transport.output()]) - @transport.event_handler("on_first_other_participant_joined") - async def on_first_other_participant_joined(transport, participant): - transport.render_participant_video(participant["id"], framerate=0) + task = PipelineTask(pipeline) + await task.queue_frame(quiet_frame) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + transport.capture_participant_transcription(participant["id"]) + transport.capture_participant_video(participant["id"], framerate=0) ir.set_participant_id(participant["id"]) - await pipeline.queue_frames([LLMMessagesFrame(messages)]) + await task.queue_frames([LLMMessagesFrame(messages)]) - transport.transcription_settings["extra"]["endpointing"] = True - transport.transcription_settings["extra"]["punctuate"] = True + runner = PipelineRunner() - await asyncio.gather(transport.run(pipeline)) + await runner.run(task) if __name__ == "__main__": diff --git a/examples/moondream-chatbot/requirements.txt b/examples/moondream-chatbot/requirements.txt index f0dc27585..0257a03b6 100644 --- a/examples/moondream-chatbot/requirements.txt +++ b/examples/moondream-chatbot/requirements.txt @@ -2,4 +2,4 @@ python-dotenv requests fastapi[all] uvicorn -dailyai[daily,moondream,openai,silero] +pipecat-ai[daily,moondream,openai,silero] diff --git a/examples/moondream-chatbot/server.py b/examples/moondream-chatbot/server.py index fe383c0aa..c568fdc40 100644 --- a/examples/moondream-chatbot/server.py +++ b/examples/moondream-chatbot/server.py @@ -7,7 +7,7 @@ from fastapi import FastAPI, Request, HTTPException from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse -from utils.daily_helpers import create_room as _create_room, get_token, get_name_from_url +from utils.daily_helpers import create_room as _create_room, get_token MAX_BOTS_PER_ROOM = 1 diff --git a/examples/simple-chatbot/bot.py b/examples/simple-chatbot/bot.py index 95f7f8bc1..293ab2719 100644 --- a/examples/simple-chatbot/bot.py +++ b/examples/simple-chatbot/bot.py @@ -21,6 +21,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.vad.silero import SileroVAD from runner import configure @@ -81,15 +82,17 @@ async def main(room_url: str, token): token, "Chatbot", DailyParams( + audio_in_enabled=True, audio_out_enabled=True, camera_out_enabled=True, camera_out_width=1024, camera_out_height=576, - transcription_enabled=True, - vad_enabled=True + transcription_enabled=True ) ) + vad = SileroVAD() + tts = ElevenLabsTTSService( aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), @@ -111,7 +114,8 @@ async def main(room_url: str, token): ta = TalkingAnimation() - pipeline = Pipeline([transport.input(), user_response, llm, tts, ta, transport.output()]) + pipeline = Pipeline([transport.input(), vad, user_response, + llm, tts, ta, transport.output()]) task = PipelineTask(pipeline) await task.queue_frame(quiet_frame)