wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

examples: fix moondream-chatbot

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2024-05-13 15:28:11 -07:00
parent fdfcfd1d5e
commit 22cd1ac5f2
5 changed files with 82 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/moondream-chatbot/README.md
View File

@@ -10,8 +10,8 @@ This app connects you to a chatbot powered by GPT-4, complete with animations ge
## Get started ## Get started
```python ```python
python3 -m venv env python3 -m venv .venv
source env/bin/activate source .venv/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
cp env.example .env # and add your credentials cp env.example .env # and add your credentials

156
examples/moondream-chatbot/bot.py
View File

@@ -1,43 +1,46 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
import sys
from PIL import Image from PIL import Image
from typing import AsyncGenerator from typing import AsyncGenerator
from dailyai.pipeline.aggregators import ( from pipecat.frames.frames import (
LLMUserResponseAggregator, ImageRawFrame,
ParallelPipeline,
VisionImageFrameAggregator,
SentenceAggregator
)
from dailyai.pipeline.frames import (
ImageFrame,
SpriteFrame, SpriteFrame,
Frame, Frame,
LLMMessagesFrame, LLMMessagesFrame,
AudioFrame, AudioRawFrame,
PipelineStartedFrame, TTSStoppedFrame,
TTSEndFrame,
TextFrame, TextFrame,
UserImageFrame, UserImageRawFrame,
UserImageRequestFrame, UserImageRequestFrame,
) )
from dailyai.services.moondream_ai_service import MoondreamService
from dailyai.pipeline.pipeline import FrameProcessor, Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from dailyai.transports.daily_transport import DailyTransport from pipecat.pipeline.pipeline import Pipeline
from dailyai.services.open_ai_services import OpenAILLMService from pipecat.pipeline.runner import PipelineRunner
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator
from pipecat.processors.aggregators.sentence import SentenceAggregator
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.moondream import MoondreamService
from pipecat.services.openai import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.vad.silero import SileroVAD
from runner import configure from runner import configure
from loguru import logger
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv(override=True) load_dotenv(override=True)
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") logger.remove(0)
logger = logging.getLogger("dailyai") logger.add(sys.stderr, level="DEBUG")
logger.setLevel(logging.DEBUG)
user_request_answer = "Let me take a look." user_request_answer = "Let me take a look."
@@ -51,13 +54,13 @@ for i in range(1, 26):
# Get the filename without the extension to use as the dictionary key # Get the filename without the extension to use as the dictionary key
# Open the image and convert it to bytes # Open the image and convert it to bytes
with Image.open(full_path) as img: with Image.open(full_path) as img:
sprites.append(img.tobytes()) sprites.append(ImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
flipped = sprites[::-1] flipped = sprites[::-1]
sprites.extend(flipped) sprites.extend(flipped)
# When the bot isn't talking, show a static image of the cat listening # When the bot isn't talking, show a static image of the cat listening
quiet_frame = ImageFrame(sprites[0], (1024, 576)) quiet_frame = sprites[0]
talking_frame = SpriteFrame(images=sprites) talking_frame = SpriteFrame(images=sprites)
@@ -71,37 +74,18 @@ class TalkingAnimation(FrameProcessor):
super().__init__() super().__init__()
self._is_talking = False self._is_talking = False
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, AudioFrame): if isinstance(frame, AudioRawFrame):
if not self._is_talking: if not self._is_talking:
yield talking_frame await self.push_frame(talking_frame)
yield frame
self._is_talking = True self._is_talking = True
else: elif isinstance(frame, TTSStoppedFrame):
yield frame await self.push_frame(quiet_frame)
elif isinstance(frame, TTSEndFrame):
yield quiet_frame
yield frame
self._is_talking = False self._is_talking = False
else: await self.push_frame(frame)
yield frame
class AnimationInitializer(FrameProcessor):
def __init__(self):
super().__init__()
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, PipelineStartedFrame):
yield quiet_frame
yield frame
else:
yield frame
class UserImageRequester(FrameProcessor): class UserImageRequester(FrameProcessor):
participant_id: str | None
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.participant_id = None self.participant_id = None
@@ -109,33 +93,32 @@ class UserImageRequester(FrameProcessor):
def set_participant_id(self, participant_id: str): def set_participant_id(self, participant_id: str):
self.participant_id = participant_id self.participant_id = participant_id
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame, direction: FrameDirection):
if self.participant_id and isinstance(frame, TextFrame): if self.participant_id and isinstance(frame, TextFrame):
if frame.text == user_request_answer: if frame.text == user_request_answer:
yield UserImageRequestFrame(self.participant_id) await self.push_frame(UserImageRequestFrame(self.participant_id), FrameDirection.UPSTREAM)
yield TextFrame("Describe the image in a short sentence.") await self.push_frame(TextFrame("Describe the image in a short sentence."))
elif isinstance(frame, UserImageFrame): elif isinstance(frame, UserImageRawFrame):
yield frame await self.push_frame(frame)
class TextFilterProcessor(FrameProcessor): class TextFilterProcessor(FrameProcessor):
text: str
def __init__(self, text: str): def __init__(self, text: str):
super().__init__()
self.text = text self.text = text
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
if frame.text != self.text: if frame.text != self.text:
yield frame await self.push_frame(frame)
else: else:
yield frame await self.push_frame(frame)
class ImageFilterProcessor(FrameProcessor): class ImageFilterProcessor(FrameProcessor):
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame, direction: FrameDirection):
if not isinstance(frame, ImageFrame): if not isinstance(frame, ImageRawFrame):
yield frame await self.push_frame(frame)
async def main(room_url: str, token): async def main(room_url: str, token):
@@ -144,17 +127,18 @@ async def main(room_url: str, token):
room_url, room_url,
token, token,
"Chatbot", "Chatbot",
duration_minutes=5, DailyParams(
start_transcription=True, audio_in_enabled=True,
mic_enabled=True, audio_out_enabled=True,
mic_sample_rate=16000, camera_out_enabled=True,
camera_enabled=True, camera_out_width=1024,
camera_width=1024, camera_out_height=576,
camera_height=576, transcription_enabled=True
vad_enabled=True, )
video_rendering_enabled=True
) )
vad = SileroVAD()
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
@@ -166,11 +150,11 @@ async def main(room_url: str, token):
model="gpt-4-turbo-preview") model="gpt-4-turbo-preview")
ta = TalkingAnimation() ta = TalkingAnimation()
ai = AnimationInitializer()
sa = SentenceAggregator() sa = SentenceAggregator()
ir = UserImageRequester() ir = UserImageRequester()
va = VisionImageFrameAggregator() va = VisionImageFrameAggregator()
# If you run into weird description, try with use_cpu=True # If you run into weird description, try with use_cpu=True
moondream = MoondreamService() moondream = MoondreamService()
@@ -186,23 +170,25 @@ async def main(room_url: str, token):
ura = LLMUserResponseAggregator(messages) ura = LLMUserResponseAggregator(messages)
pipeline = Pipeline([ pipeline = Pipeline([transport.input(), vad, ura, llm,
ai, ura, llm, ParallelPipeline( ParallelPipeline(
[[sa, ir, va, moondream], [tf, imgf]] [sa, ir, va, moondream],
), [tf, imgf]),
tts, ta tts, ta, transport.output()])
])
@transport.event_handler("on_first_other_participant_joined") task = PipelineTask(pipeline)
async def on_first_other_participant_joined(transport, participant): await task.queue_frame(quiet_frame)
transport.render_participant_video(participant["id"], framerate=0)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
transport.capture_participant_transcription(participant["id"])
transport.capture_participant_video(participant["id"], framerate=0)
ir.set_participant_id(participant["id"]) ir.set_participant_id(participant["id"])
await pipeline.queue_frames([LLMMessagesFrame(messages)]) await task.queue_frames([LLMMessagesFrame(messages)])
transport.transcription_settings["extra"]["endpointing"] = True runner = PipelineRunner()
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(pipeline)) await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":

2
examples/moondream-chatbot/requirements.txt
View File

@@ -2,4 +2,4 @@ python-dotenv
requests requests
fastapi[all] fastapi[all]
uvicorn uvicorn
dailyai[daily,moondream,openai,silero] pipecat-ai[daily,moondream,openai,silero]

2
examples/moondream-chatbot/server.py
View File

@@ -7,7 +7,7 @@ from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, RedirectResponse from fastapi.responses import JSONResponse, RedirectResponse
from utils.daily_helpers import create_room as _create_room, get_token, get_name_from_url from utils.daily_helpers import create_room as _create_room, get_token
MAX_BOTS_PER_ROOM = 1 MAX_BOTS_PER_ROOM = 1

10
examples/simple-chatbot/bot.py
View File

@@ -21,6 +21,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.openai import OpenAILLMService from pipecat.services.openai import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.vad.silero import SileroVAD
from runner import configure from runner import configure
@@ -81,15 +82,17 @@ async def main(room_url: str, token):
token, token,
"Chatbot", "Chatbot",
DailyParams( DailyParams(
audio_in_enabled=True,
audio_out_enabled=True, audio_out_enabled=True,
camera_out_enabled=True, camera_out_enabled=True,
camera_out_width=1024, camera_out_width=1024,
camera_out_height=576, camera_out_height=576,
transcription_enabled=True, transcription_enabled=True
vad_enabled=True
) )
) )
vad = SileroVAD()
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
@@ -111,7 +114,8 @@ async def main(room_url: str, token):
ta = TalkingAnimation() ta = TalkingAnimation()
pipeline = Pipeline([transport.input(), user_response, llm, tts, ta, transport.output()]) pipeline = Pipeline([transport.input(), vad, user_response,
llm, tts, ta, transport.output()])
task = PipelineTask(pipeline) task = PipelineTask(pipeline)
await task.queue_frame(quiet_frame) await task.queue_frame(quiet_frame)