wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

examples: fix simple-chatbot

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2024-05-13 13:19:11 -07:00
parent d380b02a44
commit ed31c7924e
6 changed files with 63 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/simple-chatbot/README.md
View File

@@ -13,8 +13,8 @@ And a quick video walkthrough of the code: https://www.loom.com/share/13df196716
## Get started ## Get started
```python ```python
python3 -m venv env python3 -m venv .venv
source env/bin/activate source .venv/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
cp env.example .env # and add your credentials cp env.example .env # and add your credentials

119
examples/simple-chatbot/bot.py
View File

@@ -1,37 +1,36 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
from PIL import Image import sys
from typing import AsyncGenerator
from dailyai.pipeline.aggregators import ( from PIL import Image
LLMAssistantResponseAggregator,
LLMUserResponseAggregator, from pipecat.pipeline.pipeline import Pipeline
) from pipecat.pipeline.runner import PipelineRunner
from dailyai.pipeline.frames import ( from pipecat.pipeline.task import PipelineTask
ImageFrame, from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator
from pipecat.frames.frames import (
AudioRawFrame,
ImageRawFrame,
SpriteFrame, SpriteFrame,
Frame, Frame,
LLMMessagesFrame, LLMMessagesFrame,
AudioFrame, TTSStoppedFrame
PipelineStartedFrame,
TTSEndFrame,
) )
from dailyai.services.ai_services import AIService from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from dailyai.pipeline.pipeline import Pipeline from pipecat.services.elevenlabs import ElevenLabsTTSService
from dailyai.transports.daily_transport import DailyTransport from pipecat.services.openai import OpenAILLMService
from dailyai.services.open_ai_services import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from runner import configure from runner import configure
from loguru import logger
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv(override=True) load_dotenv(override=True)
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") logger.remove(0)
logger = logging.getLogger("dailyai") logger.add(sys.stderr, level="DEBUG")
logger.setLevel(logging.DEBUG)
sprites = [] sprites = []
@@ -43,17 +42,17 @@ for i in range(1, 26):
# Get the filename without the extension to use as the dictionary key # Get the filename without the extension to use as the dictionary key
# Open the image and convert it to bytes # Open the image and convert it to bytes
with Image.open(full_path) as img: with Image.open(full_path) as img:
sprites.append(img.tobytes()) sprites.append(ImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
flipped = sprites[::-1] flipped = sprites[::-1]
sprites.extend(flipped) sprites.extend(flipped)
# When the bot isn't talking, show a static image of the cat listening # When the bot isn't talking, show a static image of the cat listening
quiet_frame = ImageFrame(sprites[0], (1024, 576)) quiet_frame = sprites[0]
talking_frame = SpriteFrame(images=sprites) talking_frame = SpriteFrame(images=sprites)
class TalkingAnimation(AIService): class TalkingAnimation(FrameProcessor):
""" """
This class starts a talking animation when it receives an first AudioFrame, This class starts a talking animation when it receives an first AudioFrame,
and then returns to a "quiet" sprite when it sees a LLMResponseEndFrame. and then returns to a "quiet" sprite when it sees a LLMResponseEndFrame.
@@ -63,32 +62,16 @@ class TalkingAnimation(AIService):
super().__init__() super().__init__()
self._is_talking = False self._is_talking = False
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, AudioFrame): if isinstance(frame, AudioRawFrame):
if not self._is_talking: if not self._is_talking:
yield talking_frame await self.push_frame(talking_frame)
yield frame
self._is_talking = True self._is_talking = True
else: elif isinstance(frame, TTSStoppedFrame):
yield frame await self.push_frame(quiet_frame)
elif isinstance(frame, TTSEndFrame):
yield quiet_frame
yield frame
self._is_talking = False self._is_talking = False
else:
yield frame
await self.push_frame(frame)
class AnimationInitializer(AIService):
def __init__(self):
super().__init__()
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, PipelineStartedFrame):
yield quiet_frame
yield frame
else:
yield frame
async def main(room_url: str, token): async def main(room_url: str, token):
@@ -97,14 +80,14 @@ async def main(room_url: str, token):
room_url, room_url,
token, token,
"Chatbot", "Chatbot",
duration_minutes=5, DailyParams(
start_transcription=True, audio_out_enabled=True,
mic_enabled=True, camera_out_enabled=True,
mic_sample_rate=16000, camera_out_width=1024,
camera_enabled=True, camera_out_height=576,
camera_width=1024, transcription_enabled=True,
camera_height=576, vad_enabled=True
vad_enabled=True, )
) )
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
@@ -117,9 +100,6 @@ async def main(room_url: str, token):
api_key=os.getenv("OPENAI_API_KEY"), api_key=os.getenv("OPENAI_API_KEY"),
model="gpt-4-turbo-preview") model="gpt-4-turbo-preview")
ta = TalkingAnimation()
ai = AnimationInitializer()
pipeline = Pipeline([ai, llm, tts, ta])
messages = [ messages = [
{ {
"role": "system", "role": "system",
@@ -127,22 +107,23 @@ async def main(room_url: str, token):
}, },
] ]
@transport.event_handler("on_first_other_participant_joined") user_response = LLMUserResponseAggregator()
async def on_first_other_participant_joined(transport, participant):
print(f"!!! in here, pipeline.source is {pipeline.source}")
await pipeline.queue_frames([LLMMessagesFrame(messages)])
async def run_conversation(): ta = TalkingAnimation()
await transport.run_interruptible_pipeline( pipeline = Pipeline([transport.input(), user_response, llm, tts, ta, transport.output()])
pipeline,
post_processor=LLMAssistantResponseAggregator(messages),
pre_processor=LLMUserResponseAggregator(messages),
)
transport.transcription_settings["extra"]["endpointing"] = True task = PipelineTask(pipeline)
transport.transcription_settings["extra"]["punctuate"] = True await task.queue_frame(quiet_frame)
await asyncio.gather(transport.run(), run_conversation())
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
transport.capture_participant_transcription(participant["id"])
await task.queue_frames([LLMMessagesFrame(messages)])
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":

2
examples/simple-chatbot/requirements.txt
View File

@@ -2,4 +2,4 @@ python-dotenv
requests requests
fastapi[all] fastapi[all]
uvicorn uvicorn
dailyai[daily,openai] pipecat-ai[daily,openai]

7
examples/simple-chatbot/server.py
View File

@@ -2,15 +2,12 @@ import os
import argparse import argparse
import subprocess import subprocess
import atexit import atexit
from pathlib import Path
from typing import Optional
from fastapi import FastAPI, Request, HTTPException from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles from fastapi.responses import JSONResponse, RedirectResponse
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
from utils.daily_helpers import create_room as _create_room, get_token, get_name_from_url from utils.daily_helpers import create_room as _create_room, get_token
MAX_BOTS_PER_ROOM = 1 MAX_BOTS_PER_ROOM = 1

11
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
from typing import List
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.frames.frames import ( from pipecat.frames.frames import (
Frame, Frame,
@@ -22,7 +24,7 @@ class LLMResponseAggregator(FrameProcessor):
def __init__( def __init__(
self, self,
*, *,
messages: list[dict] | None, messages: List[dict],
role: str, role: str,
start_frame, start_frame,
end_frame, end_frame,
@@ -65,9 +67,6 @@ class LLMResponseAggregator(FrameProcessor):
# and T2 would be dropped. # and T2 would be dropped.
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
if not self._messages:
return
send_aggregation = False send_aggregation = False
if isinstance(frame, self._start_frame): if isinstance(frame, self._start_frame):
@@ -116,7 +115,7 @@ class LLMResponseAggregator(FrameProcessor):
class LLMAssistantResponseAggregator(LLMResponseAggregator): class LLMAssistantResponseAggregator(LLMResponseAggregator):
def __init__(self, messages: list[dict]): def __init__(self, messages: List[dict] = []):
super().__init__( super().__init__(
messages=messages, messages=messages,
role="assistant", role="assistant",
@@ -127,7 +126,7 @@ class LLMAssistantResponseAggregator(LLMResponseAggregator):
class LLMUserResponseAggregator(LLMResponseAggregator): class LLMUserResponseAggregator(LLMResponseAggregator):
def __init__(self, messages: list[dict]): def __init__(self, messages: List[dict] = []):
super().__init__( super().__init__(
messages=messages, messages=messages,
role="user", role="user",

6
src/pipecat/transports/base_input.py
View File

@@ -33,7 +33,7 @@ class BaseInputTransport(FrameProcessor):
self._running = True self._running = True
# Start media threads. # Start media threads.
if self._params.audio_in_enabled: if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_queue = queue.Queue() self._audio_in_queue = queue.Queue()
self._audio_in_thread = threading.Thread(target=self._audio_in_thread_handler) self._audio_in_thread = threading.Thread(target=self._audio_in_thread_handler)
self._audio_out_thread = threading.Thread(target=self._audio_out_thread_handler) self._audio_out_thread = threading.Thread(target=self._audio_out_thread_handler)
@@ -41,7 +41,7 @@ class BaseInputTransport(FrameProcessor):
self._stopped_event = asyncio.Event() self._stopped_event = asyncio.Event()
async def start(self): async def start(self):
if self._params.audio_in_enabled: if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_thread.start() self._audio_in_thread.start()
self._audio_out_thread.start() self._audio_out_thread.start()
@@ -62,7 +62,7 @@ class BaseInputTransport(FrameProcessor):
# #
async def cleanup(self): async def cleanup(self):
if self._params.audio_in_enabled: if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_thread.join() self._audio_in_thread.join()
self._audio_out_thread.join() self._audio_out_thread.join()