examples: fix translation-chatbot

This commit is contained in:
Aleix Conchillo Flaqué
2024-05-13 16:13:01 -07:00
parent 1b21867a6f
commit 12ff6d08fe
5 changed files with 43 additions and 48 deletions

View File

@@ -4,7 +4,6 @@ import os
import sys import sys
from PIL import Image from PIL import Image
from typing import AsyncGenerator
from pipecat.frames.frames import ( from pipecat.frames.frames import (
ImageRawFrame, ImageRawFrame,

View File

@@ -1,35 +1,29 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
from typing import AsyncGenerator import sys
from dailyai.pipeline.aggregators import ( from pipecat.frames.frames import Frame, LLMMessagesFrame, TextFrame, TransportMessageFrame
SentenceAggregator, from pipecat.pipeline.pipeline import Pipeline
) from pipecat.pipeline.runner import PipelineRunner
from dailyai.pipeline.frames import ( from pipecat.pipeline.task import PipelineTask
Frame, from pipecat.processors.aggregators.llm_response import LLMFullResponseAggregator
LLMMessagesFrame, from pipecat.processors.aggregators.sentence import SentenceAggregator
TextFrame, from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
SendAppMessageFrame, from pipecat.services.azure import AzureTTSService
) from pipecat.services.openai import OpenAILLMService
from dailyai.pipeline.frame_processor import FrameProcessor from pipecat.transports.services.daily import DailyParams, DailyTransport, DailyTransportMessageFrame
from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.azure_ai_services import AzureTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.pipeline.aggregators import LLMFullResponseAggregator
from runner import configure from runner import configure
from loguru import logger
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv(override=True) load_dotenv(override=True)
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") logger.remove(0)
logger = logging.getLogger("dailyai") logger.add(sys.stderr, level="DEBUG")
logger.setLevel(logging.DEBUG)
""" """
This example looks a bit different than the chatbot example, because it isn't waiting on the user to stop talking to start translating. This example looks a bit different than the chatbot example, because it isn't waiting on the user to stop talking to start translating.
@@ -40,10 +34,11 @@ It also isn't saving what the user or bot says into the context object for use i
# We need to use a custom service here to yield LLM frames without saving # We need to use a custom service here to yield LLM frames without saving
# any context # any context
class TranslationProcessor(FrameProcessor): class TranslationProcessor(FrameProcessor):
def __init__(self, language): def __init__(self, language):
self._language = language self._language = language
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
context = [ context = [
{ {
@@ -52,25 +47,24 @@ class TranslationProcessor(FrameProcessor):
}, },
{"role": "user", "content": frame.text}, {"role": "user", "content": frame.text},
] ]
yield LLMMessagesFrame(context) await self.push_frame(LLMMessagesFrame(context))
else: else:
yield frame await self.push_frame(frame)
class TranslationSubtitles(FrameProcessor): class TranslationSubtitles(FrameProcessor):
def __init__(self, language): def __init__(self, language):
self._language = language self._language = language
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
app_message = { message = {
"language": self._language, "language": self._language,
"text": frame.text "text": frame.text
} }
yield SendAppMessageFrame(app_message, None) await self.push_frame(DailyTransportMessageFrame(message))
yield frame
else: await self.push_frame(frame)
yield frame
async def main(room_url: str, token): async def main(room_url: str, token):
@@ -79,29 +73,34 @@ async def main(room_url: str, token):
room_url, room_url,
token, token,
"Translator", "Translator",
duration_minutes=5, DailyParams(
start_transcription=True, audio_out_enabled=True,
mic_enabled=True, transcription_enabled=True,
mic_sample_rate=16000, )
camera_enabled=False,
) )
tts = AzureTTSService( tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"), api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"), region=os.getenv("AZURE_SPEECH_REGION"),
voice="es-ES-AlvaroNeural", voice="es-ES-AlvaroNeural",
) )
llm = OpenAILLMService( llm = OpenAILLMService(
api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4-turbo-preview" api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4-turbo-preview"
) )
sa = SentenceAggregator() sa = SentenceAggregator()
tp = TranslationProcessor("Spanish") tp = TranslationProcessor("Spanish")
lfra = LLMFullResponseAggregator() lfra = LLMFullResponseAggregator()
ts = TranslationSubtitles("spanish") ts = TranslationSubtitles("spanish")
pipeline = Pipeline([sa, tp, llm, lfra, ts, tts])
transport.transcription_settings["extra"]["endpointing"] = True pipeline = Pipeline([transport.input(), sa, tp, llm, lfra, ts, tts, transport.output()])
transport.transcription_settings["extra"]["punctuate"] = True
await transport.run(pipeline) task = PipelineTask(pipeline)
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -1,4 +1,4 @@
python-dotenv python-dotenv
requests requests
fastapi[all] fastapi[all]
dailyai[daily,openai,azure] pipecat-ai[daily,openai,azure]

View File

@@ -2,15 +2,12 @@ import os
import argparse import argparse
import subprocess import subprocess
import atexit import atexit
from pathlib import Path
from typing import Optional
from fastapi import FastAPI, Request, HTTPException from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles from fastapi.responses import JSONResponse, RedirectResponse
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
from utils.daily_helpers import create_room as _create_room, get_token, get_name_from_url from utils.daily_helpers import create_room as _create_room, get_token
MAX_BOTS_PER_ROOM = 1 MAX_BOTS_PER_ROOM = 1

View File

@@ -9,7 +9,7 @@ class FrameSerializer:
@abstractmethod @abstractmethod
def serialize(self, frame: Frame) -> bytes: def serialize(self, frame: Frame) -> bytes:
raise NotImplementedError() raise NotImplementedError
@abstractmethod @abstractmethod
def deserialize(self, data: bytes) -> Frame: def deserialize(self, data: bytes) -> Frame: