wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

processors(rtvi): add support for "tts-text" messages

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2024-07-22 11:40:17 -07:00
parent 43932220f7
commit 9233bb490c
1 changed files with 32 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -34,7 +34,7 @@ from pipecat.processors.aggregators.llm_response import (
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.ai_services import AIService from pipecat.services.ai_services import AIService
from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.cartesia import CartesiaTTSService
from pipecat.services.openai import OpenAILLMService, OpenAILLMContext, OpenAILLMContextFrame from pipecat.services.openai import OpenAILLMService, OpenAILLMContext
from pipecat.transports.base_transport import BaseTransport from pipecat.transports.base_transport import BaseTransport
DEFAULT_MESSAGES = [ DEFAULT_MESSAGES = [
@@ -122,6 +122,16 @@ class RTVILLMContextMessage(BaseModel):
data: RTVILLMContextMessageData data: RTVILLMContextMessageData
class RTVITTSTextMessageData(BaseModel):
text: str
class RTVITTSTextMessage(BaseModel):
label: Literal["rtvi"] = "rtvi"
type: Literal["tts-text"] = "tts-text"
data: RTVITTSTextMessageData
class RTVIBotReady(BaseModel): class RTVIBotReady(BaseModel):
label: Literal["rtvi"] = "rtvi" label: Literal["rtvi"] = "rtvi"
type: Literal["bot-ready"] = "bot-ready" type: Literal["bot-ready"] = "bot-ready"
@@ -157,6 +167,7 @@ class RTVIJSONCompletion(BaseModel):
class FunctionCaller(FrameProcessor): class FunctionCaller(FrameProcessor):
def __init__(self, context): def __init__(self, context):
super().__init__() super().__init__()
self._checking = False self._checking = False
@@ -191,6 +202,8 @@ class FunctionCaller(FrameProcessor):
await self._start_callbacks[function_name](self) await self._start_callbacks[function_name](self)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, LLMFullResponseStartFrame): if isinstance(frame, LLMFullResponseStartFrame):
self._checking = True self._checking = True
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -236,6 +249,21 @@ class FunctionCaller(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
class RTVITTSTextProcessor(FrameProcessor):
def __init__(self):
super().__init__()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, TextFrame):
message = RTVITTSTextMessage(data=RTVITTSTextMessageData(text=frame.text))
await self.push_frame(TransportMessageFrame(message=message.model_dump(exclude_none=True)))
class RTVIProcessor(FrameProcessor): class RTVIProcessor(FrameProcessor):
def __init__( def __init__(
@@ -404,11 +432,14 @@ class RTVIProcessor(FrameProcessor):
context = OpenAILLMContext(messages=messages) context = OpenAILLMContext(messages=messages)
self._fc = FunctionCaller(context) self._fc = FunctionCaller(context)
self._tts_text = RTVITTSTextProcessor()
pipeline = Pipeline([ pipeline = Pipeline([
self._tma_in, self._tma_in,
self._llm, self._llm,
self._fc, self._fc,
self._tts, self._tts,
self._tts_text,
self._tma_out, self._tma_out,
self._transport.output(), self._transport.output(),
]) ])