diff --git a/CHANGELOG.md b/CHANGELOG.md index f5d2d40df..62637b317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.0.43] - 2024-10-03 +## [Unreleased] + +### Added + +- Added new `RTVIUserLLMTextProcessor`. This processor will send an RTVI + `user-llm-text` message with the user content's that was sent to the LLM. ### Changed diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 7a6054c3c..8a7763342 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -35,7 +35,10 @@ from pipecat.frames.frames import ( FunctionCallResultFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from loguru import logger @@ -297,16 +300,6 @@ class RTVIBotAudioMessage(BaseModel): data: RTVIAudioMessageData -class RTVIBotTranscriptionMessageData(BaseModel): - text: str - - -class RTVIBotTranscriptionMessage(BaseModel): - label: Literal["rtvi-ai"] = "rtvi-ai" - type: Literal["bot-transcription"] = "bot-transcription" - data: RTVIBotTranscriptionMessageData - - class RTVIUserTranscriptionMessageData(BaseModel): text: str user_id: str @@ -320,6 +313,12 @@ class RTVIUserTranscriptionMessage(BaseModel): data: RTVIUserTranscriptionMessageData +class RTVIUserLLMTextMessage(BaseModel): + label: Literal["rtvi-ai"] = "rtvi-ai" + type: Literal["user-llm-text"] = "user-llm-text" + data: RTVITextMessageData + + class RTVIUserStartedSpeakingMessage(BaseModel): label: Literal["rtvi-ai"] = "rtvi-ai" type: Literal["user-started-speaking"] = "user-started-speaking" @@ -422,6 +421,27 @@ class RTVIUserTranscriptionProcessor(RTVIFrameProcessor): await self._push_transport_message(message) +class RTVIUserLLMTextProcessor(RTVIFrameProcessor): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + await self.push_frame(frame, direction) + + if isinstance(frame, OpenAILLMContextFrame): + await self._handle_context(frame) + + async def _handle_context(self, frame: OpenAILLMContextFrame): + messages = frame.context.messages + if len(messages) > 0: + message = messages[-1] + if message["role"] == "user": + message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=message["content"])) + await self._push_transport_message(message) + + class RTVIBotLLMProcessor(RTVIFrameProcessor): def __init__(self, **kwargs): super().__init__(**kwargs)