diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 79fd24322..09c475702 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -177,6 +177,15 @@ class LLMMessagesUpdateFrame(DataFrame): messages: List[dict] +@dataclass +class TTSSpeakFrame(DataFrame): + """A frame that contains a text that should be spoken by the TTS in the + pipeline (if any). + + """ + text: str + + @dataclass class TransportMessageFrame(DataFrame): message: Any diff --git a/src/pipecat/processors/frameworks/realtimeai.py b/src/pipecat/processors/frameworks/realtimeai.py index 742e85198..3f032e4e7 100644 --- a/src/pipecat/processors/frameworks/realtimeai.py +++ b/src/pipecat/processors/frameworks/realtimeai.py @@ -16,6 +16,7 @@ from pipecat.frames.frames import ( LLMMessagesUpdateFrame, LLMModelUpdateFrame, StartFrame, + TTSSpeakFrame, TTSVoiceUpdateFrame, TransportMessageFrame) from pipecat.pipeline.pipeline import Pipeline @@ -57,9 +58,19 @@ class RealtimeAISetup(BaseModel): config: Optional[RealtimeAIConfig] = None +class RealtimeAILLMMessageData(BaseModel): + messages: List[dict] + + +class RealtimeAITTSMessageData(BaseModel): + text: str + + class RealtimeAIMessageData(BaseModel): setup: Optional[RealtimeAISetup] = None config: Optional[RealtimeAIConfig] = None + llm: Optional[RealtimeAILLMMessageData] = None + tts: Optional[RealtimeAITTSMessageData] = None class RealtimeAIMessage(BaseModel): @@ -135,9 +146,11 @@ class RealtimeAIProcessor(FrameProcessor): case "llm-get-context": await self._handle_llm_get_context() case "llm-append-context": - await self._handle_llm_append_context(message.data.config) + await self._handle_llm_append_context(message.data.llm) case "llm-update-context": - await self._handle_llm_update_context(message.data.config) + await self._handle_llm_update_context(message.data.llm) + case "tts-speak": + await self._handle_tts_speak(message.data.tts) except ValidationError as e: await self._send_response(message.type, False, f"invalid message: {e}") @@ -206,14 +219,19 @@ class RealtimeAIProcessor(FrameProcessor): message = TransportMessageFrame(message=response.model_dump(exclude_none=True)) await self.push_frame(message) - async def _handle_llm_append_context(self, config: RealtimeAIConfig): - if config.llm and config.llm.messages: - frame = LLMMessagesAppendFrame(config.llm.messages) + async def _handle_llm_append_context(self, data: RealtimeAILLMMessageData): + if data and data.messages: + frame = LLMMessagesAppendFrame(data.messages) await self.push_frame(frame) - async def _handle_llm_update_context(self, config: RealtimeAIConfig): - if config.llm and config.llm.messages: - frame = LLMMessagesUpdateFrame(config.llm.messages) + async def _handle_llm_update_context(self, data: RealtimeAILLMMessageData): + if data and data.messages: + frame = LLMMessagesUpdateFrame(data.messages) + await self.push_frame(frame) + + async def _handle_tts_speak(self, data: RealtimeAITTSMessageData): + if data and data.text: + frame = TTSSpeakFrame(text=data.text) await self.push_frame(frame) async def _send_response(self, type: str, success: bool, error: str | None = None): diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index ec89b9eba..bc00accdf 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -19,6 +19,7 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, + TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, TTSVoiceUpdateFrame, @@ -178,7 +179,7 @@ class TTSService(AIService): if text: await self._push_tts_frames(text) - async def _push_tts_frames(self, text: str): + async def _push_tts_frames(self, text: str, text_passthrough: bool = True): text = text.strip() if not text: return @@ -209,6 +210,8 @@ class TTSService(AIService): await self.push_frame(frame, direction) else: await self.push_frame(frame, direction) + elif isinstance(frame, TTSSpeakFrame): + await self._push_tts_frames(frame.text, False) elif isinstance(frame, TTSVoiceUpdateFrame): await self.set_voice(frame.voice) else: