processors(realtime-ai): add support making TTS to speak

src/pipecat/frames/frames.py

@@ -177,6 +177,15 @@ class LLMMessagesUpdateFrame(DataFrame):
     messages: List[dict]
 
 
+@dataclass
+class TTSSpeakFrame(DataFrame):
+    """A frame that contains a text that should be spoken by the TTS in the
+    pipeline (if any).
+
+    """
+    text: str
+
+
 @dataclass
 class TransportMessageFrame(DataFrame):
     message: Any

src/pipecat/processors/frameworks/realtimeai.py

@@ -16,6 +16,7 @@ from pipecat.frames.frames import (
     LLMMessagesUpdateFrame,
     LLMModelUpdateFrame,
     StartFrame,
+    TTSSpeakFrame,
     TTSVoiceUpdateFrame,
     TransportMessageFrame)
 from pipecat.pipeline.pipeline import Pipeline
@@ -57,9 +58,19 @@ class RealtimeAISetup(BaseModel):
     config: Optional[RealtimeAIConfig] = None
 
 
+class RealtimeAILLMMessageData(BaseModel):
+    messages: List[dict]
+
+
+class RealtimeAITTSMessageData(BaseModel):
+    text: str
+
+
 class RealtimeAIMessageData(BaseModel):
     setup: Optional[RealtimeAISetup] = None
     config: Optional[RealtimeAIConfig] = None
+    llm: Optional[RealtimeAILLMMessageData] = None
+    tts: Optional[RealtimeAITTSMessageData] = None
 
 
 class RealtimeAIMessage(BaseModel):
@@ -135,9 +146,11 @@ class RealtimeAIProcessor(FrameProcessor):
                 case "llm-get-context":
                     await self._handle_llm_get_context()
                 case "llm-append-context":
-                    await self._handle_llm_append_context(message.data.config)
+                    await self._handle_llm_append_context(message.data.llm)
                 case "llm-update-context":
-                    await self._handle_llm_update_context(message.data.config)
+                    await self._handle_llm_update_context(message.data.llm)
+                case "tts-speak":
+                    await self._handle_tts_speak(message.data.tts)
         except ValidationError as e:
             await self._send_response(message.type, False, f"invalid message: {e}")
 
@@ -206,14 +219,19 @@ class RealtimeAIProcessor(FrameProcessor):
         message = TransportMessageFrame(message=response.model_dump(exclude_none=True))
         await self.push_frame(message)
 
-    async def _handle_llm_append_context(self, config: RealtimeAIConfig):
-        if config.llm and config.llm.messages:
-            frame = LLMMessagesAppendFrame(config.llm.messages)
+    async def _handle_llm_append_context(self, data: RealtimeAILLMMessageData):
+        if data and data.messages:
+            frame = LLMMessagesAppendFrame(data.messages)
             await self.push_frame(frame)
 
-    async def _handle_llm_update_context(self, config: RealtimeAIConfig):
-        if config.llm and config.llm.messages:
-            frame = LLMMessagesUpdateFrame(config.llm.messages)
+    async def _handle_llm_update_context(self, data: RealtimeAILLMMessageData):
+        if data and data.messages:
+            frame = LLMMessagesUpdateFrame(data.messages)
+            await self.push_frame(frame)
+
+    async def _handle_tts_speak(self, data: RealtimeAITTSMessageData):
+        if data and data.text:
+            frame = TTSSpeakFrame(text=data.text)
             await self.push_frame(frame)
 
     async def _send_response(self, type: str, success: bool, error: str | None = None):

src/pipecat/services/ai_services.py

@@ -19,6 +19,7 @@ from pipecat.frames.frames import (
     LLMFullResponseEndFrame,
     StartFrame,
     StartInterruptionFrame,
+    TTSSpeakFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
     TTSVoiceUpdateFrame,
@@ -178,7 +179,7 @@ class TTSService(AIService):
         if text:
             await self._push_tts_frames(text)
 
-    async def _push_tts_frames(self, text: str):
+    async def _push_tts_frames(self, text: str, text_passthrough: bool = True):
         text = text.strip()
         if not text:
             return
@@ -209,6 +210,8 @@ class TTSService(AIService):
                     await self.push_frame(frame, direction)
             else:
                 await self.push_frame(frame, direction)
+        elif isinstance(frame, TTSSpeakFrame):
+            await self._push_tts_frames(frame.text, False)
         elif isinstance(frame, TTSVoiceUpdateFrame):
             await self.set_voice(frame.voice)
         else: