diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 321f8514b..6e5fa744f 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -177,6 +177,20 @@ class TextFrame(DataFrame): return f"{self.name}(pts: {pts}, text: [{self.text}])" +@dataclass +class LLMTextFrame(TextFrame): + """A text frame generated by LLM services.""" + + pass + + +@dataclass +class TTSTextFrame(TextFrame): + """A text frame generated by TTS services.""" + + pass + + @dataclass class TranscriptionFrame(TextFrame): """A text frame with transcription-specific data. Will be placed in the diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 96d51cb5d..36b782f80 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -34,14 +34,15 @@ from pipecat.frames.frames import ( InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, + LLMTextFrame, MetricsFrame, StartFrame, SystemFrame, - TextFrame, TranscriptionFrame, TransportMessageUrgentFrame, TTSStartedFrame, TTSStoppedFrame, + TTSTextFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -479,7 +480,7 @@ class RTVIBotTranscriptionProcessor(RTVIFrameProcessor): if isinstance(frame, UserStartedSpeakingFrame): await self._push_aggregation() - elif isinstance(frame, TextFrame): + elif isinstance(frame, LLMTextFrame): self._aggregation += frame.text if match_endofsentence(self._aggregation): await self._push_aggregation() @@ -504,7 +505,7 @@ class RTVIBotLLMProcessor(RTVIFrameProcessor): await self._push_transport_message_urgent(RTVIBotLLMStartedMessage()) elif isinstance(frame, LLMFullResponseEndFrame): await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage()) - elif type(frame) is TextFrame: + elif isinstance(frame, LLMTextFrame): message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) await self._push_transport_message_urgent(message) @@ -522,7 +523,7 @@ class RTVIBotTTSProcessor(RTVIFrameProcessor): await self._push_transport_message_urgent(RTVIBotTTSStartedMessage()) elif isinstance(frame, TTSStoppedFrame): await self._push_transport_message_urgent(RTVIBotTTSStoppedMessage()) - elif type(frame) is TextFrame: + elif isinstance(frame, TTSTextFrame): message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text)) await self._push_transport_message_urgent(message) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 973332623..0a03b2eaa 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -30,6 +30,7 @@ from pipecat.frames.frames import ( TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, + TTSTextFrame, TTSUpdateSettingsFrame, UserImageRequestFrame, VisionImageRawFrame, @@ -358,7 +359,7 @@ class TTSService(AIService): if self._push_text_frames: # We send the original text after the audio. This way, if we are # interrupted, the text is not added to the assistant context. - await self.push_frame(TextFrame(text)) + await self.push_frame(TTSTextFrame(text)) async def _stop_frame_handler(self): try: @@ -437,7 +438,7 @@ class WordTTSService(TTSService): frame = TTSStoppedFrame() frame.pts = last_pts else: - frame = TextFrame(word) + frame = TTSTextFrame(word) frame.pts = self._initial_word_timestamp + timestamp if frame: last_pts = frame.pts diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 08bde31b7..1590296af 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -26,10 +26,10 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, + LLMTextFrame, LLMUpdateSettingsFrame, OpenAILLMContextAssistantTimestampFrame, StartInterruptionFrame, - TextFrame, UserImageRawFrame, UserImageRequestFrame, VisionImageRawFrame, @@ -191,7 +191,7 @@ class AnthropicLLMService(LLMService): if event.type == "content_block_delta": if hasattr(event.delta, "text"): - await self.push_frame(TextFrame(event.delta.text)) + await self.push_frame(LLMTextFrame(event.delta.text)) completion_tokens_estimate += self._estimate_tokens(event.delta.text) elif hasattr(event.delta, "partial_json") and tool_use_block: json_accumulator += event.delta.partial_json diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 6ce09e721..57753e9cd 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -28,10 +28,10 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesAppendFrame, LLMSetToolsFrame, + LLMTextFrame, LLMUpdateSettingsFrame, StartFrame, StartInterruptionFrame, - TextFrame, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, @@ -295,7 +295,7 @@ class GeminiMultimodalLiveLLMService(LLMService): # definitely feels like a hack. Need to revisit when the API evolves. # context.add_message({"role": "assistant", "content": [{"type": "text", "text": text}]}) await self.push_frame(LLMFullResponseStartFrame()) - await self.push_frame(TextFrame(text=text)) + await self.push_frame(LLMTextFrame(text=text)) await self.push_frame(LLMFullResponseEndFrame()) async def _transcribe_audio(self, audio, context): @@ -628,7 +628,7 @@ class GeminiMultimodalLiveLLMService(LLMService): await self.push_frame(LLMFullResponseStartFrame()) self._bot_text_buffer += text - await self.push_frame(TextFrame(text=text)) + await self.push_frame(LLMTextFrame(text=text)) inline_data = part.inlineData if not inline_data: diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 341c51636..4e4db5990 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -23,9 +23,9 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, + LLMTextFrame, LLMUpdateSettingsFrame, OpenAILLMContextAssistantTimestampFrame, - TextFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, @@ -698,7 +698,7 @@ class GoogleLLMService(LLMService): try: for c in chunk.parts: if c.text: - await self.push_frame(TextFrame(c.text)) + await self.push_frame(LLMTextFrame(c.text)) elif c.function_call: logger.debug(f"!!! Function call: {c.function_call}") args = type(c.function_call).to_dict(c.function_call).get("args", {}) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 4a70838f4..19c8f6d99 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -25,10 +25,10 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, + LLMTextFrame, LLMUpdateSettingsFrame, OpenAILLMContextAssistantTimestampFrame, StartInterruptionFrame, - TextFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, @@ -258,7 +258,7 @@ class BaseOpenAILLMService(LLMService): # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments elif chunk.choices[0].delta.content: - await self.push_frame(TextFrame(chunk.choices[0].delta.content)) + await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) # if we got a function name and arguments, check to see if it's a function with # a registered handler. If so, run the registered callback, save the result to diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 44057606f..c1d7c4a4d 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -24,11 +24,11 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesAppendFrame, LLMSetToolsFrame, + LLMTextFrame, LLMUpdateSettingsFrame, StartFrame, StartInterruptionFrame, StopInterruptionFrame, - TextFrame, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, @@ -458,7 +458,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_audio_transcript_delta(self, evt): if evt.delta: - await self.push_frame(TextFrame(evt.delta)) + await self.push_frame(LLMTextFrame(evt.delta)) async def _handle_evt_speech_started(self, evt): await self._truncate_current_audio_response()