wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

frames: add LLMTextFrame and TTSTextFrame

Browse Source 
This is to distinguish what type of service has generated the TextFrames.
This commit is contained in:
Aleix Conchillo Flaqué
2025-01-14 14:44:06 -08:00
parent 7626d7b04b
commit 2d0f3341c3
8 changed files with 33 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/pipecat/frames/frames.py
View File

@@ -177,6 +177,20 @@ class TextFrame(DataFrame):
return f"{self.name}(pts: {pts}, text: [{self.text}])"
@dataclass
class LLMTextFrame(TextFrame):
"""A text frame generated by LLM services."""
pass
@dataclass
class TTSTextFrame(TextFrame):
"""A text frame generated by TTS services."""
pass
@dataclass
class TranscriptionFrame(TextFrame):
"""A text frame with transcription-specific data. Will be placed in the

9
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -34,14 +34,15 @@ from pipecat.frames.frames import (
InterimTranscriptionFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMTextFrame,
MetricsFrame,
StartFrame,
SystemFrame,
TextFrame,
TranscriptionFrame,
TransportMessageUrgentFrame,
TTSStartedFrame,
TTSStoppedFrame,
TTSTextFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
@@ -479,7 +480,7 @@ class RTVIBotTranscriptionProcessor(RTVIFrameProcessor):
if isinstance(frame, UserStartedSpeakingFrame):
await self._push_aggregation()
elif isinstance(frame, TextFrame):
elif isinstance(frame, LLMTextFrame):
self._aggregation += frame.text
if match_endofsentence(self._aggregation):
await self._push_aggregation()
@@ -504,7 +505,7 @@ class RTVIBotLLMProcessor(RTVIFrameProcessor):
await self._push_transport_message_urgent(RTVIBotLLMStartedMessage())
elif isinstance(frame, LLMFullResponseEndFrame):
await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage())
elif type(frame) is TextFrame:
elif isinstance(frame, LLMTextFrame):
message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text))
await self._push_transport_message_urgent(message)
@@ -522,7 +523,7 @@ class RTVIBotTTSProcessor(RTVIFrameProcessor):
await self._push_transport_message_urgent(RTVIBotTTSStartedMessage())
elif isinstance(frame, TTSStoppedFrame):
await self._push_transport_message_urgent(RTVIBotTTSStoppedMessage())
elif type(frame) is TextFrame:
elif isinstance(frame, TTSTextFrame):
message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text))
await self._push_transport_message_urgent(message)

5
src/pipecat/services/ai_services.py
View File

@@ -30,6 +30,7 @@ from pipecat.frames.frames import (
TTSSpeakFrame,
TTSStartedFrame,
TTSStoppedFrame,
TTSTextFrame,
TTSUpdateSettingsFrame,
UserImageRequestFrame,
VisionImageRawFrame,
@@ -358,7 +359,7 @@ class TTSService(AIService):
if self._push_text_frames:
# We send the original text after the audio. This way, if we are
# interrupted, the text is not added to the assistant context.
await self.push_frame(TextFrame(text))
await self.push_frame(TTSTextFrame(text))
async def _stop_frame_handler(self):
try:
@@ -437,7 +438,7 @@ class WordTTSService(TTSService):
frame = TTSStoppedFrame()
frame.pts = last_pts
else:
frame = TextFrame(word)
frame = TTSTextFrame(word)
frame.pts = self._initial_word_timestamp + timestamp
if frame:
last_pts = frame.pts

4
src/pipecat/services/anthropic.py
View File

@@ -26,10 +26,10 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
OpenAILLMContextAssistantTimestampFrame,
StartInterruptionFrame,
TextFrame,
UserImageRawFrame,
UserImageRequestFrame,
VisionImageRawFrame,
@@ -191,7 +191,7 @@ class AnthropicLLMService(LLMService):
if event.type == "content_block_delta":
if hasattr(event.delta, "text"):
await self.push_frame(TextFrame(event.delta.text))
await self.push_frame(LLMTextFrame(event.delta.text))
completion_tokens_estimate += self._estimate_tokens(event.delta.text)
elif hasattr(event.delta, "partial_json") and tool_use_block:
json_accumulator += event.delta.partial_json

6
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -28,10 +28,10 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesAppendFrame,
LLMSetToolsFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
StartInterruptionFrame,
TextFrame,
TranscriptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
@@ -295,7 +295,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
# definitely feels like a hack. Need to revisit when the API evolves.
# context.add_message({"role": "assistant", "content": [{"type": "text", "text": text}]})
await self.push_frame(LLMFullResponseStartFrame())
await self.push_frame(TextFrame(text=text))
await self.push_frame(LLMTextFrame(text=text))
await self.push_frame(LLMFullResponseEndFrame())
async def _transcribe_audio(self, audio, context):
@@ -628,7 +628,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self.push_frame(LLMFullResponseStartFrame())
self._bot_text_buffer += text
await self.push_frame(TextFrame(text=text))
await self.push_frame(LLMTextFrame(text=text))
inline_data = part.inlineData
if not inline_data:

4
src/pipecat/services/google.py
View File

@@ -23,9 +23,9 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
OpenAILLMContextAssistantTimestampFrame,
TextFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
@@ -698,7 +698,7 @@ class GoogleLLMService(LLMService):
try:
for c in chunk.parts:
if c.text:
await self.push_frame(TextFrame(c.text))
await self.push_frame(LLMTextFrame(c.text))
elif c.function_call:
logger.debug(f"!!! Function call: {c.function_call}")
args = type(c.function_call).to_dict(c.function_call).get("args", {})

4
src/pipecat/services/openai.py
View File

@@ -25,10 +25,10 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
OpenAILLMContextAssistantTimestampFrame,
StartInterruptionFrame,
TextFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
@@ -258,7 +258,7 @@ class BaseOpenAILLMService(LLMService):
# Keep iterating through the response to collect all the argument fragments
arguments += tool_call.function.arguments
elif chunk.choices[0].delta.content:
await self.push_frame(TextFrame(chunk.choices[0].delta.content))
await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content))
# if we got a function name and arguments, check to see if it's a function with
# a registered handler. If so, run the registered callback, save the result to

4
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -24,11 +24,11 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesAppendFrame,
LLMSetToolsFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
TextFrame,
TranscriptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
@@ -458,7 +458,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
async def _handle_evt_audio_transcript_delta(self, evt):
if evt.delta:
await self.push_frame(TextFrame(evt.delta))
await self.push_frame(LLMTextFrame(evt.delta))
async def _handle_evt_speech_started(self, evt):
await self._truncate_current_audio_response()