Merge pull request #853 from pipecat-ai/revert-849-aleix/no-need-for-super-process-frame

Revert "no longer necessary to call super().process_frame(frame, direction)"
This commit is contained in:
Aleix Conchillo Flaqué
2024-12-12 17:21:20 -08:00
committed by GitHub
57 changed files with 212 additions and 56 deletions

View File

@@ -13,12 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian,
Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa). Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa).
### Changed
- It's no longer necessary to call `super().process_frame(frame, direction)` if
you subclass and implement `FrameProcessor.process_frame()`. This is all now
done internally and will avoid possible issues if you forget to add it.
### Deprecated ### Deprecated
- `AWSTTSService` is now deprecated, use `PollyTTSService` instead. - `AWSTTSService` is now deprecated, use `PollyTTSService` instead.

View File

@@ -56,6 +56,8 @@ class MonthPrepender(FrameProcessor):
self.prepend_to_next_text_frame = False self.prepend_to_next_text_frame = False
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, MonthFrame): if isinstance(frame, MonthFrame):
self.most_recent_month = frame.month self.most_recent_month = frame.month
elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame): elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame):

View File

@@ -62,6 +62,8 @@ async def main():
self.text = "" self.text = ""
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
self.text = frame.text self.text = frame.text
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -73,6 +75,8 @@ async def main():
self.frame = None self.frame = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TTSAudioRawFrame): if isinstance(frame, TTSAudioRawFrame):
self.audio.extend(frame.audio) self.audio.extend(frame.audio)
self.frame = OutputAudioRawFrame( self.frame = OutputAudioRawFrame(
@@ -86,6 +90,8 @@ async def main():
self.frame = None self.frame = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, URLImageRawFrame): if isinstance(frame, URLImageRawFrame):
self.frame = frame self.frame = frame
await self.push_frame(frame, direction) await self.push_frame(frame, direction)

View File

@@ -47,6 +47,8 @@ class ImageSyncAggregator(FrameProcessor):
self._waiting_image_bytes = self._waiting_image.tobytes() self._waiting_image_bytes = self._waiting_image.tobytes()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if not isinstance(frame, SystemFrame) and direction == FrameDirection.DOWNSTREAM: if not isinstance(frame, SystemFrame) and direction == FrameDirection.DOWNSTREAM:
await self.push_frame( await self.push_frame(
OutputImageRawFrame( OutputImageRawFrame(

View File

@@ -82,6 +82,8 @@ class UserAudioCollector(FrameProcessor):
self._user_speaking = False self._user_speaking = False
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
# We could gracefully handle both audio input and text/transcription input ... # We could gracefully handle both audio input and text/transcription input ...
# but let's leave that as an exercise to the reader. :-) # but let's leave that as an exercise to the reader. :-)
@@ -124,6 +126,7 @@ class TranscriptExtractor(FrameProcessor):
self._accumulating_transcript = False self._accumulating_transcript = False
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, LLMFullResponseStartFrame): if isinstance(frame, LLMFullResponseStartFrame):
self._processing_llm_response = True self._processing_llm_response = True
self._accumulating_transcript = True self._accumulating_transcript = True
@@ -177,6 +180,8 @@ class TanscriptionContextFixup(FrameProcessor):
self._context.messages[-1].parts[-1].text += f"\n\n{marker}\n{self._transcript}\n" self._context.messages[-1].parts[-1].text += f"\n\n{marker}\n{self._transcript}\n"
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, MagicDemoTranscriptionFrame): if isinstance(frame, MagicDemoTranscriptionFrame):
self._transcript = frame.text self._transcript = frame.text
elif isinstance(frame, LLMFullResponseEndFrame) or isinstance( elif isinstance(frame, LLMFullResponseEndFrame) or isinstance(

View File

@@ -35,6 +35,8 @@ logger.add(sys.stderr, level="DEBUG")
class MirrorProcessor(FrameProcessor): class MirrorProcessor(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, InputAudioRawFrame): if isinstance(frame, InputAudioRawFrame):
await self.push_frame( await self.push_frame(
OutputAudioRawFrame( OutputAudioRawFrame(

View File

@@ -39,6 +39,8 @@ logger.add(sys.stderr, level="DEBUG")
class MirrorProcessor(FrameProcessor): class MirrorProcessor(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, InputAudioRawFrame): if isinstance(frame, InputAudioRawFrame):
await self.push_frame( await self.push_frame(
OutputAudioRawFrame( OutputAudioRawFrame(

View File

@@ -60,6 +60,8 @@ for file in sound_files:
class OutboundSoundEffectWrapper(FrameProcessor): class OutboundSoundEffectWrapper(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, LLMFullResponseEndFrame): if isinstance(frame, LLMFullResponseEndFrame):
await self.push_frame(sounds["ding1.wav"]) await self.push_frame(sounds["ding1.wav"])
# In case anything else downstream needs it # In case anything else downstream needs it
@@ -70,6 +72,8 @@ class OutboundSoundEffectWrapper(FrameProcessor):
class InboundSoundEffectWrapper(FrameProcessor): class InboundSoundEffectWrapper(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, OpenAILLMContextFrame): if isinstance(frame, OpenAILLMContextFrame):
await self.push_frame(sounds["ding2.wav"]) await self.push_frame(sounds["ding2.wav"])
# In case anything else downstream needs it # In case anything else downstream needs it

View File

@@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor):
self._participant_id = participant_id self._participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self._participant_id and isinstance(frame, TextFrame): if self._participant_id and isinstance(frame, TextFrame):
await self.push_frame( await self.push_frame(
UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM

View File

@@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor):
self._participant_id = participant_id self._participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self._participant_id and isinstance(frame, TextFrame): if self._participant_id and isinstance(frame, TextFrame):
await self.push_frame( await self.push_frame(
UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM

View File

@@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor):
self._participant_id = participant_id self._participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self._participant_id and isinstance(frame, TextFrame): if self._participant_id and isinstance(frame, TextFrame):
await self.push_frame( await self.push_frame(
UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM

View File

@@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor):
self._participant_id = participant_id self._participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self._participant_id and isinstance(frame, TextFrame): if self._participant_id and isinstance(frame, TextFrame):
await self.push_frame( await self.push_frame(
UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM

View File

@@ -30,6 +30,8 @@ logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")

View File

@@ -28,6 +28,8 @@ logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")

View File

@@ -31,6 +31,8 @@ logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")

View File

@@ -29,6 +29,8 @@ logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")

View File

@@ -29,6 +29,8 @@ logger.add(sys.stderr, level="DEBUG")
class TranscriptionLogger(FrameProcessor): class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
print(f"Transcription: {frame.text}") print(f"Transcription: {frame.text}")

View File

@@ -64,6 +64,7 @@ class StatementJudgeContextFilter(FrameProcessor):
self._notifier = notifier self._notifier = notifier
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We must not block system frames. # We must not block system frames.
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -117,6 +118,7 @@ class CompletenessCheck(FrameProcessor):
self._notifier = notifier self._notifier = notifier
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame) and frame.text == "YES": if isinstance(frame, TextFrame) and frame.text == "YES":
logger.debug("Completeness check YES") logger.debug("Completeness check YES")
await self.push_frame(UserStoppedSpeakingFrame()) await self.push_frame(UserStoppedSpeakingFrame())
@@ -139,6 +141,8 @@ class OutputGate(FrameProcessor):
self._gate_open = True self._gate_open = True
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We must not block system frames. # We must not block system frames.
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):

View File

@@ -101,12 +101,12 @@ HIGH PRIORITY SIGNALS:
Examples: Examples:
# Complete Wh-question # Complete Wh-question
[{"role": "assistant", "content": "I can help you learn."}, [{"role": "assistant", "content": "I can help you learn."},
{"role": "user", "content": "What's the fastest way to learn Spanish"}] {"role": "user", "content": "What's the fastest way to learn Spanish"}]
Output: YES Output: YES
# Complete Yes/No question despite STT error # Complete Yes/No question despite STT error
[{"role": "assistant", "content": "I know about planets."}, [{"role": "assistant", "content": "I know about planets."},
{"role": "user", "content": "Is is Jupiter the biggest planet"}] {"role": "user", "content": "Is is Jupiter the biggest planet"}]
Output: YES Output: YES
@@ -118,12 +118,12 @@ Output: YES
Examples: Examples:
# Direct instruction # Direct instruction
[{"role": "assistant", "content": "I can explain many topics."}, [{"role": "assistant", "content": "I can explain many topics."},
{"role": "user", "content": "Tell me about black holes"}] {"role": "user", "content": "Tell me about black holes"}]
Output: YES Output: YES
# Action demand # Action demand
[{"role": "assistant", "content": "I can help with math."}, [{"role": "assistant", "content": "I can help with math."},
{"role": "user", "content": "Solve this equation x plus 5 equals 12"}] {"role": "user", "content": "Solve this equation x plus 5 equals 12"}]
Output: YES Output: YES
@@ -134,12 +134,12 @@ Output: YES
Examples: Examples:
# Specific answer # Specific answer
[{"role": "assistant", "content": "What's your favorite color?"}, [{"role": "assistant", "content": "What's your favorite color?"},
{"role": "user", "content": "I really like blue"}] {"role": "user", "content": "I really like blue"}]
Output: YES Output: YES
# Option selection # Option selection
[{"role": "assistant", "content": "Would you prefer morning or evening?"}, [{"role": "assistant", "content": "Would you prefer morning or evening?"},
{"role": "user", "content": "Morning"}] {"role": "user", "content": "Morning"}]
Output: YES Output: YES
@@ -153,17 +153,17 @@ MEDIUM PRIORITY SIGNALS:
Examples: Examples:
# Self-correction reaching completion # Self-correction reaching completion
[{"role": "assistant", "content": "What would you like to know?"}, [{"role": "assistant", "content": "What would you like to know?"},
{"role": "user", "content": "Tell me about... no wait, explain how rainbows form"}] {"role": "user", "content": "Tell me about... no wait, explain how rainbows form"}]
Output: YES Output: YES
# Topic change with complete thought # Topic change with complete thought
[{"role": "assistant", "content": "The weather is nice today."}, [{"role": "assistant", "content": "The weather is nice today."},
{"role": "user", "content": "Actually can you tell me who invented the telephone"}] {"role": "user", "content": "Actually can you tell me who invented the telephone"}]
Output: YES Output: YES
# Mid-sentence completion # Mid-sentence completion
[{"role": "assistant", "content": "Hello I'm ready."}, [{"role": "assistant", "content": "Hello I'm ready."},
{"role": "user", "content": "What's the capital of? France"}] {"role": "user", "content": "What's the capital of? France"}]
Output: YES Output: YES
@@ -175,12 +175,12 @@ Output: YES
Examples: Examples:
# Acknowledgment # Acknowledgment
[{"role": "assistant", "content": "Should we talk about history?"}, [{"role": "assistant", "content": "Should we talk about history?"},
{"role": "user", "content": "Sure"}] {"role": "user", "content": "Sure"}]
Output: YES Output: YES
# Disagreement with completion # Disagreement with completion
[{"role": "assistant", "content": "Is that what you meant?"}, [{"role": "assistant", "content": "Is that what you meant?"},
{"role": "user", "content": "No not really"}] {"role": "user", "content": "No not really"}]
Output: YES Output: YES
@@ -194,12 +194,12 @@ LOW PRIORITY SIGNALS:
Examples: Examples:
# Word repetition but complete # Word repetition but complete
[{"role": "assistant", "content": "I can help with that."}, [{"role": "assistant", "content": "I can help with that."},
{"role": "user", "content": "What what is the time right now"}] {"role": "user", "content": "What what is the time right now"}]
Output: YES Output: YES
# Missing punctuation but complete # Missing punctuation but complete
[{"role": "assistant", "content": "I can explain that."}, [{"role": "assistant", "content": "I can explain that."},
{"role": "user", "content": "Please tell me how computers work"}] {"role": "user", "content": "Please tell me how computers work"}]
Output: YES Output: YES
@@ -211,12 +211,12 @@ Output: YES
Examples: Examples:
# Filler words but complete # Filler words but complete
[{"role": "assistant", "content": "What would you like to know?"}, [{"role": "assistant", "content": "What would you like to know?"},
{"role": "user", "content": "Um uh how do airplanes fly"}] {"role": "user", "content": "Um uh how do airplanes fly"}]
Output: YES Output: YES
# Thinking pause but incomplete # Thinking pause but incomplete
[{"role": "assistant", "content": "I can explain anything."}, [{"role": "assistant", "content": "I can explain anything."},
{"role": "user", "content": "Well um I want to know about the"}] {"role": "user", "content": "Well um I want to know about the"}]
Output: NO Output: NO
@@ -241,17 +241,17 @@ DECISION RULES:
Examples: Examples:
# Incomplete despite corrections # Incomplete despite corrections
[{"role": "assistant", "content": "What would you like to know about?"}, [{"role": "assistant", "content": "What would you like to know about?"},
{"role": "user", "content": "Can you tell me about"}] {"role": "user", "content": "Can you tell me about"}]
Output: NO Output: NO
# Complete despite multiple artifacts # Complete despite multiple artifacts
[{"role": "assistant", "content": "I can help you learn."}, [{"role": "assistant", "content": "I can help you learn."},
{"role": "user", "content": "How do you I mean what's the best way to learn programming"}] {"role": "user", "content": "How do you I mean what's the best way to learn programming"}]
Output: YES Output: YES
# Trailing off incomplete # Trailing off incomplete
[{"role": "assistant", "content": "I can explain anything."}, [{"role": "assistant", "content": "I can explain anything."},
{"role": "user", "content": "I was wondering if you could tell me why"}] {"role": "user", "content": "I was wondering if you could tell me why"}]
Output: NO Output: NO
""" """
@@ -268,6 +268,7 @@ class StatementJudgeContextFilter(FrameProcessor):
self._notifier = notifier self._notifier = notifier
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We must not block system frames. # We must not block system frames.
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -319,6 +320,8 @@ class CompletenessCheck(FrameProcessor):
self._notifier = notifier self._notifier = notifier
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame) and frame.text == "YES": if isinstance(frame, TextFrame) and frame.text == "YES":
logger.debug("!!! Completeness check YES") logger.debug("!!! Completeness check YES")
await self.push_frame(UserStoppedSpeakingFrame()) await self.push_frame(UserStoppedSpeakingFrame())
@@ -341,6 +344,8 @@ class OutputGate(FrameProcessor):
self._gate_open = True self._gate_open = True
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We must not block system frames. # We must not block system frames.
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):

View File

@@ -90,6 +90,8 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor):
self._user_speaking = False self._user_speaking = False
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# ignore context frame # ignore context frame
if isinstance(frame, OpenAILLMContextFrame): if isinstance(frame, OpenAILLMContextFrame):
return return
@@ -131,6 +133,8 @@ class CompletenessCheck(FrameProcessor):
self._audio_accumulator = audio_accumulator self._audio_accumulator = audio_accumulator
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame) and frame.text.startswith("YES"): if isinstance(frame, TextFrame) and frame.text.startswith("YES"):
logger.debug("Completeness check YES") logger.debug("Completeness check YES")
await self.push_frame(UserStoppedSpeakingFrame()) await self.push_frame(UserStoppedSpeakingFrame())
@@ -155,6 +159,8 @@ class OutputGate(FrameProcessor):
self._gate_open = True self._gate_open = True
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We must not block system frames. # We must not block system frames.
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):

View File

@@ -95,6 +95,8 @@ class UserAudioCollector(FrameProcessor):
self._user_speaking = False self._user_speaking = False
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
# We could gracefully handle both audio input and text/transcription input ... # We could gracefully handle both audio input and text/transcription input ...
# but let's leave that as an exercise to the reader. :-) # but let's leave that as an exercise to the reader. :-)
@@ -133,6 +135,8 @@ class InputTranscriptionContextFilter(FrameProcessor):
""" """
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
# We don't want to block system frames. # We don't want to block system frames.
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -206,6 +210,8 @@ class InputTranscriptionFrameEmitter(FrameProcessor):
self._aggregation = "" self._aggregation = ""
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
self._aggregation += frame.text self._aggregation += frame.text
elif isinstance(frame, LLMFullResponseEndFrame): elif isinstance(frame, LLMFullResponseEndFrame):
@@ -256,6 +262,8 @@ class TranscriptionContextFixup(FrameProcessor):
audio_part.text = self._transcript audio_part.text = self._transcript
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, LLMDemoTranscriptionFrame): if isinstance(frame, LLMDemoTranscriptionFrame):
logger.info(f"Transcription from Gemini: {frame.text}") logger.info(f"Transcription from Gemini: {frame.text}")
self._transcript = frame.text self._transcript = frame.text

View File

@@ -81,6 +81,8 @@ class TalkingAnimation(FrameProcessor):
self._is_talking = False self._is_talking = False
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, BotStartedSpeakingFrame): if isinstance(frame, BotStartedSpeakingFrame):
if not self._is_talking: if not self._is_talking:
await self.push_frame(talking_frame) await self.push_frame(talking_frame)
@@ -101,6 +103,8 @@ class UserImageRequester(FrameProcessor):
self.participant_id = participant_id self.participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self.participant_id and isinstance(frame, TextFrame): if self.participant_id and isinstance(frame, TextFrame):
if frame.text == user_request_answer: if frame.text == user_request_answer:
await self.push_frame( await self.push_frame(
@@ -117,6 +121,8 @@ class TextFilterProcessor(FrameProcessor):
self.text = text self.text = text
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
if frame.text != self.text: if frame.text != self.text:
await self.push_frame(frame) await self.push_frame(frame)
@@ -126,6 +132,8 @@ class TextFilterProcessor(FrameProcessor):
class ImageFilterProcessor(FrameProcessor): class ImageFilterProcessor(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if not isinstance(frame, ImageRawFrame): if not isinstance(frame, ImageRawFrame):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)

View File

@@ -95,6 +95,8 @@ class TalkingAnimation(FrameProcessor):
frame: The incoming frame to process frame: The incoming frame to process
direction: The direction of frame flow in the pipeline direction: The direction of frame flow in the pipeline
""" """
await super().process_frame(frame, direction)
# Switch to talking animation when bot starts speaking # Switch to talking animation when bot starts speaking
if isinstance(frame, BotStartedSpeakingFrame): if isinstance(frame, BotStartedSpeakingFrame):
if not self._is_talking: if not self._is_talking:

View File

@@ -95,6 +95,8 @@ class TalkingAnimation(FrameProcessor):
frame: The incoming frame to process frame: The incoming frame to process
direction: The direction of frame flow in the pipeline direction: The direction of frame flow in the pipeline
""" """
await super().process_frame(frame, direction)
# Switch to talking animation when bot starts speaking # Switch to talking animation when bot starts speaking
if isinstance(frame, BotStartedSpeakingFrame): if isinstance(frame, BotStartedSpeakingFrame):
if not self._is_talking: if not self._is_talking:

View File

@@ -54,6 +54,8 @@ class StoryImageProcessor(FrameProcessor):
self._fal_service = fal_service self._fal_service = fal_service
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StoryImageFrame): if isinstance(frame, StoryImageFrame):
try: try:
async with timeout(7): async with timeout(7):
@@ -88,6 +90,8 @@ class StoryProcessor(FrameProcessor):
self._story = story self._story = story
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, UserStoppedSpeakingFrame): if isinstance(frame, UserStoppedSpeakingFrame):
# Send an app message to the UI # Send an app message to the UI
await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN)) await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN))

View File

@@ -51,6 +51,8 @@ class TranslationProcessor(FrameProcessor):
self._language = language self._language = language
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
context = [ context = [
{ {
@@ -76,6 +78,8 @@ class TranslationSubtitles(FrameProcessor):
# subtitles. # subtitles.
# #
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
message = {"language": self._language, "text": frame.text} message = {"language": self._language, "text": frame.text}
await self.push_frame(DailyTransportMessageFrame(message)) await self.push_frame(DailyTransportMessageFrame(message))

View File

@@ -28,6 +28,8 @@ class Source(FrameProcessor):
self._push_frame_func = push_frame_func self._push_frame_func = push_frame_func
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
match direction: match direction:
case FrameDirection.UPSTREAM: case FrameDirection.UPSTREAM:
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
@@ -49,6 +51,8 @@ class Sink(FrameProcessor):
self._push_frame_func = push_frame_func self._push_frame_func = push_frame_func
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
match direction: match direction:
case FrameDirection.UPSTREAM: case FrameDirection.UPSTREAM:
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -116,6 +120,8 @@ class ParallelPipeline(BasePipeline):
self._down_task = loop.create_task(self._process_down_queue()) self._down_task = loop.create_task(self._process_down_queue())
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
await self._start_tasks() await self._start_tasks()

View File

@@ -17,6 +17,8 @@ class PipelineSource(FrameProcessor):
self._upstream_push_frame = upstream_push_frame self._upstream_push_frame = upstream_push_frame
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
match direction: match direction:
case FrameDirection.UPSTREAM: case FrameDirection.UPSTREAM:
await self._upstream_push_frame(frame, direction) await self._upstream_push_frame(frame, direction)
@@ -30,6 +32,8 @@ class PipelineSink(FrameProcessor):
self._downstream_push_frame = downstream_push_frame self._downstream_push_frame = downstream_push_frame
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
match direction: match direction:
case FrameDirection.UPSTREAM: case FrameDirection.UPSTREAM:
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -70,6 +74,8 @@ class Pipeline(BasePipeline):
await self._cleanup_processors() await self._cleanup_processors()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if direction == FrameDirection.DOWNSTREAM: if direction == FrameDirection.DOWNSTREAM:
await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM) await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM)
elif direction == FrameDirection.UPSTREAM: elif direction == FrameDirection.UPSTREAM:

View File

@@ -31,6 +31,8 @@ class Source(FrameProcessor):
self._up_queue = upstream_queue self._up_queue = upstream_queue
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
match direction: match direction:
case FrameDirection.UPSTREAM: case FrameDirection.UPSTREAM:
await self._up_queue.put(frame) await self._up_queue.put(frame)
@@ -44,6 +46,8 @@ class Sink(FrameProcessor):
self._down_queue = downstream_queue self._down_queue = downstream_queue
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
match direction: match direction:
case FrameDirection.UPSTREAM: case FrameDirection.UPSTREAM:
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -99,6 +103,8 @@ class SyncParallelPipeline(BasePipeline):
# #
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# The last processor of each pipeline needs to be synchronous otherwise # The last processor of each pipeline needs to be synchronous otherwise
# this element won't work. Since, we know it should be synchronous we # this element won't work. Since, we know it should be synchronous we
# push a SyncFrame. Since frames are ordered we know this frame will be # push a SyncFrame. Since frames are ordered we know this frame will be

View File

@@ -45,6 +45,8 @@ class Source(FrameProcessor):
self._up_queue = up_queue self._up_queue = up_queue
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
match direction: match direction:
case FrameDirection.UPSTREAM: case FrameDirection.UPSTREAM:
await self._handle_upstream_frame(frame) await self._handle_upstream_frame(frame)
@@ -73,6 +75,8 @@ class Sink(FrameProcessor):
self._down_queue = down_queue self._down_queue = down_queue
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We really just want to know when the EndFrame reached the sink. # We really just want to know when the EndFrame reached the sink.
if isinstance(frame, EndFrame): if isinstance(frame, EndFrame):
await self._down_queue.put(frame) await self._down_queue.put(frame)

View File

@@ -56,6 +56,8 @@ class GatedAggregator(FrameProcessor):
self._accumulator: List[Tuple[Frame, FrameDirection]] = [] self._accumulator: List[Tuple[Frame, FrameDirection]] = []
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We must not block system frames. # We must not block system frames.
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)

View File

@@ -24,6 +24,8 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
self._last_context_frame = None self._last_context_frame = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
await self.push_frame(frame) await self.push_frame(frame)
await self._start() await self._start()

View File

@@ -86,6 +86,8 @@ class LLMResponseAggregator(FrameProcessor):
# and T2 would be dropped. # and T2 would be dropped.
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
send_aggregation = False send_aggregation = False
if isinstance(frame, self._start_frame): if isinstance(frame, self._start_frame):
@@ -238,6 +240,8 @@ class LLMFullResponseAggregator(FrameProcessor):
self._aggregation = "" self._aggregation = ""
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
self._aggregation += frame.text self._aggregation += frame.text
elif isinstance(frame, LLMFullResponseEndFrame): elif isinstance(frame, LLMFullResponseEndFrame):

View File

@@ -33,6 +33,8 @@ class SentenceAggregator(FrameProcessor):
self._aggregation = "" self._aggregation = ""
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We ignore interim description at this point. # We ignore interim description at this point.
if isinstance(frame, InterimTranscriptionFrame): if isinstance(frame, InterimTranscriptionFrame):
return return

View File

@@ -85,6 +85,8 @@ class ResponseAggregator(FrameProcessor):
# and T2 would be dropped. # and T2 would be dropped.
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
send_aggregation = False send_aggregation = False
if isinstance(frame, self._start_frame): if isinstance(frame, self._start_frame):

View File

@@ -31,6 +31,8 @@ class VisionImageFrameAggregator(FrameProcessor):
self._describe_text = None self._describe_text = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
self._describe_text = frame.text self._describe_text = frame.text
elif isinstance(frame, InputImageRawFrame): elif isinstance(frame, InputImageRawFrame):

View File

@@ -24,6 +24,8 @@ class AsyncGeneratorProcessor(FrameProcessor):
self._data_queue = asyncio.Queue() self._data_queue = asyncio.Queue()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, (CancelFrame, EndFrame)): if isinstance(frame, (CancelFrame, EndFrame)):

View File

@@ -68,6 +68,8 @@ class AudioBufferProcessor(FrameProcessor):
self._bot_audio_buffer = bytearray() self._bot_audio_buffer = bytearray()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Include all audio from the user. # Include all audio from the user.
if isinstance(frame, InputAudioRawFrame): if isinstance(frame, InputAudioRawFrame):
resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate) resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate)

View File

@@ -39,6 +39,8 @@ class SileroVAD(FrameProcessor):
# #
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, AudioRawFrame): if isinstance(frame, AudioRawFrame):
await self._analyze_audio(frame) await self._analyze_audio(frame)
if self._audio_passthrough: if self._audio_passthrough:

View File

@@ -26,5 +26,7 @@ class FrameFilter(FrameProcessor):
return isinstance(frame, ControlFrame) or isinstance(frame, SystemFrame) return isinstance(frame, ControlFrame) or isinstance(frame, SystemFrame)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self._should_passthrough_frame(frame): if self._should_passthrough_frame(frame):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)

View File

@@ -29,6 +29,8 @@ class FunctionFilter(FrameProcessor):
return isinstance(frame, SystemFrame) or direction != self._direction return isinstance(frame, SystemFrame) or direction != self._direction
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
passthrough = self._should_passthrough_frame(frame, direction) passthrough = self._should_passthrough_frame(frame, direction)
allowed = await self._filter(frame) allowed = await self._filter(frame)
if passthrough or allowed: if passthrough or allowed:

View File

@@ -26,4 +26,5 @@ class IdentityFilter(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process an incoming frame by passing it through unchanged.""" """Process an incoming frame by passing it through unchanged."""
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)

View File

@@ -45,6 +45,8 @@ class WakeCheckFilter(FrameProcessor):
self._wake_patterns.append(pattern) self._wake_patterns.append(pattern)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
try: try:
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
p = self._participant_states.get(frame.user_id) p = self._participant_states.get(frame.user_id)

View File

@@ -32,6 +32,8 @@ class WakeNotifierFilter(FrameProcessor):
self._filter = filter self._filter = filter
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, self._types) and await self._filter(frame): if isinstance(frame, self._types) and await self._filter(frame):
await self._notifier.notify() await self._notifier.notify()

View File

@@ -161,13 +161,6 @@ class FrameProcessor:
def get_clock(self) -> BaseClock: def get_clock(self) -> BaseClock:
return self._clock return self._clock
async def pause_processing_frames(self):
self.__should_block_frames = True
async def resume_processing_frames(self):
self.__input_event.set()
self.__should_block_frames = False
async def queue_frame( async def queue_frame(
self, self,
frame: Frame, frame: Frame,
@@ -182,13 +175,32 @@ class FrameProcessor:
if isinstance(frame, SystemFrame): if isinstance(frame, SystemFrame):
# We don't want to queue system frames. # We don't want to queue system frames.
await self._process_frame(frame, direction) await self.process_frame(frame, direction)
else: else:
# We queue everything else. # We queue everything else.
await self.__input_queue.put((frame, direction, callback)) await self.__input_queue.put((frame, direction, callback))
async def pause_processing_frames(self):
self.__should_block_frames = True
async def resume_processing_frames(self):
self.__input_event.set()
self.__should_block_frames = False
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
pass if isinstance(frame, StartFrame):
self._clock = frame.clock
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
elif isinstance(frame, StartInterruptionFrame):
await self._start_interruption()
await self.stop_all_metrics()
elif isinstance(frame, StopInterruptionFrame):
self._should_report_ttfb = True
elif isinstance(frame, CancelFrame):
self._cancelling = True
async def push_error(self, error: ErrorFrame): async def push_error(self, error: ErrorFrame):
await self.push_frame(error, FrameDirection.UPSTREAM) await self.push_frame(error, FrameDirection.UPSTREAM)
@@ -216,28 +228,6 @@ class FrameProcessor:
raise Exception(f"Event handler {event_name} already registered") raise Exception(f"Event handler {event_name} already registered")
self._event_handlers[event_name] = [] self._event_handlers[event_name] = []
#
# Frame processing
#
async def _process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, StartFrame):
self._clock = frame.clock
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
elif isinstance(frame, StartInterruptionFrame):
await self._start_interruption()
await self.stop_all_metrics()
elif isinstance(frame, StopInterruptionFrame):
self._should_report_ttfb = True
elif isinstance(frame, CancelFrame):
self._cancelling = True
# Call subclass.
await self.process_frame(frame, direction)
# #
# Handle interruptions # Handle interruptions
# #
@@ -299,7 +289,7 @@ class FrameProcessor:
(frame, direction, callback) = await self.__input_queue.get() (frame, direction, callback) = await self.__input_queue.get()
# Process the frame. # Process the frame.
await self._process_frame(frame, direction) await self.process_frame(frame, direction)
# If this frame has an associated callback, call it now. # If this frame has an associated callback, call it now.
if callback: if callback:

View File

@@ -36,6 +36,8 @@ class LangchainProcessor(FrameProcessor):
self._participant_id = participant_id self._participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, LLMMessagesFrame): if isinstance(frame, LLMMessagesFrame):
# Messages are accumulated on the context as a list of messages. # Messages are accumulated on the context as a list of messages.
# The last one by the human is the one we want to send to the LLM. # The last one by the human is the one we want to send to the LLM.

View File

@@ -380,6 +380,8 @@ class RTVISpeakingProcessor(RTVIFrameProcessor):
super().__init__(**kwargs) super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)): if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)):
@@ -413,6 +415,8 @@ class RTVIUserTranscriptionProcessor(RTVIFrameProcessor):
super().__init__(**kwargs) super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)): if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)):
@@ -442,6 +446,8 @@ class RTVIUserLLMTextProcessor(RTVIFrameProcessor):
super().__init__(**kwargs) super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, OpenAILLMContextFrame): if isinstance(frame, OpenAILLMContextFrame):
@@ -467,6 +473,8 @@ class RTVIBotTranscriptionProcessor(RTVIFrameProcessor):
self._aggregation = "" self._aggregation = ""
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, UserStartedSpeakingFrame):
@@ -488,6 +496,8 @@ class RTVIBotLLMProcessor(RTVIFrameProcessor):
super().__init__(**kwargs) super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, LLMFullResponseStartFrame): if isinstance(frame, LLMFullResponseStartFrame):
@@ -504,6 +514,8 @@ class RTVIBotTTSProcessor(RTVIFrameProcessor):
super().__init__(**kwargs) super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, TTSStartedFrame): if isinstance(frame, TTSStartedFrame):
@@ -520,6 +532,8 @@ class RTVIMetricsProcessor(RTVIFrameProcessor):
super().__init__(**kwargs) super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
if isinstance(frame, MetricsFrame): if isinstance(frame, MetricsFrame):
@@ -628,6 +642,8 @@ class RTVIProcessor(FrameProcessor):
await self._push_transport_message(message, exclude_none=False) await self._push_transport_message(message, exclude_none=False)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Specific system frames # Specific system frames
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be # Push StartFrame before start(), because we want StartFrame to be

View File

@@ -66,6 +66,8 @@ class GStreamerPipelineSource(FrameProcessor):
bus.connect("message", self._on_gstreamer_message) bus.connect("message", self._on_gstreamer_message)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Specific system frames # Specific system frames
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be # Push StartFrame before start(), because we want StartFrame to be

View File

@@ -35,6 +35,8 @@ class IdleFrameProcessor(FrameProcessor):
self._create_idle_task() self._create_idle_task()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
# If we are not waiting for any specific frame set the event, otherwise # If we are not waiting for any specific frame set the event, otherwise

View File

@@ -27,6 +27,8 @@ class StatelessTextTransformer(FrameProcessor):
self._transform_fn = transform_fn self._transform_fn = transform_fn
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):
result = self._transform_fn(frame.text) result = self._transform_fn(frame.text)
if isinstance(result, Coroutine): if isinstance(result, Coroutine):

View File

@@ -43,6 +43,8 @@ class UserIdleProcessor(FrameProcessor):
await self._idle_task await self._idle_task
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Check for end frames before processing # Check for end frames before processing
if isinstance(frame, (EndFrame, CancelFrame)): if isinstance(frame, (EndFrame, CancelFrame)):
await self._stop() await self._stop()

View File

@@ -110,6 +110,8 @@ class AIService(FrameProcessor):
logger.warning(f"Unknown setting for {self.name} service: {key}") logger.warning(f"Unknown setting for {self.name} service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
await self.start(frame) await self.start(frame)
elif isinstance(frame, CancelFrame): elif isinstance(frame, CancelFrame):

View File

@@ -92,6 +92,7 @@ class SimliVideoService(FrameProcessor):
pass pass
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
await self._start_connection() await self._start_connection()

View File

@@ -79,6 +79,8 @@ class BaseInputTransport(FrameProcessor):
# #
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Specific system frames # Specific system frames
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be # Push StartFrame before start(), because we want StartFrame to be

View File

@@ -120,6 +120,8 @@ class BaseOutputTransport(FrameProcessor):
# #
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# #
# System frames (like StartInterruptionFrame) are pushed # System frames (like StartInterruptionFrame) are pushed
# immediately. Other frames require order so they are put in the sink # immediately. Other frames require order so they are put in the sink

View File

@@ -13,6 +13,8 @@ class TestFrameProcessor(FrameProcessor):
super().__init__() super().__init__()
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if not self.test_frames[ if not self.test_frames[
0 0
]: # then we've run out of required frames but the generator is still going? ]: # then we've run out of required frames but the generator is still going?

View File

@@ -42,6 +42,8 @@ class TestLangchain(unittest.IsolatedAsyncioTestCase):
return self.name return self.name
async def process_frame(self, frame, direction): async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if isinstance(frame, LLMFullResponseStartFrame): if isinstance(frame, LLMFullResponseStartFrame):
self.start_collecting = True self.start_collecting = True
elif isinstance(frame, TextFrame) and self.start_collecting: elif isinstance(frame, TextFrame) and self.start_collecting: