Add sync_with_audio support for OutputImageRawFrame
Add a `sync_with_audio` field to `OutputImageRawFrame` that routes image frames through the audio queue in the output transport, ensuring images are only displayed after all preceding audio has been sent. This enables proper audio/image synchronization in pipelines like the calendar month narration example. Update the 05-sync-speech-and-image example to use an `ImageAudioSync` processor that sets this flag on image frames.
This commit is contained in:
committed by
Mark Backman
parent
f39472b150
commit
b68495ce0a
@@ -16,6 +16,7 @@ from pipecat.frames.frames import (
|
||||
Frame,
|
||||
LLMContextFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
OutputImageRawFrame,
|
||||
TextFrame,
|
||||
)
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
@@ -44,6 +45,18 @@ class MonthFrame(DataFrame):
|
||||
return f"{self.name}(month: {self.month})"
|
||||
|
||||
|
||||
class ImageAudioSync(FrameProcessor):
|
||||
"""Marks output image frames to be synchronized with audio playback."""
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, OutputImageRawFrame):
|
||||
frame.sync_with_audio = True
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
|
||||
class MonthPrepender(FrameProcessor):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -129,7 +142,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
sentence_aggregator, # Aggregates LLM output into full sentences
|
||||
SyncParallelPipeline( # Run pipelines in parallel aggregating the result
|
||||
[month_prepender, tts], # Create "Month: sentence" and output audio
|
||||
[imagegen], # Generate image
|
||||
[
|
||||
imagegen, # Generate image
|
||||
ImageAudioSync(), # Mark image as needing sync output w/audio
|
||||
],
|
||||
),
|
||||
transport.output(), # Transport output
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user