wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add sync_with_audio support for OutputImageRawFrame

Browse Source 
Add a `sync_with_audio` field to `OutputImageRawFrame` that routes image
frames through the audio queue in the output transport, ensuring images
are only displayed after all preceding audio has been sent. This enables
proper audio/image synchronization in pipelines like the calendar month
narration example.

Update the 05-sync-speech-and-image example to use an `ImageAudioSync`
processor that sets this flag on image frames.
This commit is contained in:
Paul Kompfner
2026-03-13 15:15:45 -04:00
committed by Mark Backman
parent f39472b150
commit b68495ce0a
3 changed files with 30 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/pipecat/frames/frames.py
View File

@@ -274,8 +274,16 @@ class OutputImageRawFrame(DataFrame, ImageRawFrame):
An image that will be shown by the transport. If the transport supports
multiple video destinations (e.g. multiple video tracks) the destination
name can be specified in transport_destination.
Parameters:
sync_with_audio: If True, the image is queued with audio frames so
it is only displayed after all preceding audio has been sent.
Defaults to False (image is displayed immediately when the output
transport receives it).
"""
sync_with_audio: bool = field(default=False, init=False)
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, destination: {self.transport_destination}, size: {self.size}, format: {self.format})"

6
src/pipecat/transports/base_output.py
View File

@@ -569,7 +569,11 @@ class BaseOutputTransport(FrameProcessor):
if not self._params.video_out_enabled:
return
if self._params.video_out_is_live and isinstance(frame, OutputImageRawFrame):
if isinstance(frame, OutputImageRawFrame) and frame.sync_with_audio:
# Route through the audio queue so the image is only
# displayed after all preceding audio has been sent.
await self._audio_queue.put(frame)
elif self._params.video_out_is_live and isinstance(frame, OutputImageRawFrame):
await self._video_queue.put(frame)
elif isinstance(frame, OutputImageRawFrame):
await self._set_video_image(frame)