wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

transport: allow requesting a user frame

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2024-04-09 22:16:05 -07:00
parent b117a185e3
commit a5eba0106b
2 changed files with 35 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/dailyai/transports/daily_transport.py
View File

@@ -169,8 +169,12 @@ class DailyTransport(ThreadedTransport, EventHandler):
if self._mic_enabled:
self.mic.write_frames(frame)
def send_app_message(self, message: Any, participantId: str | None):
self.client.send_app_message(message, participantId)
def request_participant_image(self, participant_id: str):
if participant_id in self._video_renderers:
self._video_renderers[participant_id]["render_next_frame"] = True
def send_app_message(self, message: Any, participant_id: str | None):
self.client.send_app_message(message, participant_id)
def read_audio_frames(self, desired_frame_count):
bytes = b""
@@ -302,6 +306,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
self._video_renderers[participant_id] = {
"framerate": framerate,
"timestamp": 0,
"render_next_frame": False,
}
self.client.set_video_renderer(
participant_id,
@@ -310,17 +315,28 @@ class DailyTransport(ThreadedTransport, EventHandler):
color_format=color_format)
def on_participant_video_frame(self, participant_id, video_frame):
if not self._loop:
return
render_frame = False
curr_time = time.time()
prev_time = self._video_renderers[participant_id]["timestamp"]
diff_time = curr_time - prev_time
period = 1 / self._video_renderers[participant_id]["framerate"]
if diff_time > period and self._loop:
self._video_renderers[participant_id]["timestamp"] = curr_time
framerate = self._video_renderers[participant_id]["framerate"]
if framerate > 0:
prev_time = self._video_renderers[participant_id]["timestamp"]
next_time = prev_time + 1 / framerate
render_frame = curr_time > next_time
elif self._video_renderers[participant_id]["render_next_frame"]:
self._video_renderers[participant_id]["render_next_frame"] = False
render_frame = True
if render_frame:
frame = UserImageFrame(participant_id, video_frame.buffer,
(video_frame.width, video_frame.height))
asyncio.run_coroutine_threadsafe(
self.receive_queue.put(frame), self._loop
)
asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self._loop)
self._video_renderers[participant_id]["timestamp"] = curr_time
def on_error(self, error):
self._logger.error(f"on_error: {error}")

12
src/dailyai/transports/threaded_transport.py
View File

@@ -20,6 +20,7 @@ from dailyai.pipeline.frames import (
SpriteFrame,
StartFrame,
TextFrame,
UserImageRequestFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
@@ -382,7 +383,11 @@ class ThreadedTransport(AbstractTransport):
def _set_images(self, images: list[bytes], start_frame=0):
self._images = itertools.cycle(images)
def send_app_message(self, message: Any, participantId: str | None):
def request_participant_image(self, participant_id: str):
""" Child classes should override this to force an image from a user. """
pass
def send_app_message(self, message: Any, participant_id: str | None):
""" Child classes should override this to send a custom message to the room. """
pass
@@ -458,9 +463,10 @@ class ThreadedTransport(AbstractTransport):
self._set_image(frame.image)
elif isinstance(frame, SpriteFrame):
self._set_images(frame.images)
elif isinstance(frame, UserImageRequestFrame):
self.request_participant_image(frame.user_id)
elif isinstance(frame, SendAppMessageFrame):
self.send_app_message(
frame.message, frame.participantId)
self.send_app_message(frame.message, frame.participant_id)
elif len(b):
self.write_frame_to_mic(bytes(b))
b = bytearray()