From a5eba0106bea2a6157b0f99f25bab04b5e82e985 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Tue, 9 Apr 2024 22:16:05 -0700
Subject: [PATCH] transport: allow requesting a user frame

---
 src/dailyai/transports/daily_transport.py    | 36 ++++++++++++++------
 src/dailyai/transports/threaded_transport.py | 12 +++++--
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/src/dailyai/transports/daily_transport.py b/src/dailyai/transports/daily_transport.py
index 3adbf57a2..9d32d501b 100644
--- a/src/dailyai/transports/daily_transport.py
+++ b/src/dailyai/transports/daily_transport.py
@@ -169,8 +169,12 @@ class DailyTransport(ThreadedTransport, EventHandler):
         if self._mic_enabled:
             self.mic.write_frames(frame)
 
-    def send_app_message(self, message: Any, participantId: str | None):
-        self.client.send_app_message(message, participantId)
+    def request_participant_image(self, participant_id: str):
+        if participant_id in self._video_renderers:
+            self._video_renderers[participant_id]["render_next_frame"] = True
+
+    def send_app_message(self, message: Any, participant_id: str | None):
+        self.client.send_app_message(message, participant_id)
 
     def read_audio_frames(self, desired_frame_count):
         bytes = b""
@@ -302,6 +306,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
         self._video_renderers[participant_id] = {
             "framerate": framerate,
             "timestamp": 0,
+            "render_next_frame": False,
         }
         self.client.set_video_renderer(
             participant_id,
@@ -310,17 +315,28 @@ class DailyTransport(ThreadedTransport, EventHandler):
             color_format=color_format)
 
     def on_participant_video_frame(self, participant_id, video_frame):
+        if not self._loop:
+            return
+
+        render_frame = False
+
         curr_time = time.time()
-        prev_time = self._video_renderers[participant_id]["timestamp"]
-        diff_time = curr_time - prev_time
-        period = 1 / self._video_renderers[participant_id]["framerate"]
-        if diff_time > period and self._loop:
-            self._video_renderers[participant_id]["timestamp"] = curr_time
+        framerate = self._video_renderers[participant_id]["framerate"]
+
+        if framerate > 0:
+            prev_time = self._video_renderers[participant_id]["timestamp"]
+            next_time = prev_time + 1 / framerate
+            render_frame = curr_time > next_time
+        elif self._video_renderers[participant_id]["render_next_frame"]:
+            self._video_renderers[participant_id]["render_next_frame"] = False
+            render_frame = True
+
+        if render_frame:
             frame = UserImageFrame(participant_id, video_frame.buffer,
                                    (video_frame.width, video_frame.height))
-            asyncio.run_coroutine_threadsafe(
-                self.receive_queue.put(frame), self._loop
-            )
+            asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self._loop)
+
+        self._video_renderers[participant_id]["timestamp"] = curr_time
 
     def on_error(self, error):
         self._logger.error(f"on_error: {error}")
diff --git a/src/dailyai/transports/threaded_transport.py b/src/dailyai/transports/threaded_transport.py
index 334bffce3..52736a008 100644
--- a/src/dailyai/transports/threaded_transport.py
+++ b/src/dailyai/transports/threaded_transport.py
@@ -20,6 +20,7 @@ from dailyai.pipeline.frames import (
     SpriteFrame,
     StartFrame,
     TextFrame,
+    UserImageRequestFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
@@ -382,7 +383,11 @@ class ThreadedTransport(AbstractTransport):
     def _set_images(self, images: list[bytes], start_frame=0):
         self._images = itertools.cycle(images)
 
-    def send_app_message(self, message: Any, participantId: str | None):
+    def request_participant_image(self, participant_id: str):
+        """ Child classes should override this to force an image from a user. """
+        pass
+
+    def send_app_message(self, message: Any, participant_id: str | None):
         """ Child classes should override this to send a custom message to the room. """
         pass
 
@@ -458,9 +463,10 @@ class ThreadedTransport(AbstractTransport):
                                 self._set_image(frame.image)
                             elif isinstance(frame, SpriteFrame):
                                 self._set_images(frame.images)
+                            elif isinstance(frame, UserImageRequestFrame):
+                                self.request_participant_image(frame.user_id)
                             elif isinstance(frame, SendAppMessageFrame):
-                                self.send_app_message(
-                                    frame.message, frame.participantId)
+                                self.send_app_message(frame.message, frame.participant_id)
                         elif len(b):
                             self.write_frame_to_mic(bytes(b))
                             b = bytearray()