Working vision example

src/dailyai/services/ai_services.py

@@ -148,6 +148,8 @@ class VisionService(AIService):
         if isinstance(frame, VisionFrame):
             async for frame in self.run_vision(frame.prompt, frame.image):
                 yield frame
+        else:
+            yield frame
 
 
 class FrameLogger(AIService):

src/dailyai/services/daily_transport_service.py

@@ -230,14 +230,12 @@ class DailyTransportService(BaseTransportService, EventHandler):
         self.client.release()
 
     def _handle_video_frame(self, participant_id, video_frame):
-        # TODO-CB: What about multiple participants?
         if (not participant_id in self._participant_frame_times) or (time.time() > self._participant_frame_times[participant_id] + 1.0/self._receive_video_fps):
-            print(f"### sending frame now")
             self._participant_frame_times[participant_id] = time.time()
-            asyncio.run_coroutine_threadsafe(
+
+            future = asyncio.run_coroutine_threadsafe(
                 self.receive_queue.put(
-                    VideoImageFrame(participant_id, video_frame)), self._loop
-            )
+                    VideoImageFrame(participant_id, video_frame)), self._loop)
 
     def on_first_other_participant_joined(self):
         pass

src/dailyai/services/elevenlabs_ai_service.py

@@ -35,6 +35,7 @@ class ElevenLabsTTSService(TTSService):
             "xi-api-key": self._api_key,
             "Content-Type": "application/json",
         }
+
         async with self._aiohttp_session.post(
             url, json=payload, headers=headers, params=querystring
         ) as r:

src/dailyai/services/open_ai_services.py

@@ -71,7 +71,24 @@ class OpenAIVisionService(VisionService):
         self._client = AsyncOpenAI(api_key=api_key)
 
     async def run_vision(self, prompt: str, image: bytes):
-        base64_image = base64.b64encode(image).decode('utf-8')
+        IMAGE_WIDTH = image.width
+        IMAGE_HEIGHT = image.height
+        COLOR_FORMAT = image.color_format
+        a_image = Image.frombytes(
+            'RGBA', (IMAGE_WIDTH, IMAGE_HEIGHT), image.buffer)
+        new_image = a_image.convert('RGB')
+
+        # Uncomment these lines to write the frame to a jpg in the same directory.
+        # current_path = os.getcwd()
+        # image_path = os.path.join(current_path, "image.jpg")
+        # image.save(image_path, format="JPEG")
+
+        jpeg_buffer = io.BytesIO()
+
+        new_image.save(jpeg_buffer, format='JPEG')
+
+        jpeg_bytes = jpeg_buffer.getvalue()
+        base64_image = base64.b64encode(jpeg_bytes).decode('utf-8')
         messages = [
             {
                 "role": "user",
@@ -94,5 +111,7 @@ class OpenAIVisionService(VisionService):
             )
         )
         async for chunk in chunks:
-            print(f"!!! chunk: {chunk}")
-            yield TextFrame(chunk)
+            if len(chunk.choices) == 0:
+                continue
+            if chunk.choices[0].delta.content:
+                yield TextFrame(chunk.choices[0].delta.content)