added VisionImageFrame and VisionImageFrameAggregator

src/dailyai/services/ai_services.py

@@ -15,6 +15,7 @@ from dailyai.pipeline.frames import (
     TextFrame,
     TranscriptionFrame,
     URLImageFrame,
+    VisionImageFrame,
 )
 
 from abc import abstractmethod
@@ -108,19 +109,13 @@ class VisionService(AIService):
         self._describe_text = None
 
     @abstractmethod
-    async def run_vision(self, describe_text: str, frame: ImageFrame) -> str:
+    async def run_vision(self, frame: VisionImageFrame) -> str:
         pass
 
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, TextFrame):
-            self._describe_text = frame.text
-        elif isinstance(frame, ImageFrame):
-            if self._describe_text:
-                description = await self.run_vision(self._describe_text, frame)
-                self._describe_text = None
-                yield TextFrame(description)
-            else:
-                yield frame
+        if isinstance(frame, VisionImageFrame):
+            description = await self.run_vision(frame)
+            yield TextFrame(description)
         else:
             yield frame
 

src/dailyai/services/moondream_ai_service.py

@@ -1,4 +1,4 @@
-from dailyai.pipeline.frames import ImageFrame
+from dailyai.pipeline.frames import ImageFrame, VisionImageFrame
 from dailyai.services.ai_services import VisionService
 
 from PIL import Image
@@ -42,11 +42,11 @@ class MoondreamService(VisionService):
         ).to(device=device, dtype=dtype)
         self._model.eval()
 
-    async def run_vision(self, describe_text: str, frame: ImageFrame) -> str:
+    async def run_vision(self, frame: VisionImageFrame) -> str:
         image = Image.frombytes("RGB", (frame.size[0], frame.size[1]), frame.image)
         image_embeds = self._model.encode_image(image)
         description = self._model.answer_question(
             image_embeds=image_embeds,
-            question=describe_text,
+            question=frame.text,
             tokenizer=self._tokenizer)
         return description