initial commit for new pipecat architecture

examples/foundational/12-describe-video.py

@@ -1,38 +1,50 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
 import asyncio
 import aiohttp
-import logging
 import os
+import sys
 
-from typing import AsyncGenerator
-
-from dailyai.pipeline.aggregators import FrameProcessor, UserResponseAggregator, VisionImageFrameAggregator
-
-from dailyai.pipeline.frames import Frame, TextFrame, UserImageRequestFrame
-from dailyai.pipeline.pipeline import Pipeline
-from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-from dailyai.services.moondream_ai_service import MoondreamService
-from dailyai.transports.daily_transport import DailyTransport
+from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.aggregators.user_response import UserResponseAggregator
+from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.moondream import MoondreamService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+from pipecat.vad.silero import SileroVAD
 
 from runner import configure
 
+from loguru import logger
+
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("dailyai")
-logger.setLevel(logging.DEBUG)
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
 
 
 class UserImageRequester(FrameProcessor):
-    participant_id: str
+
+    def __init__(self, participant_id: str | None = None):
+        super().__init__()
+        self._participant_id = participant_id
 
     def set_participant_id(self, participant_id: str):
-        self.participant_id = participant_id
+        self._participant_id = participant_id
 
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if self.participant_id and isinstance(frame, TextFrame):
-            yield UserImageRequestFrame(self.participant_id)
-        yield frame
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if self._participant_id and isinstance(frame, TextFrame):
+            await self.push_frame(UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM)
+        await self.push_frame(frame, direction)
 
 
 async def main(room_url: str, token):
@@ -41,14 +53,15 @@ async def main(room_url: str, token):
             room_url,
             token,
             "Describe participant video",
-            duration_minutes=5,
-            mic_enabled=True,
-            mic_sample_rate=16000,
-            vad_enabled=True,
-            start_transcription=True,
-            video_rendering_enabled=True
+            DailyParams(
+                audio_in_enabled=True,  # This is so Silero VAD can get audio data
+                audio_out_enabled=True,
+                transcription_enabled=True
+            )
         )
 
+        vad = SileroVAD()
+
         tts = ElevenLabsTTSService(
             aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
@@ -70,15 +83,21 @@ async def main(room_url: str, token):
             voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
         )
 
-        @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
-            await transport.say("Hi there! Feel free to ask me what I see.", tts)
-            transport.render_participant_video(participant["id"], framerate=0)
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            await tts.say("Hi there! Feel free to ask me what I see.")
+            transport.capture_participant_video(participant["id"], framerate=0)
+            transport.capture_participant_transcription(participant["id"])
             image_requester.set_participant_id(participant["id"])
 
-        pipeline = Pipeline([user_response, image_requester, vision_aggregator, moondream, tts])
+        pipeline = Pipeline([transport.input(), vad, user_response, image_requester,
+                             vision_aggregator, moondream, tts, transport.output()])
 
-        await transport.run(pipeline)
+        task = PipelineTask(pipeline)
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
 
 if __name__ == "__main__":
     (url, token) = configure()