Update sample 5!

src/dailyai/pipeline/aggregators.py

@@ -9,6 +9,7 @@ from dailyai.pipeline.frames import (
     EndParallelPipeQueueFrame,
     EndStreamQueueFrame,
     LLMMessagesQueueFrame,
+    LLMResponseEndQueueFrame,
     QueueFrame,
     TextQueueFrame,
     TranscriptionQueueFrame,
@@ -16,7 +17,7 @@ from dailyai.pipeline.frames import (
 from dailyai.pipeline.pipeline import Pipeline
 from dailyai.services.ai_services import AIService
 
-from typing import AsyncGenerator, Coroutine, List
+from typing import AsyncGenerator, Coroutine, List, Text
 
 
 class LLMContextAggregator(AIService):
@@ -122,6 +123,23 @@ class SentenceAggregator(FrameProcessor):
             yield frame
 
 
+class LLMFullResponseAggregator(FrameProcessor):
+    def __init__(self):
+        self.aggregation = ""
+
+    async def process_frame(
+        self, frame: QueueFrame
+    ) -> AsyncGenerator[QueueFrame, None]:
+        if isinstance(frame, TextQueueFrame):
+            self.aggregation += frame.text
+        elif isinstance(frame, LLMResponseEndQueueFrame):
+            yield TextQueueFrame(self.aggregation)
+            self.aggregation = ""
+        else:
+            yield frame
+
+
+
 class StatelessTextTransformer(FrameProcessor):
     def __init__(self, transform_fn):
         self.transform_fn = transform_fn
@@ -158,7 +176,7 @@ class ParallelPipeline(FrameProcessor):
             if not isinstance(frame, EndParallelPipeQueueFrame):
                 yield frame
 
-class GatedAccumulator(FrameProcessor):
+class GatedAggregator(FrameProcessor):
     def __init__(self, gate_open_fn, gate_close_fn, start_open):
         self.gate_open_fn = gate_open_fn
         self.gate_close_fn = gate_close_fn

src/dailyai/services/ai_services.py

@@ -12,6 +12,7 @@ from dailyai.pipeline.frames import (
     ImageQueueFrame,
     LLMMessagesQueueFrame,
     LLMResponseEndQueueFrame,
+    LLMResponseStartQueueFrame,
     QueueFrame,
     TextQueueFrame,
     TranscriptionQueueFrame,
@@ -78,6 +79,7 @@ class LLMService(AIService):
 
     async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
         if isinstance(frame, LLMMessagesQueueFrame):
+            yield LLMResponseStartQueueFrame()
             async for text_chunk in self.run_llm_async(frame.messages):
                 yield TextQueueFrame(text_chunk)
             yield LLMResponseEndQueueFrame()

src/dailyai/tests/test_aggregators.py

@@ -3,7 +3,7 @@ import functools
 import unittest
 
 from dailyai.pipeline.aggregators import (
-    GatedAccumulator,
+    GatedAggregator,
     ParallelPipeline,
     SentenceAggregator,
     StatelessTextTransformer,
@@ -43,7 +43,7 @@ class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase):
         self.assertEqual(expected_sentences, [])
 
     async def test_gated_accumulator(self):
-        gated_accumulator = GatedAccumulator(
+        gated_aggregator = GatedAggregator(
             gate_open_fn=lambda frame: isinstance(frame, ImageQueueFrame),
             gate_close_fn=lambda frame: isinstance(frame, LLMResponseStartQueueFrame),
             start_open=False,
@@ -69,7 +69,7 @@ class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase):
             LLMResponseEndQueueFrame(),
         ]
         for frame in frames:
-            async for out_frame in gated_accumulator.process_frame(frame):
+            async for out_frame in gated_aggregator.process_frame(frame):
                 self.assertEqual(out_frame, expected_output_frames.pop(0))
         self.assertEqual(expected_output_frames, [])
 

src/examples/foundational/05-sync-speech-and-image.py

@@ -1,8 +1,11 @@
 import asyncio
+from re import S
 import aiohttp
 import os
+from dailyai.pipeline.aggregators import GatedAggregator, LLMFullResponseAggregator, ParallelPipeline, SentenceAggregator
 
-from dailyai.pipeline.frames import AudioQueueFrame, ImageQueueFrame
+from dailyai.pipeline.frames import AudioQueueFrame, EndStreamQueueFrame, ImageQueueFrame, LLMMessagesQueueFrame, LLMResponseStartQueueFrame
+from dailyai.pipeline.pipeline import Pipeline
 from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.services.daily_transport_service import DailyTransportService
@@ -35,98 +38,54 @@ async def main(room_url):
             aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
             voice_id="ErXwobaYiN019PkySvjV")
-        # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
 
         dalle = FalImageGenService(
             image_size="1024x1024",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"))
-        # dalle = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
-        # dalle = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
 
-        # Get a complete audio chunk from the given text. Splitting this into its own
+        source_queue = asyncio.Queue()
-        # coroutine lets us ensure proper ordering of the audio chunks on the send queue.
-        async def get_all_audio(text):
-            all_audio = bytearray()
-            async for audio in tts.run_tts(text):
-                all_audio.extend(audio)
 
-            return all_audio
+        for month in ["January", "February"]:
-
-        async def get_month_data(month):
             messages = [
                 {
                     "role": "system",
                     "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
                 }
             ]
+            await source_queue.put(LLMMessagesQueueFrame(messages))
 
-            image_description = await llm.run_llm(messages)
+        await source_queue.put(EndStreamQueueFrame())
-            if not image_description:
-                return
 
-            to_speak = f"{month}: {image_description}"
+        gated_aggregator = GatedAggregator(
-            audio_task = asyncio.create_task(get_all_audio(to_speak))
+            gate_open_fn=lambda frame: isinstance(frame, ImageQueueFrame),
-            image_task = asyncio.create_task(dalle.run_image_gen(image_description))
+            gate_close_fn=lambda frame: isinstance(frame, LLMResponseStartQueueFrame),
-            print(f"about to gather tasks for {month}")
+            start_open=False,
-            (audio, image_data) = await asyncio.gather(
+        )
-                audio_task, image_task
+
-            )
+        sentence_aggregator = SentenceAggregator()
-            print(f"about to return from get_month_data for {month}")
+        llm_full_response_aggregator = LLMFullResponseAggregator()
-            return {
+
-                "month": month,
+        pipeline = Pipeline(
-                "text": image_description,
+            source=source_queue,
-                "image_url": image_data[0],
+            sink=transport.send_queue,
-                "image": image_data[1],
+            processors=[
-                "audio": audio,
+                llm,
-            }
+                sentence_aggregator,
+                ParallelPipeline([[tts], [llm_full_response_aggregator, dalle]]),
+                gated_aggregator,
+            ],
+        )
+        pipeline_task = pipeline.run_pipeline()
 
-        months: list[str] = [
-            "January",
-            "February",
-            "March",
-            "April",
-            "May",
-            "June"
-        ]
-        """
-        "February",
-        "March",
-        "April",
-        "May",
-        "June",
-        "July",
-        "August",
-        "September",
-        "October",
-        "November",
-        "December",
-        """
         @transport.event_handler("on_first_other_participant_joined")
         async def on_first_other_participant_joined(transport):
-            # This will play the months in the order they're completed. The benefit
+            await pipeline_task
-            # is we'll have as little delay as possible before the first month, and
-            # likely no delay between months, but the months won't display in order.
-            for month_data_task in asyncio.as_completed(month_tasks):
-                print(f"month_data_task: {month_data_task}")
-                try:
-                    data = await month_data_task
-                except Exception:
-                    print("OMG EXCEPTION!!!!")
-                if data:
-                    await transport.send_queue.put(
-                        [
-                            ImageQueueFrame(data["image_url"], data["image"]),
-                            AudioQueueFrame(data["audio"]),
-                        ]
-                    )
 
             # wait for the output queue to be empty, then leave the meeting
             await transport.stop_when_done()
 
-        month_tasks = [asyncio.create_task(get_month_data(month)) for month in months]
-
         await transport.run()
 
 if __name__ == "__main__":