Merge pull request #50 from daily-co/khk/launch-samples

Khk/launch samples

src/dailyai/services/fal_ai_services.py

@@ -9,17 +9,19 @@ from dailyai.services.ai_services import ImageGenService
 
 
 from dailyai.services.ai_services import ImageGenService
+
 # Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
 
 
 class FalImageGenService(ImageGenService):
     def __init__(
-            self,
+        self,
-            *,
+        *,
-            image_size,
+        image_size,
-            aiohttp_session: aiohttp.ClientSession,
+        aiohttp_session: aiohttp.ClientSession,
-            key_id=None,
+        key_id=None,
-            key_secret=None):
+        key_secret=None
+    ):
         super().__init__(image_size)
         self._aiohttp_session = aiohttp_session
         if key_id:
@@ -30,10 +32,9 @@ class FalImageGenService(ImageGenService):
     async def run_image_gen(self, sentence) -> tuple[str, bytes]:
         def get_image_url(sentence, size):
             handler = fal.apps.submit(
-                "110602490-fast-sdxl",
+                # "110602490-fast-sdxl",
-                arguments={
+                "fal-ai/fast-sdxl",
-                    "prompt": sentence
+                arguments={"prompt": sentence},
-                },
             )
             for event in handler.iter_events():
                 if isinstance(event, fal.apps.InProgress):
@@ -46,6 +47,7 @@ class FalImageGenService(ImageGenService):
                 raise Exception("Image generation failed")
 
             return image_url
+
         image_url = await asyncio.to_thread(get_image_url, sentence, self.image_size)
         # Load the image from the url
         async with self._aiohttp_session.get(image_url) as response:

src/examples/foundational/01-say-one-thing.py

@@ -5,7 +5,6 @@ import os
 
 from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-from dailyai.services.playht_ai_service import PlayHTAIService
 
 from examples.support.runner import configure
 
@@ -13,19 +12,14 @@ logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
 logger.setLevel(logging.DEBUG)
 
+
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        # create a transport service object using environment variables for
-        # the transport service's API key, room url, and any other configuration.
-        # services can all define and document the environment variables they use.
-        # services all also take an optional config object that is used instead of
-        # environment variables.
-        #
-        # the abstract transport service APIs presumably can map pretty closely
-        # to the daily-python basic API
-        meeting_duration_minutes = 5
         transport = DailyTransportService(
-            room_url, None, "Say One Thing", meeting_duration_minutes, mic_enabled=True
+            room_url,
+            None,
+            "Say One Thing",
+            mic_enabled=True,
         )
 
         tts = ElevenLabsTTSService(
@@ -37,7 +31,6 @@ async def main(room_url):
         # Register an event handler so we can play the audio when the participant joins.
         @transport.event_handler("on_participant_joined")
         async def on_participant_joined(transport, participant):
-            nonlocal tts
             if participant["info"]["isLocal"]:
                 return
 

src/examples/foundational/02-llm-say-one-thing.py

@@ -6,24 +6,22 @@ import aiohttp
 
 from dailyai.pipeline.frames import LLMMessagesQueueFrame
 from dailyai.services.daily_transport_service import DailyTransportService
-from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-from dailyai.services.deepgram_ai_services import DeepgramTTSService
 from dailyai.services.open_ai_services import OpenAILLMService
+
 from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
 logger.setLevel(logging.DEBUG)
 
+
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        meeting_duration_minutes = 1
         transport = DailyTransportService(
             room_url,
             None,
             "Say One Thing From an LLM",
-            duration_minutes=meeting_duration_minutes,
             mic_enabled=True,
         )
 
@@ -32,25 +30,24 @@ async def main(room_url):
             api_key=os.getenv("ELEVENLABS_API_KEY"),
             voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
         )
+
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
         )
+
         messages = [
             {
                 "role": "system",
                 "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
             }
         ]
-        tts_task = asyncio.create_task(
-            tts.run_to_queue(
-                transport.send_queue,
-                llm.run([LLMMessagesQueueFrame(messages)]),
-            )
-        )
 
         @transport.event_handler("on_first_other_participant_joined")
         async def on_first_other_participant_joined(transport):
-            await tts_task
+            await tts.run_to_queue(
+                transport.send_queue,
+                llm.run([LLMMessagesQueueFrame(messages)]),
+            )
             await transport.stop_when_done()
 
         await transport.run()

src/examples/foundational/03-still-frame.py

@@ -6,48 +6,37 @@ import os
 from dailyai.pipeline.frames import TextFrame
 from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.fal_ai_services import FalImageGenService
-from dailyai.services.open_ai_services import OpenAIImageGenService
-from dailyai.services.azure_ai_services import AzureImageGenServiceREST
 
 from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
 logger.setLevel(logging.DEBUG)
-local_joined = False
-participant_joined = False
 
 
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        meeting_duration_minutes = 1
         transport = DailyTransportService(
             room_url,
             None,
             "Show a still frame image",
-            duration_minutes=meeting_duration_minutes,
-            mic_enabled=False,
             camera_enabled=True,
             camera_width=1024,
             camera_height=1024,
         )
 
         imagegen = FalImageGenService(
-            image_size="1024x1024",
+            image_size="square_hd",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"),
         )
 
-        image_task = asyncio.create_task(
-            imagegen.run_to_queue(
-                transport.send_queue, [TextFrame("a cat in the style of picasso")]
-            )
-        )
-
         @transport.event_handler("on_first_other_participant_joined")
         async def on_first_other_participant_joined(transport):
-            await image_task
+            await imagegen.run_to_queue(
+                transport.send_queue, [TextFrame("a cat in the style of picasso")]
+            )
 
         await transport.run()
 

src/examples/foundational/04-utterance-and-speech.py

@@ -7,6 +7,7 @@ from dailyai.pipeline.pipeline import Pipeline
 
 from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
 from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from examples.support.runner import configure
@@ -25,7 +26,6 @@ async def main(room_url: str):
             duration_minutes=1,
             mic_enabled=True,
             mic_sample_rate=16000,
-            camera_enabled=False,
         )
 
         llm = AzureLLMService(
@@ -37,6 +37,11 @@ async def main(room_url: str):
             api_key=os.getenv("AZURE_SPEECH_API_KEY"),
             region=os.getenv("AZURE_SPEECH_REGION"),
         )
+
+        deepgram_tts = DeepgramTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("DEEPGRAM_API_KEY"),
+        )
         elevenlabs_tts = ElevenLabsTTSService(
             aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
@@ -64,6 +69,15 @@ async def main(room_url: str):
                 transport.send_queue,
             )
 
+            # khk: deepgram_tts.say() doesn't seem to put bytes in the transport
+            # queue. I get a debug log line that indicates we're set up okay, but
+            # no further log lines or audio bytes. debug this later:
+            #   20 2024-03-10 13:24:46,235 Running deepgram tts for My friend the LLM is now going to tell a joke about llamas.
+            # await deepgram_tts.say(
+            #    "My friend the LLM is now going to tell a joke about llamas.",
+            #    transport.send_queue,
+            # )
+
             async def buffer_to_send_queue():
                 while True:
                     frame = await buffer_queue.get()
@@ -73,7 +87,6 @@ async def main(room_url: str):
                         break
 
             await asyncio.gather(pipeline_run_task, buffer_to_send_queue())
-
             await transport.stop_when_done()
 
         await transport.run()

src/examples/foundational/05-sync-speech-and-image.py

@@ -4,6 +4,9 @@ import aiohttp
 import os
 import logging
 
+from dataclasses import dataclass
+from typing import AsyncGenerator
+
 from dailyai.pipeline.aggregators import (
     GatedAggregator,
     LLMFullResponseAggregator,
@@ -11,17 +14,20 @@ from dailyai.pipeline.aggregators import (
     SentenceAggregator,
 )
 from dailyai.pipeline.frames import (
-    AudioFrame,
+    Frame,
+    TextFrame,
     EndFrame,
     ImageFrame,
     LLMMessagesQueueFrame,
     LLMResponseStartFrame,
 )
+from dailyai.pipeline.frame_processor import FrameProcessor
+
 from dailyai.pipeline.pipeline import Pipeline
-from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.services.daily_transport_service import DailyTransportService
-from dailyai.services.fal_ai_services import FalImageGenService
 from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
 
 from examples.support.runner import configure
 
@@ -30,14 +36,35 @@ logger = logging.getLogger("dailyai")
 logger.setLevel(logging.DEBUG)
 
 
+@dataclass
+class MonthFrame(Frame):
+    month: str
+
+
+class MonthPrepender(FrameProcessor):
+    def __init__(self):
+        self.most_recent_month = "Placeholder, month frame not yet received"
+        self.prepend_to_next_text_frame = False
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, MonthFrame):
+            self.most_recent_month = frame.month
+        elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame):
+            yield TextFrame(f"{self.most_recent_month}: {frame.text}")
+            self.prepend_to_next_text_frame = False
+        elif isinstance(frame, LLMResponseStartFrame):
+            self.prepend_to_next_text_frame = True
+            yield frame
+        else:
+            yield frame
+
+
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        meeting_duration_minutes = 5
         transport = DailyTransportService(
             room_url,
             None,
             "Month Narration Bot",
-            duration_minutes=meeting_duration_minutes,
             mic_enabled=True,
             camera_enabled=True,
             mic_sample_rate=16000,
@@ -55,8 +82,8 @@ async def main(room_url):
             api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
         )
 
-        dalle = FalImageGenService(
+        imagegen = FalImageGenService(
-            image_size="1024x1024",
+            image_size="square_hd",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"),
@@ -84,6 +111,7 @@ async def main(room_url):
                     "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
                 }
             ]
+            await source_queue.put(MonthFrame(month))
             await source_queue.put(LLMMessagesQueueFrame(messages))
 
         await source_queue.put(EndFrame())
@@ -95,6 +123,7 @@ async def main(room_url):
         )
 
         sentence_aggregator = SentenceAggregator()
+        month_prepender = MonthPrepender()
         llm_full_response_aggregator = LLMFullResponseAggregator()
 
         pipeline = Pipeline(
@@ -103,7 +132,9 @@ async def main(room_url):
             processors=[
                 llm,
                 sentence_aggregator,
-                ParallelPipeline([[tts], [llm_full_response_aggregator, dalle]]),
+                ParallelPipeline(
+                    [[month_prepender, tts], [llm_full_response_aggregator, imagegen]]
+                ),
                 gated_aggregator,
             ],
         )
@@ -112,8 +143,6 @@ async def main(room_url):
         @transport.event_handler("on_first_other_participant_joined")
         async def on_first_other_participant_joined(transport):
             await pipeline_task
-
-            # wait for the output queue to be empty, then leave the meeting
             await transport.stop_when_done()
 
         await transport.run()