move src/examples to examples

examples/foundational/01-say-one-thing.py

@@ -0,0 +1,51 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from dailyai.pipeline.frames import EndFrame, TextFrame
+from dailyai.pipeline.pipeline import Pipeline
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            None,
+            "Say One Thing",
+            mic_enabled=True,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        pipeline = Pipeline([tts])
+
+        # Register an event handler so we can play the audio when the
+        # participant joins.
+        @transport.event_handler("on_participant_joined")
+        async def on_participant_joined(transport, participant):
+            if participant["info"]["isLocal"]:
+                return
+
+            participant_name = participant["info"]["userName"] or ''
+            await pipeline.queue_frames([TextFrame("Hello there, " + participant_name + "!"), EndFrame()])
+
+        await transport.run(pipeline)
+        del tts
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/01a-local-transport.py

@@ -0,0 +1,38 @@
+import asyncio
+import aiohttp
+import logging
+import os
+
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.local_transport_service import LocalTransportService
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        meeting_duration_minutes = 1
+        transport = LocalTransportService(
+            duration_minutes=meeting_duration_minutes, mic_enabled=True
+        )
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        async def say_something():
+            await asyncio.sleep(1)
+            await tts.say(
+                "Hello there.",
+                transport.send_queue,
+            )
+            await transport.stop_when_done()
+
+        await asyncio.gather(transport.run(), say_something())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/foundational/02-llm-say-one-thing.py

@@ -0,0 +1,56 @@
+import asyncio
+import os
+import logging
+
+import aiohttp
+
+from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.open_ai_services import OpenAILLMService
+
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            None,
+            "Say One Thing From an LLM",
+            mic_enabled=True,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        messages = [
+            {
+                "role": "system",
+                "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
+            }]
+
+        pipeline = Pipeline([llm, tts])
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndFrame()])
+
+        await transport.run(pipeline)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/03-still-frame.py

@@ -0,0 +1,54 @@
+import asyncio
+import aiohttp
+import logging
+import os
+
+from dailyai.pipeline.frames import EndFrame, TextFrame
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.fal_ai_services import FalImageGenService
+
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            None,
+            "Show a still frame image",
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+            duration_minutes=1
+        )
+
+        imagegen = FalImageGenService(
+            image_size="square_hd",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+
+        pipeline = Pipeline([imagegen])
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            # Note that we do not put an EndFrame() item in the pipeline for this demo.
+            # This means that the bot will stay in the channel until it times out.
+            # An EndFrame() in the pipeline would cause the transport to shut
+            # down.
+            await pipeline.queue_frames(
+                [TextFrame("a cat in the style of picasso")]
+            )
+
+        await transport.run(pipeline)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/03a-image-local.py

@@ -0,0 +1,55 @@
+import asyncio
+import aiohttp
+import logging
+import os
+
+import tkinter as tk
+
+from dailyai.pipeline.frames import TextFrame
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.services.local_transport_service import LocalTransportService
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+local_joined = False
+participant_joined = False
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        meeting_duration_minutes = 2
+        tk_root = tk.Tk()
+        tk_root.title("Calendar")
+        transport = LocalTransportService(
+            tk_root=tk_root,
+            mic_enabled=True,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+            duration_minutes=meeting_duration_minutes,
+        )
+
+        imagegen = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+        image_task = asyncio.create_task(
+            imagegen.run_to_queue(
+                transport.send_queue, [
+                    TextFrame("a cat in the style of picasso")]))
+
+        async def run_tk():
+            while not transport._stop_threads.is_set():
+                tk_root.update()
+                tk_root.update_idletasks()
+                await asyncio.sleep(0.1)
+
+        await asyncio.gather(transport.run(), image_task, run_tk())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/foundational/04-utterance-and-speech.py

@@ -0,0 +1,81 @@
+import asyncio
+import logging
+import os
+
+import aiohttp
+from dailyai.pipeline.merge_pipeline import SequentialMergePipeline
+from dailyai.pipeline.pipeline import Pipeline
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
+from dailyai.pipeline.frames import EndFrame, EndPipeFrame, LLMMessagesQueueFrame, TextFrame
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            None,
+            "Static And Dynamic Speech",
+            duration_minutes=1,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+        )
+
+        llm = AzureLLMService(
+            api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
+            endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
+            model=os.getenv("AZURE_CHATGPT_MODEL"),
+        )
+        azure_tts = AzureTTSService(
+            api_key=os.getenv("AZURE_SPEECH_API_KEY"),
+            region=os.getenv("AZURE_SPEECH_REGION"),
+        )
+
+        deepgram_tts = DeepgramTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("DEEPGRAM_API_KEY"),
+        )
+        elevenlabs_tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        messages = [{"role": "system",
+                     "content": "tell the user a joke about llamas"}]
+
+        # Start a task to run the LLM to create a joke, and convert the LLM output to audio frames. This task
+        # will run in parallel with generating and speaking the audio for static text, so there's no delay to
+        # speak the LLM response.
+        llm_pipeline = Pipeline([llm, elevenlabs_tts])
+        await llm_pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndPipeFrame()])
+
+        simple_tts_pipeline = Pipeline([azure_tts])
+        await simple_tts_pipeline.queue_frames(
+            [
+                TextFrame("My friend the LLM is going to tell a joke about llamas"),
+                EndPipeFrame(),
+            ]
+        )
+
+        merge_pipeline = SequentialMergePipeline(
+            [simple_tts_pipeline, llm_pipeline])
+
+        await asyncio.gather(
+            transport.run(merge_pipeline),
+            simple_tts_pipeline.run_pipeline(),
+            llm_pipeline.run_pipeline(),
+        )
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/05-sync-speech-and-image.py

@@ -0,0 +1,144 @@
+import asyncio
+from re import S
+import aiohttp
+import os
+import logging
+
+from dataclasses import dataclass
+from typing import AsyncGenerator
+
+from dailyai.pipeline.aggregators import (
+    GatedAggregator,
+    LLMFullResponseAggregator,
+    ParallelPipeline,
+    SentenceAggregator,
+)
+from dailyai.pipeline.frames import (
+    Frame,
+    TextFrame,
+    EndFrame,
+    ImageFrame,
+    LLMMessagesQueueFrame,
+    LLMResponseStartFrame,
+)
+from dailyai.pipeline.frame_processor import FrameProcessor
+
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+@dataclass
+class MonthFrame(Frame):
+    month: str
+
+
+class MonthPrepender(FrameProcessor):
+    def __init__(self):
+        self.most_recent_month = "Placeholder, month frame not yet received"
+        self.prepend_to_next_text_frame = False
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, MonthFrame):
+            self.most_recent_month = frame.month
+        elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame):
+            yield TextFrame(f"{self.most_recent_month}: {frame.text}")
+            self.prepend_to_next_text_frame = False
+        elif isinstance(frame, LLMResponseStartFrame):
+            self.prepend_to_next_text_frame = True
+            yield frame
+        else:
+            yield frame
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            None,
+            "Month Narration Bot",
+            mic_enabled=True,
+            camera_enabled=True,
+            mic_sample_rate=16000,
+            camera_width=1024,
+            camera_height=1024,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        imagegen = FalImageGenService(
+            image_size="square_hd",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+
+        gated_aggregator = GatedAggregator(
+            gate_open_fn=lambda frame: isinstance(
+                frame, ImageFrame), gate_close_fn=lambda frame: isinstance(
+                frame, LLMResponseStartFrame), start_open=False, )
+
+        sentence_aggregator = SentenceAggregator()
+        month_prepender = MonthPrepender()
+        llm_full_response_aggregator = LLMFullResponseAggregator()
+
+        pipeline = Pipeline(
+            processors=[
+                llm,
+                sentence_aggregator,
+                ParallelPipeline(
+                    [[month_prepender, tts], [llm_full_response_aggregator, imagegen]]
+                ),
+                gated_aggregator,
+            ],
+        )
+
+        frames = []
+        for month in [
+            "January",
+            "February",
+            "March",
+            "April",
+            "May",
+            "June",
+            "July",
+            "August",
+            "September",
+            "October",
+            "November",
+            "December",
+        ]:
+            messages = [
+                {
+                    "role": "system",
+                    "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
+                }
+            ]
+            frames.append(MonthFrame(month))
+            frames.append(LLMMessagesQueueFrame(messages))
+
+        frames.append(EndFrame())
+        await pipeline.queue_frames(frames)
+
+        await transport.run(pipeline, override_pipeline_source_queue=False)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/05a-local-sync-speech-and-text.py

@@ -0,0 +1,146 @@
+import aiohttp
+import argparse
+import asyncio
+import logging
+import tkinter as tk
+import os
+
+from dailyai.pipeline.frames import AudioFrame, ImageFrame
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.services.local_transport_service import LocalTransportService
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        meeting_duration_minutes = 5
+        tk_root = tk.Tk()
+        tk_root.title("Calendar")
+
+        transport = LocalTransportService(
+            mic_enabled=True,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+            duration_minutes=meeting_duration_minutes,
+            tk_root=tk_root,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        dalle = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+
+        # Get a complete audio chunk from the given text. Splitting this into its own
+        # coroutine lets us ensure proper ordering of the audio chunks on the
+        # send queue.
+        async def get_all_audio(text):
+            all_audio = bytearray()
+            async for audio in tts.run_tts(text):
+                all_audio.extend(audio)
+
+            return all_audio
+
+        async def get_month_data(month):
+            messages = [
+                {
+                    "role": "system",
+                    "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
+                }
+            ]
+
+            image_description = await llm.run_llm(messages)
+            if not image_description:
+                return
+
+            to_speak = f"{month}: {image_description}"
+            audio_task = asyncio.create_task(get_all_audio(to_speak))
+            image_task = asyncio.create_task(
+                dalle.run_image_gen(image_description))
+            (audio, image_data) = await asyncio.gather(audio_task, image_task)
+
+            return {
+                "month": month,
+                "text": image_description,
+                "image_url": image_data[0],
+                "image": image_data[1],
+                "audio": audio,
+            }
+
+        months: list[str] = [
+            "January",
+            "February",
+            "March",
+            "April",
+            "May",
+            "June",
+            "July",
+            "August",
+            "September",
+            "October",
+            "November",
+            "December",
+        ]
+
+        async def show_images():
+            # This will play the months in the order they're completed. The benefit
+            # is we'll have as little delay as possible before the first month, and
+            # likely no delay between months, but the months won't display in
+            # order.
+            for month_data_task in asyncio.as_completed(month_tasks):
+                data = await month_data_task
+                if data:
+                    await transport.send_queue.put(
+                        [
+                            ImageFrame(data["image_url"], data["image"]),
+                            AudioFrame(data["audio"]),
+                        ]
+                    )
+
+            await asyncio.sleep(25)
+
+            # wait for the output queue to be empty, then leave the meeting
+            await transport.stop_when_done()
+
+        async def run_tk():
+            while not transport._stop_threads.is_set():
+                tk_root.update()
+                tk_root.update_idletasks()
+                await asyncio.sleep(0.1)
+
+        month_tasks = [
+            asyncio.create_task(
+                get_month_data(month)) for month in months]
+
+        await asyncio.gather(transport.run(), show_images(), run_tk())
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=True,
+        help="URL of the Daily room to join")
+
+    args, unknown = parser.parse_known_args()
+
+    asyncio.run(main(args.url))

examples/foundational/06-listen-and-respond.py

@@ -0,0 +1,85 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from dailyai.pipeline.frames import LLMMessagesQueueFrame
+from dailyai.pipeline.pipeline import Pipeline
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.ai_services import FrameLogger
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMUserContextAggregator,
+)
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Respond bot",
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            vad_enabled=True,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+        fl = FrameLogger("Inner")
+        fl2 = FrameLogger("Outer")
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+            },
+        ]
+
+        tma_in = LLMUserContextAggregator(
+            messages, transport._my_participant_id)
+        tma_out = LLMAssistantContextAggregator(
+            messages, transport._my_participant_id
+        )
+        pipeline = Pipeline(
+            processors=[
+                fl,
+                tma_in,
+                llm,
+                fl2,
+                tts,
+                tma_out,
+            ],
+        )
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            # Kick off the conversation.
+            messages.append(
+                {"role": "system", "content": "Please introduce yourself to the user."})
+            await pipeline.queue_frames([LLMMessagesQueueFrame(messages)])
+
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await transport.run(pipeline)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/06a-image-sync.py

@@ -0,0 +1,122 @@
+import argparse
+import asyncio
+import os
+import logging
+from typing import AsyncGenerator
+import aiohttp
+import requests
+import time
+import urllib.parse
+from PIL import Image
+
+from dailyai.pipeline.frames import ImageFrame, Frame
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.ai_services import AIService
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMUserContextAggregator,
+)
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+class ImageSyncAggregator(AIService):
+    def __init__(self, speaking_path: str, waiting_path: str):
+        self._speaking_image = Image.open(speaking_path)
+        self._speaking_image_bytes = self._speaking_image.tobytes()
+
+        self._waiting_image = Image.open(waiting_path)
+        self._waiting_image_bytes = self._waiting_image.tobytes()
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        yield ImageFrame(None, self._speaking_image_bytes)
+        yield frame
+        yield ImageFrame(None, self._waiting_image_bytes)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Respond bot",
+            5,
+        )
+        transport._camera_enabled = True
+        transport._camera_width = 1024
+        transport._camera_height = 1024
+        transport._mic_enabled = True
+        transport._mic_sample_rate = 16000
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        img = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+
+        async def get_images():
+            get_speaking_task = asyncio.create_task(
+                img.run_image_gen("An image of a cat speaking")
+            )
+            get_waiting_task = asyncio.create_task(
+                img.run_image_gen("An image of a cat waiting")
+            )
+
+            (speaking_data, waiting_data) = await asyncio.gather(
+                get_speaking_task, get_waiting_task
+            )
+
+            return speaking_data, waiting_data
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi, I'm listening!", transport.send_queue)
+
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            image_sync_aggregator = ImageSyncAggregator(
+                os.path.join(
+                    os.path.dirname(__file__), "assets", "speaking.png"), os.path.join(
+                    os.path.dirname(__file__), "assets", "waiting.png"), )
+            await tts.run_to_queue(
+                transport.send_queue,
+                image_sync_aggregator.run(
+                    tma_out.run(llm.run(tma_in.run(transport.get_receive_frames())))
+                ),
+            )
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/07-interruptible.py

@@ -0,0 +1,74 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMResponseAggregator,
+    LLMUserContextAggregator,
+    UserResponseAggregator,
+)
+
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.ai_services import FrameLogger
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Respond bot",
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            vad_enabled=True,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await transport.say("Hi, I'm listening!", tts)
+
+        async def run_conversation():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
+
+            await transport.run_interruptible_pipeline(
+                pipeline,
+                post_processor=LLMResponseAggregator(messages),
+                pre_processor=UserResponseAggregator(messages),
+            )
+
+        transport.transcription_settings["extra"]["punctuate"] = False
+        await asyncio.gather(transport.run(), run_conversation())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/08-bots-arguing.py

@@ -0,0 +1,143 @@
+from typing import Tuple
+import aiohttp
+import asyncio
+import logging
+import os
+from dailyai.pipeline.aggregators import SentenceAggregator
+from dailyai.pipeline.pipeline import Pipeline
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, TextFrame
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            None,
+            "Respond bot",
+            duration_minutes=10,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+        )
+
+        llm = AzureLLMService(
+            api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
+            endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
+            model=os.getenv("AZURE_CHATGPT_MODEL"),
+        )
+        tts1 = AzureTTSService(
+            api_key=os.getenv("AZURE_SPEECH_API_KEY"),
+            region=os.getenv("AZURE_SPEECH_REGION"),
+        )
+        tts2 = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="jBpfuIE2acCO8z3wKNLl",
+        )
+        dalle = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+
+        bot1_messages = [
+            {
+                "role": "system",
+                "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long.",
+            },
+        ]
+        bot2_messages = [
+            {
+                "role": "system",
+                "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich.",
+            },
+        ]
+
+        async def get_text_and_audio(messages) -> Tuple[str, bytearray]:
+            """This function streams text from the LLM and uses the TTS service to convert
+             that text to speech as it's received. """
+            source_queue = asyncio.Queue()
+            sink_queue = asyncio.Queue()
+            sentence_aggregator = SentenceAggregator()
+            pipeline = Pipeline(
+                [llm, sentence_aggregator, tts1], source_queue, sink_queue
+            )
+
+            await source_queue.put(LLMMessagesQueueFrame(messages))
+            await source_queue.put(EndFrame())
+            await pipeline.run_pipeline()
+
+            message = ""
+            all_audio = bytearray()
+            while sink_queue.qsize():
+                frame = sink_queue.get_nowait()
+                if isinstance(frame, TextFrame):
+                    message += frame.text
+                elif isinstance(frame, AudioFrame):
+                    all_audio.extend(frame.data)
+
+            return (message, all_audio)
+
+        async def get_bot1_statement():
+            message, audio = await get_text_and_audio(bot1_messages)
+
+            bot1_messages.append({"role": "assistant", "content": message})
+            bot2_messages.append({"role": "user", "content": message})
+
+            return audio
+
+        async def get_bot2_statement():
+            message, audio = await get_text_and_audio(bot2_messages)
+
+            bot2_messages.append({"role": "assistant", "content": message})
+            bot1_messages.append({"role": "user", "content": message})
+
+            return audio
+
+        async def argue():
+            for i in range(100):
+                print(f"In iteration {i}")
+
+                bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed"
+
+                (audio1, image_data1) = await asyncio.gather(
+                    get_bot1_statement(), dalle.run_image_gen(bot1_description)
+                )
+                await transport.send_queue.put(
+                    [
+                        ImageFrame(None, image_data1[1]),
+                        AudioFrame(audio1),
+                    ]
+                )
+
+                bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed"
+
+                (audio2, image_data2) = await asyncio.gather(
+                    get_bot2_statement(), dalle.run_image_gen(bot2_description)
+                )
+                await transport.send_queue.put(
+                    [
+                        ImageFrame(None, image_data2[1]),
+                        AudioFrame(audio2),
+                    ]
+                )
+
+        await asyncio.gather(transport.run(), argue())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/10-wake-word.py

@@ -0,0 +1,186 @@
+import aiohttp
+import asyncio
+import logging
+import os
+import random
+from typing import AsyncGenerator
+from PIL import Image
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.pipeline.aggregators import (
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
+from dailyai.pipeline.frames import (
+    Frame,
+    TextFrame,
+    ImageFrame,
+    SpriteFrame,
+    TranscriptionQueueFrame,
+)
+from dailyai.services.ai_services import AIService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+sprites = {}
+image_files = [
+    "sc-default.png",
+    "sc-talk.png",
+    "sc-listen-1.png",
+    "sc-think-1.png",
+    "sc-think-2.png",
+    "sc-think-3.png",
+    "sc-think-4.png",
+]
+
+script_dir = os.path.dirname(__file__)
+
+for file in image_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites[file] = img.tobytes()
+
+# When the bot isn't talking, show a static image of the cat listening
+quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
+# When the bot is talking, build an animation from two sprites
+talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
+talking = [random.choice(talking_list) for x in range(30)]
+talking_frame = SpriteFrame(images=talking)
+
+# TODO: Support "thinking" as soon as we get a valid transcript, while LLM
+# is processing
+thinking_list = [
+    sprites["sc-think-1.png"],
+    sprites["sc-think-2.png"],
+    sprites["sc-think-3.png"],
+    sprites["sc-think-4.png"],
+]
+thinking_frame = SpriteFrame(images=thinking_list)
+
+
+class TranscriptFilter(AIService):
+    def __init__(self, bot_participant_id=None):
+        self.bot_participant_id = bot_participant_id
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, TranscriptionQueueFrame):
+            if frame.participantId != self.bot_participant_id:
+                yield frame
+
+
+class NameCheckFilter(AIService):
+    def __init__(self, names: list[str]):
+        self.names = names
+        self.sentence = ""
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        content: str = ""
+
+        # TODO: split up transcription by participant
+        if isinstance(frame, TextFrame):
+            content = frame.text
+
+        self.sentence += content
+        if self.sentence.endswith((".", "?", "!")):
+            if any(name in self.sentence for name in self.names):
+                out = self.sentence
+                self.sentence = ""
+                yield TextFrame(out)
+            else:
+                out = self.sentence
+                self.sentence = ""
+
+
+class ImageSyncAggregator(AIService):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        yield talking_frame
+        yield frame
+        yield quiet_frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Santa Cat",
+            duration_minutes=3,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=True,
+            camera_width=720,
+            camera_height=1280,
+        )
+        transport._mic_enabled = True
+        transport._mic_sample_rate = 16000
+        transport._camera_enabled = True
+        transport._camera_width = 720
+        transport._camera_height = 1280
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="jBpfuIE2acCO8z3wKNLl",
+        )
+        isa = ImageSyncAggregator()
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say(
+                "Hi! If you want to talk to me, just say 'hey Santa Cat'.",
+                transport.send_queue,
+            )
+
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while.  Your responses should only be a few sentences long.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            tf = TranscriptFilter(transport._my_participant_id)
+            ncf = NameCheckFilter(["Santa Cat", "Santa"])
+            await tts.run_to_queue(
+                transport.send_queue,
+                isa.run(
+                    tma_out.run(
+                        llm.run(
+                            tma_in.run(ncf.run(tf.run(transport.get_receive_frames())))
+                        )
+                    )
+                ),
+            )
+
+        async def starting_image():
+            await transport.send_queue.put(quiet_frame)
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions(), starting_image())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/11-sound-effects.py

@@ -0,0 +1,138 @@
+import aiohttp
+import asyncio
+import logging
+import os
+import wave
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.pipeline.aggregators import (
+    LLMContextAggregator,
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
+from dailyai.services.ai_services import AIService, FrameLogger
+from dailyai.pipeline.frames import (
+    Frame,
+    AudioFrame,
+    LLMResponseEndFrame,
+    LLMMessagesQueueFrame,
+)
+from typing import AsyncGenerator
+
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+sounds = {}
+sound_files = ["ding1.wav", "ding2.wav"]
+
+script_dir = os.path.dirname(__file__)
+
+for file in sound_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with wave.open(full_path) as audio_file:
+        sounds[file] = audio_file.readframes(-1)
+
+
+class OutboundSoundEffectWrapper(AIService):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMResponseEndFrame):
+            yield AudioFrame(sounds["ding1.wav"])
+            # In case anything else up the stack needs it
+            yield frame
+        else:
+            yield frame
+
+
+class InboundSoundEffectWrapper(AIService):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMMessagesQueueFrame):
+            yield AudioFrame(sounds["ding2.wav"])
+            # In case anything else up the stack needs it
+            yield frame
+        else:
+            yield frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Respond bot",
+            duration_minutes=5,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="ErXwobaYiN019PkySvjV",
+        )
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi, I'm listening!", transport.send_queue)
+            await transport.send_queue.put(AudioFrame(sounds["ding1.wav"]))
+
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            out_sound = OutboundSoundEffectWrapper()
+            in_sound = InboundSoundEffectWrapper()
+            fl = FrameLogger("LLM Out")
+            fl2 = FrameLogger("Transcription In")
+            await out_sound.run_to_queue(
+                transport.send_queue,
+                tts.run(
+                    fl.run(
+                        tma_out.run(
+                            llm.run(
+                                fl2.run(
+                                    in_sound.run(
+                                        tma_in.run(transport.get_receive_frames())
+                                    )
+                                )
+                            )
+                        )
+                    )
+                ),
+            )
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/13-whisper-transcription.py

@@ -0,0 +1,43 @@
+import asyncio
+import logging
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.whisper_ai_services import WhisperSTTService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str):
+    transport = DailyTransportService(
+        room_url,
+        None,
+        "Transcription bot",
+        start_transcription=True,
+        mic_enabled=False,
+        camera_enabled=False,
+        speaker_enabled=True,
+    )
+
+    stt = WhisperSTTService()
+    transcription_output_queue = asyncio.Queue()
+
+    async def handle_transcription():
+        print("`````````TRANSCRIPTION`````````")
+        while True:
+            item = await transcription_output_queue.get()
+            print(item.text)
+
+    async def handle_speaker():
+        await stt.run_to_queue(
+            transcription_output_queue, transport.get_receive_frames()
+        )
+
+    await asyncio.gather(transport.run(), handle_speaker(), handle_transcription())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/13a-whisper-local.py

@@ -0,0 +1,67 @@
+import argparse
+import asyncio
+import logging
+import wave
+from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
+
+from dailyai.services.local_transport_service import LocalTransportService
+from dailyai.services.whisper_ai_services import WhisperSTTService
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str):
+    global transport
+    global stt
+
+    meeting_duration_minutes = 1
+    transport = LocalTransportService(
+        mic_enabled=True,
+        camera_enabled=False,
+        speaker_enabled=True,
+        duration_minutes=meeting_duration_minutes,
+        start_transcription=True,
+    )
+    stt = WhisperSTTService()
+    transcription_output_queue = asyncio.Queue()
+    transport_done = asyncio.Event()
+
+    async def handle_transcription():
+        print("`````````TRANSCRIPTION`````````")
+        while not transport_done.is_set():
+            item = await transcription_output_queue.get()
+            print("got item from queue", item)
+            if isinstance(item, TranscriptionQueueFrame):
+                print(item.text)
+            elif isinstance(item, EndFrame):
+                break
+        print("handle_transcription done")
+
+    async def handle_speaker():
+        await stt.run_to_queue(
+            transcription_output_queue, transport.get_receive_frames()
+        )
+        await transcription_output_queue.put(EndFrame())
+        print("handle speaker done.")
+
+    async def run_until_done():
+        await transport.run()
+        transport_done.set()
+        print("run_until_done done")
+
+    await asyncio.gather(run_until_done(), handle_speaker(), handle_transcription())
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=True,
+        help="URL of the Daily room to join")
+
+    args, unknown = parser.parse_known_args()
+    asyncio.run(main(args.url))