added examples back

examples/storytelling-chatbot/src/bot.py

@@ -0,0 +1,172 @@
+import asyncio
+import aiohttp
+import logging
+import os
+import argparse
+
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.pipeline.frames import (
+    AudioFrame,
+    ImageFrame,
+    EndPipeFrame,
+    LLMMessagesFrame,
+    SendAppMessageFrame
+)
+from dailyai.pipeline.aggregators import (
+    LLMUserResponseAggregator,
+    LLMAssistantResponseAggregator,
+)
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.fal_ai_services import FalImageGenService
+
+from processors import StoryProcessor, StoryImageProcessor
+from prompts import LLM_BASE_PROMPT, LLM_INTRO_PROMPT, CUE_USER_TURN
+from utils.helpers import load_sounds, load_images
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logging.basicConfig(format=f"[STORYBOT] %(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.INFO)
+
+
+sounds = load_sounds(["listening.wav"])
+images = load_images(["book1.png", "book2.png"])
+
+
+async def main(room_url, token=None):
+    async with aiohttp.ClientSession() as session:
+
+        # -------------- Transport --------------- #
+
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Storytelling Bot",
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            vad_enabled=True,
+            camera_framerate=30,
+            camera_bitrate=680000,
+            camera_enabled=True,
+            camera_width=768,
+            camera_height=768,
+        )
+
+        logger.debug("Transport created for room:" + room_url)
+
+        # -------------- Services --------------- #
+
+        llm_service = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo"
+        )
+
+        tts_service = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        fal_service_params = FalImageGenService.InputParams(
+            image_size={
+                "width": 768,
+                "height": 768
+            }
+        )
+
+        fal_service = FalImageGenService(
+            aiohttp_session=session,
+            model="fal-ai/fast-lightning-sdxl",
+            params=fal_service_params,
+            key=os.getenv("FAL_KEY"),
+        )
+
+        # --------------- Setup ----------------- #
+
+        message_history = [LLM_BASE_PROMPT]
+        story_pages = []
+
+        # We need aggregators to keep track of user and LLM responses
+        llm_responses = LLMAssistantResponseAggregator(message_history)
+        user_responses = LLMUserResponseAggregator(message_history)
+
+        # -------------- Processors ------------- #
+
+        story_processor = StoryProcessor(message_history, story_pages)
+        image_processor = StoryImageProcessor(fal_service)
+
+        # -------------- Story Loop ------------- #
+
+        logger.debug("Waiting for participant...")
+
+        start_storytime_event = asyncio.Event()
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport, participant):
+            logger.debug("Participant joined, storytime commence!")
+            start_storytime_event.set()
+
+        # The storytime coroutine will wait for the start_storytime_event
+        # to be set before starting the storytime pipeline
+        async def storytime():
+            await start_storytime_event.wait()
+
+            # The intro pipeline is used to start
+            # the story (as per LLM_INTRO_PROMPT)
+            intro_pipeline = Pipeline(processors=[
+                llm_service,
+                tts_service,
+            ], sink=transport.send_queue)
+
+            await intro_pipeline.queue_frames(
+                [
+                    ImageFrame(images['book1'], (768, 768)),
+                    LLMMessagesFrame([LLM_INTRO_PROMPT]),
+                    SendAppMessageFrame(CUE_USER_TURN, None),
+                    AudioFrame(sounds["listening"]),
+                    ImageFrame(images['book2'], (768, 768)),
+                    EndPipeFrame(),
+                ]
+            )
+
+            # We start the pipeline as soon as the user joins
+            await intro_pipeline.run_pipeline()
+
+            # The main story pipeline is used to continue the
+            # story based on user input
+            pipeline = Pipeline(processors=[
+                user_responses,
+                llm_service,
+                story_processor,
+                image_processor,
+                tts_service,
+                llm_responses,
+            ])
+
+            await transport.run_pipeline(pipeline)
+
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+
+        try:
+            await asyncio.gather(transport.run(), storytime())
+        except (asyncio.CancelledError, KeyboardInterrupt):
+            transport.stop()
+
+        logger.debug("Pipeline finished. Exiting.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Daily Storyteller Bot")
+    parser.add_argument("-u", type=str, help="Room URL")
+    parser.add_argument("-t", type=str, help="Token")
+    config = parser.parse_args()
+
+    asyncio.run(main(config.u, config.t))

examples/storytelling-chatbot/src/processors.py

@@ -0,0 +1,148 @@
+from typing import AsyncGenerator
+import re
+
+from dailyai.pipeline.frames import TextFrame, Frame, AudioFrame
+from dailyai.pipeline.frame_processor import FrameProcessor
+from dailyai.pipeline.frames import (
+    Frame,
+    TextFrame,
+    SendAppMessageFrame,
+    LLMResponseEndFrame,
+    UserStoppedSpeakingFrame,
+)
+
+from utils.helpers import load_sounds
+from prompts import IMAGE_GEN_PROMPT, CUE_USER_TURN, CUE_ASSISTANT_TURN
+import asyncio
+
+sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"])
+
+# -------------- Frame Types ------------- #
+
+
+class StoryPageFrame(TextFrame):
+    # Frame for each sentence in the story before a [break]
+    pass
+
+
+class StoryImageFrame(TextFrame):
+    # Frame for trigger image generation
+    pass
+
+
+class StoryPromptFrame(TextFrame):
+    # Frame for prompting the user for input
+    pass
+
+
+# ------------ Frame Processors ----------- #
+
+class StoryImageProcessor(FrameProcessor):
+    """
+    Processor for image prompt frames that will be sent to the FAL service.
+
+    This processor is responsible for consuming frames of type `StoryImageFrame`.
+    It processes the by passing it to the FAL service
+    The processed frames are then yielded back.
+
+    Attributes:
+        _fal_service (FALService): The FAL service, generates the images (fast fast!).
+    """
+
+    def __init__(self, fal_service):
+        self._fal_service = fal_service
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, StoryImageFrame):
+            try:
+                async with asyncio.timeout(7):
+                    async for i in self._fal_service.process_frame(TextFrame(IMAGE_GEN_PROMPT % frame.text)):
+                        yield i
+            except TimeoutError:
+                pass
+            pass
+        else:
+            yield frame
+
+
+class StoryProcessor(FrameProcessor):
+    """
+    Primary frame processor. It takes the frames generated by the LLM
+    and processes them into image prompts and story pages (sentences.)
+    For a clearer picture of how this works, reference prompts.py
+
+    Attributes:
+        _messages (list): A list of llm messages.
+        _text (str): A buffer to store the text from text frames.
+        _story (list): A list to store the story sentences, or 'pages'.
+
+    Methods:
+        process_frame: Processes a frame and removes any [break] or [image] tokens.
+    """
+
+    def __init__(self, messages, story):
+        self._messages = messages
+        self._text = ""
+        self._story = story
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, UserStoppedSpeakingFrame):
+            # Send an app message to the UI
+            yield SendAppMessageFrame(CUE_ASSISTANT_TURN, None)
+            yield AudioFrame(sounds["talking"])
+
+        elif isinstance(frame, TextFrame):
+            # We want to look for sentence breaks in the text
+            # but since TextFrames are streamed from the LLM
+            # we need to keep a buffer of the text we've seen so far
+            self._text += frame.text
+
+            # IMAGE PROMPT
+            # Looking for: < [image prompt] > in the LLM response
+            # We prompted our LLM to add an image prompt in the response
+            # so we use regex matching to find it and yield a StoryImageFrame
+            if re.search(r"<.*?>", self._text):
+                if not re.search(r"<.*?>.*?>", self._text):
+                    # Pass any frames until we have a closing bracket
+                    # otherwise the image prompt will be passed to TTS
+                    pass
+                # Extract the image prompt from the text using regex
+                image_prompt = re.search(r"<(.*?)>", self._text).group(1)
+                # Remove the image prompt from the text
+                self._text = re.sub(r"<.*?>", '', self._text, count=1)
+                # Process the image prompt frame
+                yield StoryImageFrame(image_prompt)
+
+            # STORY PAGE
+            # Looking for: [break] in the LLM response
+            # We prompted our LLM to add a [break] after each sentence
+            # so we use regex matching to find it in the LLM response
+            if re.search(r".*\[[bB]reak\].*", self._text):
+                # Remove the [break] token from the text
+                # so it isn't spoken out loud by the TTS
+                self._text = re.sub(r'\[[bB]reak\]', '',
+                                    self._text, flags=re.IGNORECASE)
+                self._text = self._text.replace("\n", " ")
+                if len(self._text) > 2:
+                    # Append the sentence to the story
+                    self._story.append(self._text)
+                    yield StoryPageFrame(self._text)
+                    # Assert that it's the LLMs turn, until we're finished
+                    yield SendAppMessageFrame(CUE_ASSISTANT_TURN, None)
+                # Clear the buffer
+                self._text = ""
+
+        # End of LLM response
+        # Driven by the prompt, the LLM should have asked the user for input
+        elif isinstance(frame, LLMResponseEndFrame):
+            # We use a different frame type, as to avoid image generation ingest
+            yield StoryPromptFrame(self._text)
+            self._text = ""
+            yield frame
+            # Send an app message to the UI
+            yield SendAppMessageFrame(CUE_USER_TURN, None)
+            yield AudioFrame(sounds["listening"])
+
+        # Anything that is not a TextFrame pass through
+        else:
+            yield frame

examples/storytelling-chatbot/src/prompts.py

@@ -0,0 +1,35 @@
+LLM_INTRO_PROMPT = {
+    "role": "system",
+    "content": "You are a creative storyteller who loves to tell whimsical, fantastical stories. \
+        Your goal is to craft an engaging and fun story. \
+        Start by asking the user what kind of story they'd like to hear. Don't provide any examples. \
+        Keep your reponse to only a few sentences."
+}
+
+
+LLM_BASE_PROMPT = {
+    "role": "system",
+    "content": "You are a creative storyteller who loves tell whimsical, fantastical stories. \
+        Your goal is to craft an engaging and fun story. \
+        Keep all responses short and no more than a few sentences. Include [break] after each sentence of the story. \
+        \
+        Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. \
+        Image prompts should always be wrapped in triangle braces, like this: <image prompt goes here>. \
+        You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. \
+        For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>. \
+        Please do not include any character names in the image prompts, just their descriptions. \
+        Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>. \
+        Please use the following structure for your image prompts: characters, setting, action, and mood. \
+        Image prompts should be less than 150-200 characters and start in lowercase. \
+        \
+        Responses should use the format: <...> story sentence [break] <...> story sentence [break] ... \
+        After each response, ask me how I'd like the story to continue and wait for my input. \
+        Please ensure your responses are less than 3-4 sentences long. \
+        Please refrain from using any explicit language or content. Do not tell scary stories."
+}
+
+
+IMAGE_GEN_PROMPT = "illustrative art of %s. In the style of Studio Ghibli. colorful, whimsical, painterly, concept art."
+
+CUE_USER_TURN = {"cue": "user_turn"}
+CUE_ASSISTANT_TURN = {"cue": "assistant_turn"}

examples/storytelling-chatbot/src/server.py

@@ -0,0 +1,175 @@
+import os
+import argparse
+import subprocess
+import atexit
+from pathlib import Path
+from typing import Optional
+
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
+
+from utils.daily_helpers import create_room as _create_room, get_token, get_name_from_url
+
+MAX_BOTS_PER_ROOM = 1
+
+# Bot sub-process dict for status reporting and concurrency control
+bot_procs = {}
+
+
+def cleanup():
+    # Clean up function, just to be extra safe
+    for proc in bot_procs.values():
+        proc.terminate()
+        proc.wait()
+
+
+atexit.register(cleanup)
+
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Mount the static directory
+STATIC_DIR = "frontend/out"
+
+app.mount("/static", StaticFiles(directory=STATIC_DIR, html=True), name="static")
+
+
+@app.post("/create")
+async def create_room(request: Request) -> JSONResponse:
+    data = await request.json()
+
+    if data.get('room_url') is not None:
+        room_url = data.get('room_url')
+        room_name = get_name_from_url(room_url)
+    else:
+        room_url, room_name = _create_room()
+
+    token = get_token(room_url)
+
+    return JSONResponse({"room_url": room_url, "room_name": room_name, "token": token})
+
+
+@app.post("/start")
+async def start_agent(request: Request) -> JSONResponse:
+    data = await request.json()
+
+    # Is this a webhook creation request?
+    if "test" in data:
+        return JSONResponse({"test": True})
+
+    # Ensure the room property is present
+    room_url = data.get('room_url')
+    if not room_url:
+        raise HTTPException(
+            status_code=500,
+            detail="Missing 'room' property in request data. Cannot start agent without a target room!")
+
+    # Check if there is already an existing process running in this room
+    num_bots_in_room = sum(
+        1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None)
+    if num_bots_in_room >= MAX_BOTS_PER_ROOM:
+        raise HTTPException(
+            status_code=500, detail=f"Max bot limited reach for room: {room_url}")
+
+    # Get the token for the room
+    token = get_token(room_url)
+
+    if not token:
+        raise HTTPException(
+            status_code=500, detail=f"Failed to get token for room: {room_url}")
+
+    # Spawn a new agent, and join the user session
+    # Note: this is mostly for demonstration purposes (refer to 'deployment' in README)
+    try:
+        proc = subprocess.Popen(
+            [
+                f"python3 -m bot -u {room_url} -t {token}"
+            ],
+            shell=True,
+            bufsize=1,
+            cwd=os.path.dirname(os.path.abspath(__file__))
+        )
+        bot_procs[proc.pid] = (proc, room_url)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Failed to start subprocess: {e}")
+
+    return JSONResponse({"bot_id": proc.pid, "room_url": room_url})
+
+
+@app.get("/status/{pid}")
+def get_status(pid: int):
+    # Look up the subprocess
+    proc = bot_procs.get(pid)
+
+    # If the subprocess doesn't exist, return an error
+    if not proc:
+        raise HTTPException(
+            status_code=404, detail=f"Bot with process id: {pid} not found")
+
+    # Check the status of the subprocess
+    if proc[0].poll() is None:
+        status = "running"
+    else:
+        status = "finished"
+
+    return JSONResponse({"bot_id": pid, "status": status})
+
+
+@app.get("/{path_name:path}", response_class=FileResponse)
+async def catch_all(path_name: Optional[str] = ""):
+    if path_name == "":
+        return FileResponse(f"{STATIC_DIR}/index.html")
+
+    file_path = Path(STATIC_DIR) / (path_name or "")
+
+    if file_path.is_file():
+        return file_path
+
+    html_file_path = file_path.with_suffix(".html")
+    if html_file_path.is_file():
+        return FileResponse(html_file_path)
+
+    raise HTTPException(status_code=450, detail="Incorrect API call")
+
+
+if __name__ == "__main__":
+    # Check environment variables
+    required_env_vars = ['OPENAI_API_KEY', 'DAILY_API_KEY',
+                         'FAL_KEY', 'ELEVENLABS_VOICE_ID', 'ELEVENLABS_API_KEY']
+    for env_var in required_env_vars:
+        if env_var not in os.environ:
+            raise Exception(f"Missing environment variable: {env_var}.")
+
+    import uvicorn
+
+    default_host = os.getenv("HOST", "0.0.0.0")
+    default_port = int(os.getenv("FAST_API_PORT", "7860"))
+
+    parser = argparse.ArgumentParser(
+        description="Daily Storyteller FastAPI server")
+    parser.add_argument("--host", type=str,
+                        default=default_host, help="Host address")
+    parser.add_argument("--port", type=int,
+                        default=default_port, help="Port number")
+    parser.add_argument("--reload", action="store_true",
+                        help="Reload code on change")
+
+    config = parser.parse_args()
+
+    uvicorn.run(
+        "server:app",
+        host=config.host,
+        port=config.port,
+        reload=config.reload
+    )

examples/storytelling-chatbot/src/utils/daily_helpers.py

@@ -0,0 +1,109 @@
+
+import urllib.parse
+import os
+import time
+import urllib
+import requests
+
+from dotenv import load_dotenv
+load_dotenv()
+
+
+daily_api_path = os.getenv("DAILY_API_URL")
+daily_api_key = os.getenv("DAILY_API_KEY")
+
+
+def create_room() -> tuple[str, str]:
+    """
+    Helper function to create a Daily room.
+    # See: https://docs.daily.co/reference/rest-api/rooms
+
+    Returns:
+        tuple: A tuple containing the room URL and room name.
+
+    Raises:
+        Exception: If the request to create the room fails or if the response does not contain the room URL or room name.
+    """
+    room_props = {
+        "exp": time.time() + 60 * 60,  # 1 hour
+        "enable_chat": True,
+        "enable_emoji_reactions": True,
+        "eject_at_room_exp": True,
+        "enable_prejoin_ui": False,  # Important for the bot to be able to join headlessly
+    }
+    res = requests.post(
+        f"https://{daily_api_path}/rooms",
+        headers={"Authorization": f"Bearer {daily_api_key}"},
+        json={
+            "properties": room_props
+        },
+    )
+    if res.status_code != 200:
+        raise Exception(f"Unable to create room: {res.text}")
+
+    data = res.json()
+    room_url: str = data.get("url")
+    room_name: str = data.get("name")
+    if room_url is None or room_name is None:
+        raise Exception("Missing room URL or room name in response")
+
+    return room_url, room_name
+
+
+def get_name_from_url(room_url: str) -> str:
+    """
+    Extracts the name from a given room URL.
+
+    Args:
+        room_url (str): The URL of the room.
+
+    Returns:
+        str: The extracted name from the room URL.
+    """
+    return urllib.parse.urlparse(room_url).path[1:]
+
+
+def get_token(room_url: str) -> str:
+    """
+    Retrieves a meeting token for the specified Daily room URL.
+    # See: https://docs.daily.co/reference/rest-api/meeting-tokens
+
+    Args:
+        room_url (str): The URL of the Daily room.
+
+    Returns:
+        str: The meeting token.
+
+    Raises:
+        Exception: If no room URL is specified or if no Daily API key is specified.
+        Exception: If there is an error creating the meeting token.
+    """
+    if not room_url:
+        raise Exception(
+            "No Daily room specified. You must specify a Daily room in order a token to be generated.")
+
+    if not daily_api_key:
+        raise Exception(
+            "No Daily API key specified. set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
+
+    expiration: float = time.time() + 60 * 60
+    room_name = get_name_from_url(room_url)
+
+    res: requests.Response = requests.post(
+        f"https://{daily_api_path}/meeting-tokens",
+        headers={
+            "Authorization": f"Bearer {daily_api_key}"},
+        json={
+            "properties": {
+                "room_name": room_name,
+                "is_owner": True,  # Owner tokens required for transcription
+                "exp": expiration}},
+    )
+
+    if res.status_code != 200:
+        raise Exception(
+            f"Failed to create meeting token: {res.status_code} {res.text}")
+
+    token: str = res.json()["token"]
+
+    return token

examples/storytelling-chatbot/src/utils/helpers.py

@@ -0,0 +1,33 @@
+import os
+import wave
+from PIL import Image
+
+script_dir = os.path.dirname(__file__)
+
+
+def load_images(image_files):
+    images = {}
+    for file in image_files:
+        # Build the full path to the image file
+        full_path = os.path.join(script_dir, "../assets", file)
+        # Get the filename without the extension to use as the dictionary key
+        filename = os.path.splitext(os.path.basename(full_path))[0]
+        # Open the image and convert it to bytes
+        with Image.open(full_path) as img:
+            images[filename] = img.tobytes()
+    return images
+
+
+def load_sounds(sound_files):
+    sounds = {}
+
+    for file in sound_files:
+        # Build the full path to the sound file
+        full_path = os.path.join(script_dir, "../assets", file)
+        # Get the filename without the extension to use as the dictionary key
+        filename = os.path.splitext(os.path.basename(full_path))[0]
+        # Open the sound and convert it to bytes
+        with wave.open(full_path) as audio_file:
+            sounds[filename] = audio_file.readframes(-1)
+
+    return sounds