demo: Restructure storytelling-chatbot directory, update README steps, link to vercel demo

examples/storytelling-chatbot/server/.dockerignore

@@ -0,0 +1,2 @@
+client/node_modules
+client/out

examples/storytelling-chatbot/server/Dockerfile

@@ -0,0 +1,54 @@
+FROM python:3.11-slim-bookworm
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG USE_PERSISTENT_DATA
+ENV PYTHONUNBUFFERED=1
+ENV NODE_MAJOR=20
+
+# Expose FastAPI port
+ENV FAST_API_PORT=7860
+EXPOSE 7860
+
+# Install system dependencies
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    build-essential \
+    git \
+    ffmpeg \
+    google-perftools \
+    ca-certificates curl gnupg \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install Node.js
+RUN mkdir -p /etc/apt/keyrings 
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list > /dev/null
+RUN apt-get update && apt-get install nodejs -y
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+    PYTHONUNBUFFERED=1
+
+# Switch to the "user" user
+USER user
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Install Python dependencies
+COPY ./requirements.txt requirements.txt
+RUN pip3 install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy everything else
+COPY --chown=user ./server/ server/
+
+# Copy client app and build
+COPY --chown=user ./client/ client/
+RUN cd client && npm install && npm run build
+
+# Start the FastAPI server
+CMD python3 server/bot_runner.py --port ${FAST_API_PORT}

examples/storytelling-chatbot/server/bot.py

@@ -0,0 +1,149 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import asyncio
+import os
+import sys
+
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+from processors import StoryImageProcessor, StoryProcessor
+from prompts import CUE_USER_TURN, LLM_BASE_PROMPT
+from utils.helpers import load_images, load_sounds
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.frames.frames import EndFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
+from pipecat.services.google.image import GoogleImageGenService
+from pipecat.services.google.llm import GoogleLLMService
+from pipecat.transports.services.daily import (
+    DailyParams,
+    DailyTransport,
+    DailyTransportMessageFrame,
+)
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+sounds = load_sounds(["listening.wav"])
+images = load_images(["book1.png", "book2.png"])
+
+
+async def main(room_url, token=None):
+    async with aiohttp.ClientSession() as session:
+        # -------------- Transport --------------- #
+
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Storytelling Bot",
+            DailyParams(
+                audio_in_enabled=True,
+                audio_out_enabled=True,
+                video_out_enabled=True,
+                video_out_width=1024,
+                video_out_height=1024,
+                transcription_enabled=True,
+                vad_analyzer=SileroVADAnalyzer(),
+            ),
+        )
+
+        logger.debug("Transport created for room:" + room_url)
+
+        # -------------- Services --------------- #
+
+        llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
+
+        tts_service = ElevenLabsTTSService(
+            api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")
+        )
+
+        image_gen = GoogleImageGenService(api_key=os.getenv("GOOGLE_API_KEY"))
+
+        # --------------- Setup ----------------- #
+
+        message_history = [LLM_BASE_PROMPT]
+        story_pages = []
+
+        # We need aggregators to keep track of user and LLM responses
+        context = OpenAILLMContext(message_history)
+        context_aggregator = llm_service.create_context_aggregator(context)
+
+        # -------------- Processors ------------- #
+
+        story_processor = StoryProcessor(message_history, story_pages)
+        image_processor = StoryImageProcessor(image_gen)
+
+        # -------------- Story Loop ------------- #
+
+        runner = PipelineRunner()
+
+        logger.debug("Waiting for participant...")
+        main_pipeline = Pipeline(
+            [
+                transport.input(),
+                context_aggregator.user(),
+                llm_service,
+                story_processor,
+                image_processor,
+                tts_service,
+                transport.output(),
+                context_aggregator.assistant(),
+            ]
+        )
+
+        main_task = PipelineTask(
+            main_pipeline,
+            params=PipelineParams(
+                allow_interruptions=True,
+                enable_metrics=True,
+                enable_usage_metrics=True,
+            ),
+        )
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            logger.debug("Participant joined, storytime commence!")
+            await transport.capture_participant_transcription(participant["id"])
+            await main_task.queue_frames(
+                [
+                    images["book1"],
+                    context_aggregator.user().get_context_frame(),
+                    DailyTransportMessageFrame(CUE_USER_TURN),
+                    # sounds["listening"],
+                    images["book2"],
+                ]
+            )
+
+        @transport.event_handler("on_participant_left")
+        async def on_participant_left(transport, participant, reason):
+            await main_task.cancel()
+
+        @transport.event_handler("on_call_state_updated")
+        async def on_call_state_updated(transport, state):
+            if state == "left":
+                # Here we don't want to cancel, we just want to finish sending
+                # whatever is queued, so we use an EndFrame().
+                await main_task.queue_frame(EndFrame())
+
+        await runner.run(main_task)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Daily Storyteller Bot")
+    parser.add_argument("-u", type=str, help="Room URL")
+    parser.add_argument("-t", type=str, help="Token")
+    config = parser.parse_args()
+
+    asyncio.run(main(config.u, config.t))

examples/storytelling-chatbot/server/bot_runner.py

@@ -0,0 +1,239 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import os
+import subprocess
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Optional
+
+import aiohttp
+from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+from pipecat.transports.services.helpers.daily_rest import (
+    DailyRESTHelper,
+    DailyRoomObject,
+    DailyRoomParams,
+    DailyRoomProperties,
+)
+
+load_dotenv(override=True)
+
+# ------------ Fast API Config ------------ #
+
+MAX_SESSION_TIME = 5 * 60  # 5 minutes
+
+daily_helpers = {}
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    aiohttp_session = aiohttp.ClientSession()
+    daily_helpers["rest"] = DailyRESTHelper(
+        daily_api_key=os.getenv("DAILY_API_KEY", ""),
+        daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
+        aiohttp_session=aiohttp_session,
+    )
+    yield
+    await aiohttp_session.close()
+
+
+app = FastAPI(lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Mount the static directory
+STATIC_DIR = "client/out"
+
+
+# ------------ Fast API Routes ------------ #
+
+app.mount("/static", StaticFiles(directory=STATIC_DIR, html=True), name="static")
+
+
+@app.post("/")
+async def start_bot(request: Request) -> JSONResponse:
+    if os.getenv("ENV", "dev") == "production":
+        # Only allow requests from the specified domain
+        host_header = request.headers.get("host")
+        allowed_domains = ["storytelling-chatbot.fly.dev", "www.storytelling-chatbot.fly.dev"]
+        # Check if the Host header matches the allowed domain
+        if host_header not in allowed_domains:
+            raise HTTPException(status_code=403, detail="Access denied")
+
+    try:
+        data = await request.json()
+        # Is this a webhook creation request?
+        if "test" in data:
+            return JSONResponse({"test": True})
+    except Exception as e:
+        pass
+
+    # Use specified room URL, or create a new one if not specified
+    room_url = os.getenv("DAILY_SAMPLE_ROOM_URL", "")
+
+    if not room_url:
+        params = DailyRoomParams(properties=DailyRoomProperties())
+        try:
+            room: DailyRoomObject = await daily_helpers["rest"].create_room(params=params)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Unable to provision room {e}")
+    else:
+        # Check passed room URL exists, we should assume that it already has a sip set up
+        try:
+            room: DailyRoomObject = await daily_helpers["rest"].get_room_from_url(room_url)
+        except Exception:
+            raise HTTPException(status_code=500, detail=f"Room not found: {room_url}")
+
+    # Give the agent a token to join the session
+    token = await daily_helpers["rest"].get_token(room.url, MAX_SESSION_TIME)
+
+    if not room or not token:
+        raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room_url}")
+
+    # Launch a new VM, or run as a shell process (not recommended)
+    if os.getenv("RUN_AS_VM", False):
+        try:
+            await virtualize_bot(room.url, token)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to spawn VM: {e}")
+    else:
+        try:
+            subprocess.Popen(
+                [f"python -m bot -u {room.url} -t {token}"],
+                shell=True,
+                bufsize=1,
+                cwd=os.path.dirname(os.path.abspath(__file__)),
+            )
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
+
+    # Grab a token for the user to join with
+    user_token = await daily_helpers["rest"].get_token(room.url, MAX_SESSION_TIME)
+
+    return JSONResponse(
+        {
+            "room_url": room.url,
+            "token": user_token,
+        }
+    )
+
+
+@app.get("/{path_name:path}", response_class=FileResponse)
+async def catch_all(path_name: Optional[str] = ""):
+    if path_name == "":
+        return FileResponse(f"{STATIC_DIR}/index.html")
+
+    file_path = Path(STATIC_DIR) / (path_name or "")
+
+    if file_path.is_file():
+        return file_path
+
+    html_file_path = file_path.with_suffix(".html")
+    if html_file_path.is_file():
+        return FileResponse(html_file_path)
+
+    raise HTTPException(status_code=450, detail="Incorrect API call")
+
+
+# ------------ Virtualization ------------ #
+
+
+async def virtualize_bot(room_url: str, token: str):
+    """This is an example of how to virtualize the bot using Fly.io
+    You can adapt this method to use whichever cloud provider you prefer.
+    """
+    FLY_API_HOST = os.getenv("FLY_API_HOST", "https://api.machines.dev/v1")
+    FLY_APP_NAME = os.getenv("FLY_APP_NAME", "storytelling-chatbot")
+    FLY_API_KEY = os.getenv("FLY_API_KEY", "")
+    FLY_HEADERS = {"Authorization": f"Bearer {FLY_API_KEY}", "Content-Type": "application/json"}
+
+    async with aiohttp.ClientSession() as session:
+        # Use the same image as the bot runner
+        async with session.get(
+            f"{FLY_API_HOST}/apps/{FLY_APP_NAME}/machines", headers=FLY_HEADERS
+        ) as r:
+            if r.status != 200:
+                text = await r.text()
+                raise Exception(f"Unable to get machine info from Fly: {text}")
+
+            data = await r.json()
+            image = data[0]["config"]["image"]
+
+        # Machine configuration
+        cmd = f"python server/bot.py -u {room_url} -t {token}"
+        cmd = cmd.split()
+        worker_props = {
+            "config": {
+                "image": image,
+                "auto_destroy": True,
+                "init": {"cmd": cmd},
+                "restart": {"policy": "no"},
+                "guest": {"cpu_kind": "shared", "cpus": 1, "memory_mb": 512},
+            },
+        }
+
+        # Spawn a new machine instance
+        async with session.post(
+            f"{FLY_API_HOST}/apps/{FLY_APP_NAME}/machines", headers=FLY_HEADERS, json=worker_props
+        ) as r:
+            if r.status != 200:
+                text = await r.text()
+                raise Exception(f"Problem starting a bot worker: {text}")
+
+            data = await r.json()
+            # Wait for the machine to enter the started state
+            vm_id = data["id"]
+
+        async with session.get(
+            f"{FLY_API_HOST}/apps/{FLY_APP_NAME}/machines/{vm_id}/wait?state=started",
+            headers=FLY_HEADERS,
+        ) as r:
+            if r.status != 200:
+                text = await r.text()
+                raise Exception(f"Bot was unable to enter started state: {text}")
+
+        print(f"Machine joined room: {room_url}")
+
+
+# ------------ Main ------------ #
+
+if __name__ == "__main__":
+    # Check environment variables
+    required_env_vars = [
+        "GOOGLE_API_KEY",
+        "DAILY_API_KEY",
+        "ELEVENLABS_VOICE_ID",
+        "ELEVENLABS_API_KEY",
+    ]
+    for env_var in required_env_vars:
+        if env_var not in os.environ:
+            raise Exception(f"Missing environment variable: {env_var}.")
+
+    import uvicorn
+
+    default_host = os.getenv("HOST", "0.0.0.0")
+    default_port = int(os.getenv("FAST_API_PORT", "7860"))
+
+    parser = argparse.ArgumentParser(description="Daily Storyteller FastAPI server")
+    parser.add_argument("--host", type=str, default=default_host, help="Host address")
+    parser.add_argument("--port", type=int, default=default_port, help="Port number")
+    parser.add_argument("--reload", action="store_true", help="Reload code on change")
+
+    config = parser.parse_args()
+
+    uvicorn.run("bot_runner:app", host=config.host, port=config.port, reload=config.reload)

examples/storytelling-chatbot/server/env.example

@@ -0,0 +1,6 @@
+DAILY_API_KEY=
+DAILY_SAMPLE_ROOM_URL=
+ELEVENLABS_API_KEY=
+ELEVENLABS_VOICE_ID=
+GOOGLE_API_KEY=
+ENV=dev

examples/storytelling-chatbot/server/processors.py

@@ -0,0 +1,200 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+import re
+
+import google.ai.generativelanguage as glm
+from async_timeout import timeout
+from loguru import logger
+from prompts import (
+    CUE_ASSISTANT_TURN,
+    CUE_USER_TURN,
+    FIRST_IMAGE_PROMPT,
+    IMAGE_GEN_PROMPT,
+    NEXT_IMAGE_PROMPT,
+)
+from utils.helpers import load_sounds
+
+from pipecat.frames.frames import (
+    Frame,
+    LLMFullResponseEndFrame,
+    TextFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.google.llm import GoogleLLMService
+from pipecat.transports.services.daily import DailyTransportMessageFrame
+
+sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"])
+
+# -------------- Frame Types ------------- #
+
+
+class StoryPageFrame(TextFrame):
+    # Frame for each sentence in the story before a [break]
+    pass
+
+
+class StoryImageFrame(TextFrame):
+    # Frame for trigger image generation
+    pass
+
+
+class StoryPromptFrame(TextFrame):
+    # Frame for prompting the user for input
+    pass
+
+
+# ------------ Frame Processors ----------- #
+
+
+class StoryImageProcessor(FrameProcessor):
+    """Processor for image prompt frames that will be sent to the FAL service.
+
+    This processor is responsible for consuming frames of type `StoryImageFrame`.
+    It processes them by passing it to the FAL service.
+    The processed frames are then yielded back.
+
+    Attributes:
+        _image_gen_service: The FAL service, generates the images (fast fast!).
+    """
+
+    def __init__(self, image_gen_service):
+        super().__init__()
+        self._image_gen_service = image_gen_service
+        # Create a new LLM service to use a different system prompt, etc
+        self._llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
+
+        self.pages = []
+        self.image_descriptions = []
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, StoryPageFrame):
+            # Special syntax for the first page
+            if self.pages == []:
+                prompt = FIRST_IMAGE_PROMPT % frame.text
+            else:
+                prompt = NEXT_IMAGE_PROMPT % (
+                    " ".join(self.pages),
+                    "; ".join(self.image_descriptions),
+                    frame.text,
+                )
+
+            await self.start_ttfb_metrics()
+            # TODO: This is coupled to google implementation now
+            txt = glm.Content(role="user", parts=[glm.Part(text=prompt)])
+            llm_response = await self._llm_service._client.generate_content_async(
+                contents=[txt], stream=False
+            )
+            image_description = llm_response.text
+            self.pages.append(frame.text)
+            self.image_descriptions.append(image_description)
+            try:
+                async with timeout(15):
+                    async for i in self._image_gen_service.run_image_gen(
+                        IMAGE_GEN_PROMPT % image_description
+                    ):
+                        await self.push_frame(i)
+            except TimeoutError:
+                logger.debug("Image gen timeout")
+                pass
+            await self.stop_ttfb_metrics()
+            # Push the StoryPageFrame so it gets TTS
+            await self.push_frame(frame)
+        else:
+            await self.push_frame(frame)
+
+
+class StoryProcessor(FrameProcessor):
+    """Primary frame processor. It takes the frames generated by the LLM
+    and processes them into image prompts and story pages (sentences).
+    For a clearer picture of how this works, reference prompts.py
+
+    Attributes:
+        _messages (list): A list of llm messages.
+        _text (str): A buffer to store the text from text frames.
+        _story (list): A list to store the story sentences, or 'pages'.
+
+    Methods:
+        process_frame: Processes a frame and removes any [break] or [image] tokens.
+    """
+
+    def __init__(self, messages, story):
+        super().__init__()
+        self._messages = messages
+        self._text = ""
+        self._story = story
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, UserStoppedSpeakingFrame):
+            # Send an app message to the UI
+            await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN))
+            await self.push_frame(sounds["talking"])
+
+        elif isinstance(frame, TextFrame):
+            # Add new text to the buffer
+            # (character replace hack to fix TTS sequencing)
+            self._text += frame.text.replace(";", "—")
+            # Process any complete patterns in the order they appear
+            await self.process_text_content()
+
+        # End of a full LLM response
+        # Driven by the prompt, the LLM should have asked the user for input
+        elif isinstance(frame, LLMFullResponseEndFrame):
+            # We use a different frame type, as to avoid image generation ingest
+            await self.push_frame(StoryPromptFrame(self._text))
+            self._text = ""
+            await self.push_frame(frame)
+            # Send an app message to the UI
+            await self.push_frame(DailyTransportMessageFrame(CUE_USER_TURN))
+            await self.push_frame(sounds["listening"])
+
+        # Anything that is not a TextFrame pass through
+        else:
+            await self.push_frame(frame)
+
+    async def process_text_content(self):
+        """Process text content in order of appearance, handling both image prompts and story breaks."""
+        while True:
+            # Find the first occurrence of each pattern
+            image_match = re.search(r"<(.*?)>", self._text)
+            break_match = re.search(r"\[[bB]reak\]", self._text)
+
+            # If neither pattern is found, we're done processing
+            if not image_match and not break_match:
+                break
+
+            # Find which pattern comes first in the text
+            image_pos = image_match.start() if image_match else float("inf")
+            break_pos = break_match.start() if break_match else float("inf")
+
+            if image_pos < break_pos:
+                # Process image prompt first
+                image_prompt = image_match.group(1)
+                # Remove the image prompt from the text
+                self._text = self._text[: image_match.start()] + self._text[image_match.end() :]
+                await self.push_frame(StoryImageFrame(image_prompt))
+            else:
+                # Process story break first
+                parts = re.split(r"\[[bB]reak\]", self._text, flags=re.IGNORECASE, maxsplit=1)
+                before_break = parts[0].replace("\n", " ").strip()
+
+                if len(before_break) > 2:
+                    self._story.append(before_break)
+                    await self.push_frame(StoryPageFrame(before_break))
+                    # await self.push_frame(sounds["ding"])
+                    await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN))
+
+                # Keep the remainder (if any) in the buffer
+                self._text = parts[1].strip() if len(parts) > 1 else ""

examples/storytelling-chatbot/server/prompts.py

@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+LLM_BASE_PROMPT = {
+    "role": "system",
+    "content": """You are a creative storyteller who loves tell whimsical, fantastical stories.
+        Your goal is to craft an engaging and fun story.
+        Keep all responses short and no more than a few sentences.
+        Start by asking the user what kind of story they'd like to hear. Don't provide any examples.
+        After they've answered the question, start telling the story. Include three story sentences in your response. Add [break] after each sentence of the story.
+
+        EXAMPLE OUTPUT FORMAT:
+        story sentence 1 [break]
+        story sentence 2 [break]
+        story sentence 3 [break]
+        How would you like the story to continue?
+        END OF EXAMPLE OUTPUT
+
+        Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want.
+        Please ensure your responses are less than 5 sentences long.
+        Please refrain from using any explicit language or content. Do not tell scary stories.
+        Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include [break] between story sentences. DO NOT RESPOND without story sentences and break tags.""",
+}
+
+
+IMAGE_GEN_PROMPT = "an illustration of %s. colorful, whimsical, painterly, concept art."
+
+CUE_USER_TURN = {"cue": "user_turn"}
+CUE_ASSISTANT_TURN = {"cue": "assistant_turn"}
+
+
+"""        Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene.
+        Image prompts should always be wrapped in triangle braces, like this: <image prompt goes here>.
+        You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+        For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>.
+        Please do not include any character names in the image prompts, just their descriptions.
+        Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>.
+        Please use the following structure for your image prompts: characters, setting, action, and mood.
+        Image prompts should be less than 150-200 characters and start in lowercase."""
+
+FIRST_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book.
+
+You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>.
+Please do not include any character names in the image prompts, just their descriptions.
+Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>.
+Please use the following structure for your image prompts: characters, setting, action, and mood.
+Image prompts should be less than 150-200 characters and start in lowercase.
+
+
+Here's the first page of the story:
+%s
+"""
+
+NEXT_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book.
+
+Here is the text of the story so far:
+%s
+
+Here are the previous image prompts:
+%s
+
+You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+For any recurring characters, you should try to use the same description of them in the image prompt each time.
+Please do not include any character names in the image prompts, just their descriptions.
+Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>.
+Please use the following structure for your image prompts: characters, setting, action, and mood.
+Image prompts should be less than 150-200 characters and start in lowercase.
+Here's the next page of the story:
+%s
+"""

examples/storytelling-chatbot/server/requirements.txt

@@ -0,0 +1,5 @@
+async_timeout
+fastapi
+uvicorn
+python-dotenv
+pipecat-ai[daily,silero,openai,cartesia,google]

examples/storytelling-chatbot/server/utils/helpers.py

@@ -0,0 +1,42 @@
+import os
+import wave
+
+from PIL import Image
+
+from pipecat.frames.frames import OutputAudioRawFrame, OutputImageRawFrame
+
+script_dir = os.path.dirname(__file__)
+
+
+def load_images(image_files):
+    images = {}
+    for file in image_files:
+        # Build the full path to the image file
+        full_path = os.path.join(script_dir, "../assets", file)
+        # Get the filename without the extension to use as the dictionary key
+        filename = os.path.splitext(os.path.basename(full_path))[0]
+        # Open the image and convert it to bytes
+        with Image.open(full_path) as img:
+            images[filename] = OutputImageRawFrame(
+                image=img.tobytes(), size=img.size, format=img.format
+            )
+    return images
+
+
+def load_sounds(sound_files):
+    sounds = {}
+
+    for file in sound_files:
+        # Build the full path to the sound file
+        full_path = os.path.join(script_dir, "../assets", file)
+        # Get the filename without the extension to use as the dictionary key
+        filename = os.path.splitext(os.path.basename(full_path))[0]
+        # Open the sound and convert it to bytes
+        with wave.open(full_path) as audio_file:
+            sounds[filename] = OutputAudioRawFrame(
+                audio=audio_file.readframes(-1),
+                sample_rate=audio_file.getframerate(),
+                num_channels=audio_file.getnchannels(),
+            )
+
+    return sounds